diff --git a/.circleci/config.yml b/.circleci/config.yml index 1daa55588b..dec06af98a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -228,8 +228,8 @@ jobs: name: Run migrations to create database command: scripts/run-local-exec migrations run - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - run: name: Export the seed the publish embeds command: scripts/run-local-exec export-seed rundir/seed.db @@ -315,8 +315,8 @@ jobs: name: Run migrations to create database command: scripts/run-local-exec migrations run - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - assert-clean-worktree # Just the host build, on main as much as anywhere else. This is a @@ -379,27 +379,48 @@ jobs: - run: name: Run migrations to create database command: scripts/run-local-exec migrations run + # Keyed on the PIN, not on the week: a pinned package set is immutable, so this hits on every + # release build until someone re-pins. Costs nothing when there is no pin -- `package-set.txt` + # then says `commit unset`, the key is stable, and the cached rundir is simply not used + # because the step below reloads instead. + - restore_cache: + keys: + - v1-seed-{{ checksum "package-set.txt" }} - run: - name: Load packages into database - command: scripts/build/reload-packages + name: Get the package set (the pin, or packages/) + command: scripts/build/prepare-package-set - run: name: Export the seed command: | - sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true - scripts/run-local-exec export-seed rundir/seed.db + # A no-op on the pinned path, where the fetched file already IS the seed and is settled + # out of WAL. On the reload path the store is live and this is the cut. + if [[ -f rundir/seed.db ]] && [[ "$(awk '$1 == "commit" { print $2 }' package-set.txt)" != "unset" ]]; then + echo "using the fetched seed at the pin" + else + sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true + scripts/run-local-exec export-seed rundir/seed.db + fi + - save_cache: + paths: + - rundir/seed.db + key: v1-seed-{{ checksum "package-set.txt" }} - persist_to_workspace: root: "." paths: - rundir/seed.db # Hand over the hash file too, not just the seed. # - # `reload-packages` above populates it, but it is gitignored, so the - # build jobs' own checkout has only the empty one MSBuild creates. - # They then embed THAT, and every ref in the shipped binary resolves - # to "" -> `FnNotFound` on every command. It goes unnoticed inside a - # source tree, because there the file on disk wins over the embedded - # copy and the first run regenerates it. A user running the artifact - # anywhere else has only the embedded copy. + # It is tracked now, so a checkout already has it; this carries the + # one the step above just PRODUCED, which is the one that matches the + # seed beside it. On the pinned path those can differ, and when they + # do the difference is the finding: the pin's package set is not the + # one this kernel was built against, and `assert-clean-worktree` in + # the build jobs is what says so. + # + # Embedding a mismatched one is not a loud failure. Every ref in the + # shipped binary resolves to "" -> `FnNotFound` on every command, and + # only OUTSIDE a source tree, because inside one the file on disk + # wins over the embedded copy and the first run regenerates it. - backend/src/LibExecution/package-ref-hashes.txt # All four linux artifacts from one x64 runner. The cross-compiles work diff --git a/.gitignore b/.gitignore index 017e54b430..6f6a90ee56 100644 --- a/.gitignore +++ b/.gitignore @@ -24,10 +24,6 @@ backend/src/Wasm/publish/ backend/src/Wasm/wwwroot/packages.snapshot darklang-repl.zip -# Generated by reload-packages. Untracked on purpose: two branches that both touch packages -# produce conflicting hashes, and a rebase on GitHub cannot regenerate it -- you would have to pull, -# reload and commit by hand every time. -backend/src/LibExecution/package-ref-hashes.txt # Native SQLite archives for AOT — built by scripts/build/build-sqlite.sh # from the amalgamation, cached locally + in CI. diff --git a/AGENTS.md b/AGENTS.md index 9b84febb4b..543751f3c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,24 @@ alter the serialized package format, and there's no cheap way to ask whether thi did. Narrowing it is the biggest remaining win in the loop, and it's entangled with `package-ref-hashes.txt`, so coordinate before starting. +## Where the package set comes from + +`package-set.txt` at the root says which of two, and it ships `commit unset`, which +means the first: + + commit unset built from `packages/` by reloading it, as always + commit fetched as a seed from a package server, at that commit + +`scripts/build/prepare-package-set` is the one place that answers that question, and +CI's package-reloading jobs go through it. `scripts/packages/pin` writes the pin. The +pinned path has never run against a deployed server, so treat it as +written-and-unverified until it has. + +**`docs/package-workflow.md` is the how-to**: adding a builtin and calling it from +Dark, referencing a new package type or fn from F#, what your coworker does to build +your branch, publishing, the pin, format changes, and what will bite. `dark docs +packages` is the short version from inside the CLI. + The container builds once when it starts. Rebuild-on-save is available but off by default, because a five-file change under a watcher pays for five rebuilds, four of them on half-finished states that produce real-looking failures: @@ -321,11 +339,42 @@ op log directly. ## Gotchas +**The test lock.** `run-backend-tests` refuses if another run holds `rundir/test.lock`. Wait for +it. Do not clear it with a broad `pkill -f "out/Tests"`: that pattern matches every sibling clone +on this machine and will kill somebody else's suite. Scope it to the clone if you must +(`pkill -f "boot-migrate/backend/Build/out/Tests"`). + +**`Stdlib.Sqlite` parameters are `@p0`, `@p1`, not `?`.** With `?` nothing matches and nothing +errors, so a cache silently never fills. + +**A new CLI command joins the registry sweep the day it is registered**, and the sweep runs every +command with a bogus argument. An expensive command therefore taxes the whole suite; `grep` cost +nine minutes until it learned to refuse an unscoped search. + +**Dark syntax traps.** No `let private`. No `rec` keyword. The list separator is `,`. A comment +inside a list literal breaks the parser. Parenthesise a piped qualified call: +`(Mod.f x) |> ...`. + +**Measure the artifact people actually run.** Debug, `publish -c Release`, R2R and AOT differ by +about 25x on startup. Three separate wrong conclusions in one week came from measuring the wrong +one. + **PackageRefs stale hash.** `backend/src/LibExecution/package-ref-hashes.txt` isn't in git. Empty is tolerated; non-empty with a missing key crashes at startup with "PackageRefs: X hash not found". After adding a ref: `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages` +It IS tracked, and it is a projection of the store, which is the awkward combination it has to +be: committing it is what makes a kernel entry point changing identity visible in review, and +`assert-clean-worktree` is what enforces it. So a PR that moves one of the 206 hashes has to +carry the regenerated file. + +**Resolving a conflict in it: regenerate, never hand-merge.** Two branches that both touch +packages will conflict here, and the lines are content hashes, so picking sides is meaningless. +`git checkout --theirs` it, then `./scripts/build/reload-packages` (or, on a store that came +from a seed and has no `packages/` to reload, `scripts/run-local-exec refs generate`) and commit +what that produces. + **Name resolution in test files.** `backend/testfiles/` is parsed with owner "Tests", so `Darklang.*` names need full qualification or the `Stdlib.` shortcut. `Stdlib.Json.ParseError.toString` and `Darklang.SCM.Branch.mainBranchId` resolve; `SCM.Branch.mainBranchId` doesn't. Impl: diff --git a/backend/migrations/schema/07-names.sql b/backend/migrations/schema/07-names.sql index 68078e4b59..ec6925c89c 100644 --- a/backend/migrations/schema/07-names.sql +++ b/backend/migrations/schema/07-names.sql @@ -112,6 +112,28 @@ CREATE TABLE IF NOT EXISTS package_dependencies ( ); CREATE INDEX IF NOT EXISTS idx_package_dependencies_depends_on ON package_dependencies(depends_on_hash); + +-- Which BUILTINS an item's body calls. A separate table from `package_dependencies` because a +-- builtin edge is a different kind of thing: a builtin is not content-addressed, it is a (name, +-- version) in whatever kernel you are running, so there is no hash to join on and none of the +-- location columns apply. +-- +-- This is what lets a store say what KERNEL it needs. Without it you can ask which package items +-- reference each other and not which builtins they call, so half of the kernel/package-set +-- interface is invisible and a builtin can be deleted out from under code that calls it. The +-- checks that used to answer this grepped `.dark` text off disk, which stops being possible the +-- day packages come from a seed rather than a tree. +-- +-- Derived, like `package_dependencies`: rebuilt by the fold, dropped by `Seed.export`. +CREATE TABLE IF NOT EXISTS package_builtin_deps ( + item_hash TEXT NOT NULL, + builtin_name TEXT NOT NULL, + builtin_version INTEGER NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_package_builtin_deps_unique + ON package_builtin_deps(item_hash, builtin_name, builtin_version); +CREATE INDEX IF NOT EXISTS idx_package_builtin_deps_name + ON package_builtin_deps(builtin_name); CREATE INDEX IF NOT EXISTS idx_package_dependencies_item ON package_dependencies(item_hash); -- Partial index for the propagation query: "who depends on this diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs index b7a2ebf0bd..8bd5285fe5 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs @@ -500,8 +500,38 @@ let fns (pm : PT.PackageManager) : List = deprecated = NotDeprecated } - // RELAY store: bulk-insert ops + record ownership (owner) in one transaction, NO fold - // (a relay serves blobs, not projections). The perf path for a relay recording pushes. + // SERVER store: bulk-insert ops + record ownership in one transaction, THEN fold. + // + // It folded nothing until 2026-09-12, on the grounds that a server serves blobs rather than + // projections. That also meant it could not see what it hosted: `/m` showed "Nothing here" + // for packages every client had, a seed could not be cut from the hosted set, and pushing new + // code to a server could never change what it ran. All three are wanted, so it folds. + // Asked BEFORE storing, so a refusal can be a refusal rather than a server error. The same + // rule is enforced inside `storeOpsWithOwner` as the backstop -- this exists so the answer can + // carry a status code and a list of names, not so the rule lives in two places. + { name = fn "scmReservedBindings" 0 + typeParams = [] + parameters = + [ Param.make + "records" + (TList(TTuple(TString, TString, [ TString ]))) + "(id, blobHex, originTs) triples" ] + returnType = TList TString + description = + "The reserved names these ops would bind into this store's main, and that it will not accept. Empty means the push is fine." + fn = + (function + | _, _, _, [| DList(_, records) |] -> + uply { + let! names = LibDB.Inserts.reservedBindingsIn (opRecords records) + return Dval.list KTString (names |> List.map Dval.string) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + callEffects = set [ Effect.PackageRead ] + deprecated = NotDeprecated } + { name = fn "scmStoreOps" 0 typeParams = [] parameters = @@ -524,6 +554,9 @@ let fns (pm : PT.PackageManager) : List = uply { try let! n = LibDB.Inserts.storeOpsWithOwner owner (opRecords records) + // Fold what just arrived, so the projection a seed and `/m` read is current. Cheap: + // ~116us an op, and a push is tens of ops. + let! _ = LibDB.Seed.applyUnappliedOps () return resultOk (Dval.int (bigint n)) with ex -> return resultError (Dval.string ex.Message) diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs index a82599ad2a..fa01a51dbb 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs @@ -6,6 +6,8 @@ open LibExecution.Effects module Dval = LibExecution.Dval module Builtin = LibExecution.Builtin +module C2DT = LibExecution.CommonToDarkTypes +module D = LibExecution.DvalDecoder open Builtin.Shortcuts @@ -13,19 +15,25 @@ open Builtin.Shortcuts let fns : List = [ { name = fn "pmSeedExport" 0 typeParams = [] - parameters = [ Param.make "outputPath" TString "" ] + parameters = + [ Param.make "outputPath" TString "" + Param.make + "upToCommit" + (TypeReference.option TString) + "cut the seed at this commit and its ancestors, so the same commit yields the same ops however far the store has moved since; `None` takes everything committed" ] returnType = TypeReference.result TUnit TString description = "Export a minimal seed.db from the current database" fn = let resultOk = Dval.resultOk KTUnit KTString let resultError = Dval.resultError KTUnit KTString (function - | state, vm, _, [| DString outputPath |] -> + | state, vm, _, [| DString outputPath; upToCommit |] -> uply { try let outputPath = LibExecution.Host.normalizeFilePath outputPath LibExecution.PermissionCheck.requireFileWrite state vm outputPath - do! LibDB.Seed.export outputPath + let upToCommit = C2DT.Option.fromDT D.string upToCommit + do! LibDB.Seed.exportAt outputPath upToCommit return resultOk DUnit with ex -> return resultError (DString ex.Message) diff --git a/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs b/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs index 19d8d0a1a1..564209247b 100644 --- a/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs +++ b/backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs @@ -12,6 +12,7 @@ open LibExecution.Effects open LibExecution.Builtin.Shortcuts module Dval = LibExecution.Dval +module VT = LibExecution.ValueType /// Refuse a store-level operation unless every non-root frame is bundled Darklang @@ -101,6 +102,82 @@ let fns () : List = callEffects = set [ Effect.PackageWrite ] deprecated = NotDeprecated } + // Move the store to the op-log format this build writes, and put back the copy that made. + // + // First-party only, for the same reason as backup/restore: this rewrites every op blob in the + // log and drops every projection. A guest holding package-write must not reach it through a + // wrapper. + // + // A no-op in practice until the first real format bump -- `from` and `to` are equal, and it + // says so rather than doing anything. The mechanism exists now so the bump is not also the + // first time the migration runs. + { name = fn "pmStoreUpgrade" 0 + typeParams = [] + parameters = [ Param.make "unit" TUnit "" ] + returnType = + TypeReference.result + (TTuple(TInt64, TInt64, [ TInt64; TInt64; TString ])) + TString + description = + "Rewrites this store's op log into the format this build writes. Ok is (from, to, rewritten, unreadable, backupPath)." + fn = + let okKT = + KTTuple( + VT.known KTInt64, + VT.known KTInt64, + [ VT.known KTInt64; VT.known KTInt64; VT.known KTString ] + ) + (function + | state, vm, _, [| DUnit |] -> + uply { + requireBundledCaller state vm "pmStoreUpgrade" + let! result = LibDB.StoreUpgrade.upgrade () + match result with + | Ok r -> + return + Dval.resultOk + okKT + KTString + (DTuple( + DInt64(int64 r.from), + DInt64(int64 r.to_), + [ DInt64(int64 r.rewritten) + DInt64(int64 r.unreadable) + DString r.backup ] + )) + | Error e -> return Dval.resultError okKT KTString (DString e) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + // `Native` alongside the writes: this opens SQLite directly to run the rewrite in one + // transaction, which a path rule alone cannot confine. + callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ] + deprecated = NotDeprecated } + + { name = fn "pmStoreRollback" 0 + typeParams = [] + parameters = + [ Param.make "target" TInt64 "the format version that was upgraded TO" ] + returnType = TypeReference.result TString TString + description = + "Restores the copy `pmStoreUpgrade` took on its way to . Ok is the path restored from." + fn = + (function + | state, vm, _, [| DInt64 target |] -> + uply { + requireBundledCaller state vm "pmStoreRollback" + let! result = LibDB.StoreUpgrade.rollback (uint32 target) + match result with + | Ok path -> return Dval.resultOk KTString KTString (DString path) + | Error e -> return Dval.resultError KTString KTString (DString e) + } + | _ -> incorrectArgs ()) + sqlSpec = NotQueryable + previewable = Impure + callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ] + deprecated = NotDeprecated } + // Whether a write secret is stored for a relay, WITHOUT handing it over. // // `dark sync setup` needs to know if one is already there, so that pressing Enter keeps it rather diff --git a/backend/src/LibDB/DependencyExtractor.fs b/backend/src/LibDB/DependencyExtractor.fs index 4c8ef0158f..9534a37bea 100644 --- a/backend/src/LibDB/DependencyExtractor.fs +++ b/backend/src/LibDB/DependencyExtractor.fs @@ -43,9 +43,19 @@ type private Work = | MatchCase of PT.MatchCase | PipeExpr of PT.PipeExpr -let private extract (roots : List) : List = +/// A BUILTIN this item's body calls. Not a `Dependency`: a builtin is not content-addressed, it is +/// a (name, version) in whatever kernel you are running, so there is no hash to depend ON. +/// +/// Collected by the same walk, because walking the AST twice to ask two questions about the same +/// nodes is how the two answers drift apart. +type BuiltinDependency = { name : string; version : int } + +let private extract + (roots : List) + : List * List = let work = System.Collections.Generic.Stack() let mutable dependencies : List = [] + let mutable builtins : List = [] let pushInOrder (items : List) : unit = items |> List.rev |> List.iter work.Push @@ -236,6 +246,17 @@ let private extract (roots : List) : List = | PT.EFnName(_, nr) -> addNameResolution nr PT.ItemKind.Fn PackageItem.fnPackageHash + // `fnPackageHash` answers `None` for a builtin, so the package walk drops it. This is the + // only place the call is visible, and it is what a store needs to be able to say which + // kernel it requires. + match nr.resolved with + | Ok resolved -> + match resolved.name with + | PT.FQFnName.Builtin b -> + builtins <- { name = b.name; version = b.version } :: builtins + | PT.FQFnName.Package _ -> () + | Error _ -> () + | PT.ELambda(_, _, body) -> work.Push(Expr body) | PT.EInfix(_, _, lhs, rhs) -> @@ -265,11 +286,11 @@ let private extract (roots : List) : List = work.Push(Expr next) work.Push(Expr first) - List.rev dependencies + (List.rev dependencies, List.rev builtins |> List.distinct) /// Extract all references from an expression without recursive stack use. -let extractFromExpr (expr : PT.Expr) : List = extract [ Expr expr ] +let extractFromExpr (expr : PT.Expr) : List = fst (extract [ Expr expr ]) /// Extract all references from a function definition @@ -282,6 +303,7 @@ let extractFromFn (fn : PT.PackageFn.PackageFn) : List = |> List.map (fun parameter -> TypeRef parameter.typ)) @ [ TypeRef fn.returnType ] ) + |> fst |> List.distinct @@ -294,12 +316,13 @@ let extractFromFnSignature (fn : PT.PackageFn.PackageFn) : List = |> List.map (fun parameter -> TypeRef parameter.typ)) @ [ TypeRef fn.returnType ] ) + |> fst |> List.distinct /// Extract all references from a value definition let extractFromValue (value : PT.PackageValue.PackageValue) : List = - extract [ Expr value.body ] |> List.distinct + extract [ Expr value.body ] |> fst |> List.distinct /// Extract all references from a type definition @@ -315,4 +338,18 @@ let extractFromType (typ : PT.PackageType.PackageType) : List = |> List.collect (fun case -> case.fields |> List.map (fun field -> TypeRef field.typ)) - extract roots |> List.distinct + extract roots |> fst |> List.distinct + + +/// The BUILTINS a function's body calls. Values and types cannot call one -- a type declaration has +/// no body, and a value's is evaluated through the same walk, so this is the fn entry point only +/// until that stops being true. +let builtinsInFn (fn : PT.PackageFn.PackageFn) : List = + snd (extract [ Expr fn.body ]) + + +/// The BUILTINS a value's body calls. A `val` body is an expression like any other. +let builtinsInValue + (value : PT.PackageValue.PackageValue) + : List = + snd (extract [ Expr value.body ]) diff --git a/backend/src/LibDB/Inserts.fs b/backend/src/LibDB/Inserts.fs index d65e67bf98..da34af6bff 100644 --- a/backend/src/LibDB/Inserts.fs +++ b/backend/src/LibDB/Inserts.fs @@ -285,28 +285,18 @@ let rec insertAndApplyOpsWith /// op the store already runs counts 0. Insert with applied=false, fold, then mark applied=true, so a /// mid-fold failure leaves the ops identifiable and retryable. Commit-free: no commit_hash, so every /// op is live. -/// The `owner` field is the first part of a package name, such as -/// `Darklang.Stdlib.List.map`. Names beginning with `Darklang` are treated as -/// bundled first-party code, so only trusted seeding may create those bindings; -/// guest and sync writes reject them. -let reservedOwners : Set = Set.ofList [ "Darklang" ] - -/// The first operation that binds OR unbinds a name under a protected owner. -/// `None` means no protected location is touched. (The old model also had to -/// chase renames unlisting other bindings of a shared hash; that heuristic is -/// gone -- a `SetName` changes exactly its own location, and retiring a name is -/// an explicit `Unbind` -- so the location arms here are the whole surface.) -let reservedOwnerViolation (ops : List) : Option = - let ownersBound (op : PT.PackageOp) : List = - match op with - | PT.PackageOp.SetName(loc, _, _) -> [ loc.owner ] - | PT.PackageOp.Unbind(loc, _) -> [ loc.owner ] - | _ -> [] - ops - |> List.collect ownersBound - |> List.tryFind (fun owner -> Set.contains owner reservedOwners) - |> Option.map (fun owner -> - $"cannot bind a package name under the reserved owner \"{owner}\"; it is reserved for the bundled standard library") +// No reserved owners. `Darklang` used to be refused here for every write that was not trusted +// seeding, which meant guest `run` and arriving sync ops could not bind under it. +// +// Removed deliberately, 2026-09-12: the store is becoming the source, and the only sanctioned +// writer to `Darklang.*` was the reload from `packages/` that the bootstrapping arc deletes. So +// the protection had to go or the standard library would become uneditable. +// +// What it was defending, for whoever puts a security model back: a name binds last-writer-wins +// across the whole store, so anything that could write could rebind anything -- including the +// names the kernel itself resolves through, and including `Darklang.Matter.router`, which is +// what a server serves. The replacement is an op's AUTHORITY, checked at fold time, and it does +// not exist yet. Until it does, anyone who can write can rebind anything. /// Detect a parser placeholder instead of a real content hash. Placeholders /// are empty or contain the package location; real hashes contain only hex @@ -354,16 +344,17 @@ let insertAndApplyOpsAsWip (ops : List) : Task = /// `effective = 1` is the same clause `Queries.getWipOps` carries and for the same reason: ops a client /// pushed to this store are inert, untagged and uncommitted, so without it a discard here deletes data /// this store is only holding for someone else. -/// Safely insert package operations submitted by RUNNING Dark code -- a guest -/// `run`, or ops that arrived over sync. Rejects protected `Darklang` bindings -/// and unstabilized hashes before insertion; trusted seeding does not come -/// through here. +/// Insert package operations submitted by RUNNING Dark code -- a guest `run`, or ops that arrived +/// over sync. Trusted seeding does not come through here. +/// +/// The only check left is the placeholder-hash one, which is a correctness check rather than a +/// permission: a parser that had no store to ask emits a location where a content hash belongs, +/// and storing that would bind a name to something that does not exist. let insertUntrustedOps (ops : List) : Task> = task { - match reservedOwnerViolation ops, placeholderHashViolation ops with - | Some reason, _ - | None, Some reason -> return Error reason - | None, None -> + match placeholderHashViolation ops with + | Some reason -> return Error reason + | None -> let! count = insertAndApplyOpsAsWip ops return Ok count } @@ -373,11 +364,13 @@ let draftDeletes : List = AND op_id IN (SELECT id FROM package_ops WHERE effective = 1 AND commit_hash IS NULL - AND id NOT IN (SELECT op_id FROM op_branches))" + AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners))" "DELETE FROM package_ops WHERE effective = 1 AND commit_hash IS NULL - AND id NOT IN (SELECT op_id FROM op_branches)" ] + AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners)" ] /// Every main op and what it wrote, EXCEPT the ids in `keep`: the ops this build cannot decode, which /// the caller has read by id. Deleting those would delete a peer's committed op for good because this @@ -402,8 +395,8 @@ let wholeMainDeletes (keep : Set) : List = // // Main only. A branch's rows are keyed by its own id, and no main rewrite may touch them. $"DELETE FROM propagation_policy WHERE branch_id = '{PT.BranchId.Main}'" - // `effective = 1`: excludes client-pushed inert ops; see `draftDeletes`. - $"DELETE FROM package_ops WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches){keepUnreadable}" ] + // Hosted ops excluded via `op_owners`; see `draftDeletes` for why. + $"DELETE FROM package_ops WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches) AND id NOT IN (SELECT op_id FROM op_owners){keepUnreadable}" ] /// Main's op ids this build cannot decode. What `wholeMainDeletes` keeps. let unreadableMainOpIds () : Task> = @@ -567,6 +560,94 @@ let importOpsBulk } +/// The owners a SERVER will not let a push bind into its main. +/// +/// `reservedOwners` used to sit on every write path, including local authoring, and that was +/// wrong -- your own machine is yours, and the check was deleted from all three. This is the same +/// idea at the one edge where it belongs: what a shared server's MAIN will accept from a stranger +/// over HTTP. +/// +/// Why here and not everywhere. Main is what the pin names, what every fetch gets, and what the +/// server resolves its own router through. A push that binds `Darklang.*` there changes what the +/// server serves, for everyone, with no review. Your own namespace is yours to publish to freely: +/// somebody who has just logged in and written a function should be able to share it without a +/// branch, a PR or a person. +/// +/// It stops ACCIDENTS rather than attacks -- the pushed owner is an unsigned string, so this is +/// about what a namespace accepts, not about who is asking. A branch push is unaffected: a branch +/// is isolated, nobody runs it, and review is what moves it to main. +let reservedOnServerMain : Set = Set.ofList [ "Darklang" ] + + +/// The reserved names a batch of ops would bind, if any. Empty means the push is fine. +let private reservedBindings (ops : List) : List = + ops + |> List.choose (fun op -> + let loc = + match op with + | PT.PackageOp.SetName(loc, _, _) -> Some loc + | PT.PackageOp.Decision(_, loc, _, PT.DecisionKind.Override _) -> Some loc + | _ -> None + + match loc with + | Some loc when Set.contains loc.owner reservedOnServerMain -> + let modules = String.concat "." loc.modules + Some( + if modules = "" then + $"{loc.owner}.{loc.name}" + else + $"{loc.owner}.{modules}.{loc.name}" + ) + | _ -> None) + |> List.distinct + + +/// The reserved names these records would bind into this store's main, ignoring ops it already +/// has -- those are content-addressed no-ops whatever they bind. +/// +/// Ignoring them is not an optimisation. `dark push` sends the whole log, and every client's log +/// carries the `Darklang.*` baseline it was seeded with, so checking the raw batch refuses the +/// first push anybody ever makes -- including one that only adds to their own namespace. +let reservedBindingsIn + (records : List) + : Task> = + task { + let valid = + records + |> List.choose (fun (id, blobHex, _) -> + try + Some(System.Guid.Parse id, System.Convert.FromHexString blobHex) + with _ -> + None) + + if List.isEmpty valid then + return [] + else + let! existing = + Sql.query + "SELECT id FROM package_ops WHERE id IN (SELECT value FROM json_each(@ids))" + |> Sql.parameters + [ "ids", + Sql.string ( + "[" + + (valid + |> List.map (fun (id, _) -> "\"" + string id + "\"") + |> String.concat ",") + + "]" + ) ] + |> Sql.executeAsync (fun read -> read.string "id") + + let known = existing |> List.map (fun s -> s.ToLowerInvariant()) |> Set.ofList + + return + valid + |> List.filter (fun (id, _) -> + not (Set.contains ((string id).ToLowerInvariant()) known)) + |> List.choose (fun (id, blob) -> BS.PT.PackageOp.tryDeserialize id blob) + |> reservedBindings + } + + /// RELAY store path: bulk-insert the pushed ops AND record ownership (op_id, owner) in ONE /// transaction. Unlike importOpsBulk this does NOT fold: a relay serves op blobs, not projections. /// The op_owners rows let it serve "your stuff" back by identity. Malformed records are skipped, @@ -597,41 +678,106 @@ let storeOpsWithOwner if List.isEmpty valid then return 0L else - let opRows = + + // Only ops this store does NOT already have can change anything -- ops are content + // addressed, so a re-push of one already here is a no-op whatever it binds. + // + // This is not an optimisation, it is the difference between a working rule and one that + // refuses everybody. `dark push` sends the whole log, and every client's log carries the + // `Darklang.*` baseline it was seeded with. Checking the raw batch refuses the first push + // anyone ever makes, including one that only adds to their OWN namespace. Measured: the + // newcomer case failed on `Darklang.Internal.Test.WTTest`, which they had never touched. + let! existing = + Sql.query + "SELECT id FROM package_ops WHERE id IN (SELECT value FROM json_each(@ids))" + |> Sql.parameters + [ "ids", + // Built by hand, as the merge path does: the reflection serializer is disabled + // under AOT. + Sql.string ( + "[" + + (valid + |> List.map (fun (id, _, _) -> "\"" + string id + "\"") + |> String.concat ",") + + "]" + ) ] + |> Sql.executeAsync (fun read -> read.string "id") + + let known = + existing |> List.map (fun s -> s.ToLowerInvariant()) |> Set.ofList + + // Refuse the WHOLE batch, before storing any of it. A push is one act to the person making + // it, and half of it landing is worse than none: the half that lands is live immediately, + // and they have no way to know which half. + let reserved = valid - |> List.map (fun (id, blob, ts) -> - [ "id", Sql.uuid id - "op_blob", Sql.bytes blob - "origin_ts", Sql.string ts ]) - - // `effective = 0`: in the log, NEVER folded into this store's own main. Queued-for-folding - // is not enough, since `growIfNeeded` folds everything `applied = 0 AND effective = 1` on - // the next startup. A client pushes its whole log, package tree included, and names bind - // last-writer-wins over the whole store -- `Darklang.Matter.router` among them -- so anyone - // who could write to a relay could change what that relay itself runs. Hosted ops are DATA: - // the relay serves the blobs back verbatim and its own code stays what its binary seeded. - let insertOps = - "INSERT OR IGNORE INTO package_ops (id, op_blob, applied, effective, origin_ts) - VALUES (@id, @op_blob, 0, 0, @origin_ts)" - - let statements = - if owner = "" then - [ (insertOps, opRows) ] - else - let ownerRows = - valid - |> List.map (fun (id, _, _) -> - [ "op_id", Sql.uuid id; "owner", Sql.string owner ]) - - let insertOwners = - "INSERT OR IGNORE INTO op_owners (op_id, owner) VALUES (@op_id, @owner)" - - [ (insertOps, opRows); (insertOwners, ownerRows) ] - - // One transaction; the ops-insert counts come first (statement order), so truncate to the - // op rows to report NEW ops rather than owner rows. - let affected = Sql.executeTransactionSync statements - return affected |> List.truncate (List.length opRows) |> List.sumBy int64 + |> List.filter (fun (id, _, _) -> + not (Set.contains ((string id).ToLowerInvariant()) known)) + |> List.choose (fun (id, blob, _) -> + BS.PT.PackageOp.tryDeserialize id blob) + |> reservedBindings + + if not (List.isEmpty reserved) then + let shown = reserved |> List.truncate 5 |> String.concat ", " + + let andMore = + if List.length reserved > 5 then + $" (and {List.length reserved - 5} more)" + else + "" + + return + Exception.raiseInternal + ($"this server does not accept pushes that bind {shown}{andMore} into its main. " + + "That namespace is reviewed: push a branch instead (`dark branch push`), and it " + + "lands on main when the change is merged. Your own namespace takes a plain " + + "`dark push`.") + [] + else + + let opRows = + valid + |> List.map (fun (id, blob, ts) -> + [ "id", Sql.uuid id + "op_blob", Sql.bytes blob + "origin_ts", Sql.string ts ]) + + // `effective = 1`: pushed ops ARE folded into this store's main, like any other arriving + // op. The fold happens in the builtin that calls this, and `growIfNeeded` finishes any + // that were interrupted. + // + // This used to be `effective = 0`, so hosted ops were inert data the server served back + // verbatim and never ran. That was a privilege-escalation defence: a name binds + // last-writer-wins across the store, `Darklang.Matter.router` included, so anyone who + // could push could change what the server itself runs. + // + // Given up deliberately, 2026-09-12, with `reservedOwners`. A server that cannot fold + // cannot serve a seed of what it hosts, cannot show it in `/m`, and cannot be deployed to + // by pushing -- and all three are wanted. The replacement is an op's AUTHORITY checked at + // fold time, and it does not exist yet, so for now anyone who can push can rebind + // anything here. + let insertOps = + "INSERT OR IGNORE INTO package_ops (id, op_blob, applied, effective, origin_ts) + VALUES (@id, @op_blob, 0, 1, @origin_ts)" + + let statements = + if owner = "" then + [ (insertOps, opRows) ] + else + let ownerRows = + valid + |> List.map (fun (id, _, _) -> + [ "op_id", Sql.uuid id; "owner", Sql.string owner ]) + + let insertOwners = + "INSERT OR IGNORE INTO op_owners (op_id, owner) VALUES (@op_id, @owner)" + + [ (insertOps, opRows); (insertOwners, ownerRows) ] + + // One transaction; the ops-insert counts come first (statement order), so truncate to the + // op rows to report NEW ops rather than owner rows. + let affected = Sql.executeTransactionSync statements + return affected |> List.truncate (List.length opRows) |> List.sumBy int64 } diff --git a/backend/src/LibDB/LibDB.fsproj b/backend/src/LibDB/LibDB.fsproj index b8d213c233..3ee532e216 100644 --- a/backend/src/LibDB/LibDB.fsproj +++ b/backend/src/LibDB/LibDB.fsproj @@ -51,6 +51,7 @@ + diff --git a/backend/src/LibDB/PackageManager.fs b/backend/src/LibDB/PackageManager.fs index 64c5d72ddc..1c5a49e9ba 100644 --- a/backend/src/LibDB/PackageManager.fs +++ b/backend/src/LibDB/PackageManager.fs @@ -1,6 +1,8 @@ module LibDB.PackageManager open Prelude +open Fumble +open LibDB.Sqlite open LibExecution.ProgramTypes module RT = LibExecution.RuntimeTypes @@ -630,6 +632,276 @@ let mutable private currentBranchIdOpt : Option = None /// Delta ops for branches OTHER than the active one, loaded on demand. Bounded by how many /// branches a process actually asks about, which for a CLI is one or two. +/// Teach `PackageRefs` to resolve the kernel's seventeen FN refs against this store by NAME. +/// +/// Installed here because `PackageRefs` lives in `LibExecution`, which knows nothing about a +/// store: `LibDB` depends on it, not the reverse, so the dependency has to be handed down rather +/// than reached for. +/// +/// Main's committed projection only, and by design. These are the entry points the kernel calls +/// into Dark through, so resolving them through a branch overlay would mean the binary ran a +/// branch's parser the moment you stood on one. Returns `None` for anything not bound here, and +/// `PackageRefs` falls back to the pinned hash, so a store that predates a ref behaves as before. +/// +/// Whether two versions of a fn are interchangeable to an F# caller. +/// +/// The kernel calls these seventeen and never takes one apart, so what has to hold is the +/// SIGNATURE: parameter types in order, the return type, and how many type parameters. Names, +/// descriptions and the body are the author's business. +let private sameSignature + (expected : PT.PackageFn.PackageFn) + (candidate : PT.PackageFn.PackageFn) + : bool = + let paramTypes (fn : PT.PackageFn.PackageFn) = + fn.parameters |> NEList.toList |> List.map (fun p -> p.typ) + + paramTypes expected = paramTypes candidate + && expected.returnType = candidate.returnType + && List.length expected.typeParams = List.length candidate.typeParams + +/// The net. Without it a bad rebind of `Cli.executeCliCommand` is a CLI that cannot start, and +/// the only way back is an environment variable you have to know exists. With it, a rebind whose +/// signature does not match what this build compiles against is refused, loudly, and the pinned +/// version is used: a wrong edit degrades rather than bricks. +/// What the CURRENT BRANCH's overlay binds `Darklang..` to, if anything. +/// +/// `locations` is main's projection and has no branch column -- authoring two items on a branch +/// leaves zero rows there and two in `op_branches` -- so a store query cannot see a branch's work. +/// The overlay is already in memory as an op list, so this folds it directly: latest binding wins +/// per location, the same rule `createInMemoryOver` applies and the same one main's fold applies. +/// +/// Only TYPES go through this, and the asymmetry is deliberate. Resolving a FN ref through an +/// overlay would mean the binary runs a branch's parser or pretty-printer the moment you stand on +/// that branch, which is a real hazard and not one you opted into by switching. A type declaration +/// is inert: resolving it decides what a value is TAGGED with, and executes nothing. So a branch +/// may lend the kernel its types and may not lend it its code. +let private overlayTypeBinding + (modules : string list) + (name : string) + : string option = + let wanted : PT.PackageLocation = + { owner = "Darklang"; modules = modules; name = name } + + branchOverlayOps + |> List.fold + (fun acc op -> + match op with + | PT.PackageOp.SetName(loc, PT.PackageType(Hash h), _) when loc = wanted -> + Some h + | PT.PackageOp.Decision(_, + loc, + _, + PT.DecisionKind.Override(PT.PackageType(Hash h))) when + loc = wanted + -> + Some h + // A name bound to something that is not a type, or unbound outright, means the branch has + // taken this name away from the kernel. Fall back rather than keep an older binding. + | PT.PackageOp.SetName(loc, _, _) when loc = wanted -> None + | PT.PackageOp.Unbind(loc, _) when loc = wanted -> None + | _ -> acc) + None + + +/// Every Darklang-owned binding the CURRENT BRANCH's overlay adds or changes, as +/// (modules, name, itemType, hash). +/// +/// For `PackageRefsGenerator`, which builds the pinned hash file from `locations` and therefore has +/// the same main-only blind spot the resolver had. A branch that authors a type the kernel +/// references has to be able to produce a hash file naming it, or the F# on that git branch cannot +/// be built by anybody else. +let overlayDarklangBindings () : List = + let bindings = + System.Collections.Generic.Dictionary() + + for op in branchOverlayOps do + let apply loc target = + match target with + | PT.PackageType(Hash h) -> bindings[loc] <- ("type", h) + | PT.PackageValue(Hash h) -> bindings[loc] <- ("value", h) + | PT.PackageFn(Hash h) -> bindings[loc] <- ("fn", h) + + match op with + | PT.PackageOp.SetName(loc, target, _) -> apply loc target + | PT.PackageOp.Decision(_, loc, _, PT.DecisionKind.Override target) -> + apply loc target + | PT.PackageOp.Unbind(loc, _) -> bindings.Remove loc |> ignore + | _ -> () + + bindings + |> Seq.filter (fun kv -> kv.Key.owner = "Darklang") + |> Seq.map (fun kv -> + let (itemType, hash) = kv.Value + (String.concat "." kv.Key.modules, kv.Key.name, itemType, hash)) + |> List.ofSeq + + +/// What `Darklang..` of this kind binds to on main, CACHED. +/// +/// Through `Caching.withCache`, which the fold clears, so this expires exactly when a rebinding +/// could have changed the answer. Caching is not a nicety here. These sit under Option and Result +/// construction, so they run constantly, and a raw query per call measured at **46% more +/// allocation on the reference workload** -- 9.4 MB to 13.6 MB -- with the wall clock barely +/// moving, which is why time would not have caught it. +/// +/// `withCache` does not cache `None`, which is right: a name the store does not bind yet may bind +/// later, and a cache has no way to hear about that. +let private kernelHashByName = + Caching.withCache + (fun ((itemType : string), (modulesStr : string), (name : string)) -> + uply { + return! + Sql.query + $"""SELECT item_hash + FROM locations + WHERE owner = 'Darklang' AND modules = @modules AND name = @name + AND item_type = '{itemType}' AND unlisted_at IS NULL AND source != 'unbind' + LIMIT 1""" + |> Sql.parameters + [ "modules", Sql.string modulesStr; "name", Sql.string name ] + |> Sql.executeRowOptionAsync (fun read -> read.string "item_hash") + }) + + +let private resolveKernelFnByName + (modules : string list) + (name : string) + : string option = + try + let modulesStr = String.concat "." modules + + let candidateHash = + (kernelHashByName ("fn", modulesStr, name) |> Ply.toTask).Result + + match candidateHash with + | None -> None + | Some candidate -> + match LibExecution.PackageRefs.pinnedFnHash modules name with + // Nothing pinned to compare against, or the store already agrees with the pin. The second + // is the overwhelmingly common case and costs one string compare. + | None -> Some candidate + | Some pinned when pinned = candidate -> Some candidate + | Some pinned -> + let fnFor (h : string) = (PMPT.Fn.get (PT.Hash h) |> Ply.toTask).Result + + match fnFor pinned, fnFor candidate with + | Some expected, Some actual when sameSignature expected actual -> + Some candidate + | Some _, Some _ -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the store binds Darklang.{modulesStr}.{name} to a version whose signature " + + "is not the one this build expects, so the built-in version is being used instead." + ) + None + | _, None -> + // Bound to content this store does not hold: nothing to call. + LibExecution.PackageRefs.sayOnce ( + $"warning: Darklang.{modulesStr}.{name} is bound to content this store does not have, " + + "so the built-in version is being used instead." + ) + None + | None, _ -> + // The PINNED version is missing from the store. Nothing left to compare against, and + // refusing would leave no version at all, so trust the store. + Some candidate + with _ -> + // No store yet, or one without the table: migrations run before any of this exists, and a + // ref resolved during them must fall back rather than fail the boot. + None + +LibExecution.PackageRefs.resolveFnByName <- resolveKernelFnByName + + +/// Do two type declarations have the same SHAPE? +/// +/// What has to hold is what a value carries: the kind of declaration, the fields or cases in +/// order with their types, and how many type parameters. Descriptions are the author's business, +/// so they are stripped before comparing -- a doc edit must not cost the store its binding. +let private sameDeclaration + (expected : PT.PackageType.PackageType) + (candidate : PT.PackageType.PackageType) + : bool = + let stripDefinition (d : PT.TypeDeclaration.Definition) = + match d with + | PT.TypeDeclaration.Alias t -> PT.TypeDeclaration.Alias t + | PT.TypeDeclaration.Record fields -> + fields + |> NEList.map (fun f -> { f with description = "" }) + |> PT.TypeDeclaration.Record + | PT.TypeDeclaration.Enum cases -> + cases + |> NEList.map (fun c -> + { c with + description = "" + fields = c.fields |> List.map (fun f -> { f with description = "" }) }) + |> PT.TypeDeclaration.Enum + + stripDefinition expected.declaration.definition = stripDefinition + candidate.declaration.definition + && List.length expected.declaration.typeParams = List.length + candidate.declaration.typeParams + + +/// The net for TYPE refs. See `resolveKernelFnByName`; the difference is what gets compared. +/// +/// The case this exists for is the one with NO pin: a type a branch has just authored, which F# +/// on the same git branch wants to reference. There is nothing to compare against, so the store's +/// answer is taken. An existing type whose shape moved falls back to the pin, because that is the +/// shape this build was compiled against and a `DRecord` it builds has to typecheck against it. +let private resolveKernelTypeByName + (modules : string list) + (name : string) + : string option = + try + let modulesStr = String.concat "." modules + + // The branch first, then main. A branch that has authored this name is the thing you are + // standing on, and it is the whole reason F# on a git branch can reference package code + // authored on a dark branch. + let candidateHash = + match overlayTypeBinding modules name with + | Some h -> Some h + | None -> (kernelHashByName ("type", modulesStr, name) |> Ply.toTask).Result + + match candidateHash with + | None -> None + | Some candidate -> + match LibExecution.PackageRefs.pinnedTypeHash modules name with + // Nothing pinned: a type this build has never seen, which is the branch case. + | None -> Some candidate + | Some pinned when pinned = candidate -> Some candidate + | Some pinned -> + let typeFor (h : string) = (PMPT.Type.get (PT.Hash h) |> Ply.toTask).Result + + match typeFor pinned, typeFor candidate with + | Some expected, Some actual when sameDeclaration expected actual -> + Some candidate + | Some _, Some _ -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the store binds the type Darklang.{modulesStr}.{name} to a version whose " + + "shape is not the one this build expects, so the built-in version is being used " + + "instead." + ) + None + | _, None -> + LibExecution.PackageRefs.sayOnce ( + $"warning: the type Darklang.{modulesStr}.{name} is bound to content this store does " + + "not have, so the built-in version is being used instead." + ) + None + | None, _ -> + // The PINNED version is missing. Nothing to compare against, and refusing leaves no + // type at all, so trust the store. + Some candidate + with _ -> + None + +LibExecution.PackageRefs.resolveTypeByName <- resolveKernelTypeByName + +// The ref closures memoize what the store said; the fold is when that can stop being true. +Caching.register LibExecution.PackageRefs.invalidateStoreResolution + + let private otherBranchOps = System.Collections.Concurrent.ConcurrentDictionary>() @@ -791,3 +1063,9 @@ let selectBranch (branchId : PT.BranchId) : unit = else branchOverlayOps <- (Branches.loadDeltaOps branchId).Result currentBranchIdOpt <- Some branchId + + // Moving branches changes what a NAME resolves to, and every cache here answers a question about + // a name. A one-shot `dark` selects its branch before resolving anything and never notices; a + // process that outlives a switch -- the LSP, the REPL, a daemon -- would otherwise answer for the + // branch it started on, forever. + Caching.invalidateAll () diff --git a/backend/src/LibDB/PackageOpPlayback.fs b/backend/src/LibDB/PackageOpPlayback.fs index 5ca304650b..092f91cf9f 100644 --- a/backend/src/LibDB/PackageOpPlayback.fs +++ b/backend/src/LibDB/PackageOpPlayback.fs @@ -33,6 +33,44 @@ open LibDB.PreparedBatch // Dependency table maintenance. // ------------------------------------------------------------------ +/// Record which BUILTINS an item's body calls. +/// +/// Separate from `updateDependencies` because a builtin edge is a different shape: not +/// content-addressed, so there is no hash to depend on and no location. This is the half of the +/// kernel/package-set interface that used to be invisible from the store, and the thing that lets +/// a fetched package set say which kernel it needs. +/// +/// ADDS, never replaces, for the same reason as the package edges: content is immutable, so what a +/// hash calls never changes. +let updateBuiltinDependencies + (ctx : Ctx) + (itemHash : string) + (builtins : List) + : Task = + task { + if List.isEmpty builtins then + () + else + let placeholders = + builtins + |> List.mapi (fun i _ -> $"($item_hash, $bname_{i}, $bver_{i})") + |> String.concat ", " + + let sql = + "INSERT OR IGNORE INTO package_builtin_deps " + + "(item_hash, builtin_name, builtin_version) VALUES " + + placeholders + + do! + exec ctx sql (fun cmd -> + p cmd "$item_hash" itemHash + builtins + |> List.iteri (fun i b -> + p cmd $"$bname_{i}" b.name + p cmd $"$bver_{i}" b.version)) + } + + /// Record what an item's body calls: one row per callee, by hash AND by the name this parse resolved /// it through. /// @@ -257,6 +295,7 @@ let private applyAddValue let refs = DE.extractFromValue value do! updateDependencies ctx hashStr refs + do! updateBuiltinDependencies ctx hashStr (DE.builtinsInValue value) } /// Apply a single AddFn op to the package_functions table. @@ -292,6 +331,7 @@ let private applyAddFn let refs = DE.extractFromFn fn do! updateDependencies ctx hashStr refs + do! updateBuiltinDependencies ctx hashStr (DE.builtinsInFn fn) } /// The `origin_ts` the log stamped on , or None when the log does not hold it. diff --git a/backend/src/LibDB/PackageRefsGenerator.fs b/backend/src/LibDB/PackageRefsGenerator.fs index 8423fd7f55..f0e482e29f 100644 --- a/backend/src/LibDB/PackageRefsGenerator.fs +++ b/backend/src/LibDB/PackageRefsGenerator.fs @@ -48,7 +48,19 @@ let private readExistingFile () : Map = /// Query the DB for all current Darklang-owned locations and write /// `package-ref-hashes.txt` in the source tree. -let generate () : Ply = +/// Compute the kernel's ref hashes from the store, set them in memory, and -- when +/// -- write `package-ref-hashes.txt`. +/// +/// The two are separate because they have different cadences. The hashes must be in MEMORY +/// before values are evaluated, on every reload, or `PackageRefs` lookups resolve to nothing +/// during it. The FILE moves only when the pin does. +/// +/// Writing it on every reload is what made every pair of package-touching branches conflict in a +/// 206-line generated file. It stopped being necessary when type refs started resolving from the +/// store by name: the file is a fallback now, not a contract, so between pin bumps the committed +/// copy is simply correct. A pin bump then produces one reviewable diff naming every identity +/// that moved, which is the signal the file was tracked for. +let generateWith (writeToDisk : bool) : Ply = uply { // Collect all referenced items from PackageRefs _lookup maps let typeRefKeys = @@ -91,7 +103,16 @@ let generate () : Ply = let hash = read.string "item_hash" (buildKey itemType modules name, hash)) - let dbMap = dbRows |> Map.ofList + // The branch's bindings go OVER main's. `locations` is main's projection, so without this a + // hash file generated while standing on a branch describes main and silently omits the very + // items the branch exists to add -- which is exactly the case where the file matters, because + // it is what lets somebody else build the F# that references them. + let dbMap = + PackageManager.overlayDarklangBindings () + |> List.fold + (fun acc (modules, name, itemType, hash) -> + Map.add (buildKey itemType modules name) hash acc) + (dbRows |> Map.ofList) // Preserves entries not found in the DB (e.g. RT types that share hashes with PT types and aren't in // locations), and, via `existingKeys` above, refs this process never registered. @@ -156,7 +177,7 @@ let generate () : Ply = // Write the source-tree file (skip if the directory doesn't exist, // e.g. on installed CLIs where the source tree isn't available) let dir = System.IO.Path.GetDirectoryName(sourceTreePath) - if System.IO.Directory.Exists(dir) then + if writeToDisk && System.IO.Directory.Exists(dir) then System.IO.File.WriteAllLines(sourceTreePath, lines |> Array.ofList) let totalWritten = List.length lines print $" Wrote {totalWritten} package ref hashes to {sourceTreePath}" @@ -171,3 +192,12 @@ let generate () : Ply = for key in missing do print $" - {key}" } + + +/// `generateWith`, writing the file. What `refs generate` and `scripts/packages/pin` call. +let generate () : Ply = generateWith true + + +/// `generateWith`, in memory only. What a package RELOAD calls: the hashes have to be current +/// before values are evaluated, and the file is the pin's to move. +let refreshInMemory () : Ply = generateWith false diff --git a/backend/src/LibDB/Purge.fs b/backend/src/LibDB/Purge.fs index 691365a58f..1d6dc2c1eb 100644 --- a/backend/src/LibDB/Purge.fs +++ b/backend/src/LibDB/Purge.fs @@ -32,6 +32,11 @@ let tables : List = "package_functions" "package_ops" "package_dependencies" + + // Which builtins each item calls. Derived from the bodies in the log by the same walk that + // fills `package_dependencies`, so a purged log has called nothing. + "package_builtin_deps" + "deprecations" // The LWW register for doc comments: what each `UpdateDoc` said and when. A claim about the diff --git a/backend/src/LibDB/Queries.fs b/backend/src/LibDB/Queries.fs index f8d4eba666..5d2f77ec4f 100644 --- a/backend/src/LibDB/Queries.fs +++ b/backend/src/LibDB/Queries.fs @@ -398,10 +398,15 @@ let getDraftOps () : Task> = """ SELECT id, op_blob FROM package_ops - -- effective = 1: excludes client-pushed inert ops; see Inserts.draftDeletes. + -- NOT a hosted op: `op_owners` records who pushed each op TO this store, so a row there is + -- somebody else's work this store is only holding. It is folded like any other op now + -- (a server has to be able to see and serve what it hosts), which is exactly why it has + -- to be excluded HERE: `effective` used to carry that distinction, and a discard that + -- counted a peer's push as this store's draft would delete their data. WHERE effective = 1 AND commit_hash IS NULL AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners) ORDER BY created_at ASC, rowid ASC """ |> Sql.executeAsync (fun read -> @@ -471,9 +476,10 @@ let getWipOps () : Task> = -- Excluding them keeps main authoring's WIP-refresh from sweeping a branch's ops into -- main (re-inserting them effective=1 + folding). Branch isolation. -- - -- effective = 1: excludes client-pushed inert ops; see Inserts.draftDeletes. + -- Hosted ops are excluded for the reason `getDraftOps` above spells out. WHERE effective = 1 AND id NOT IN (SELECT op_id FROM op_branches) + AND id NOT IN (SELECT op_id FROM op_owners) -- rowid breaks ties: created_at is second-resolution and a batch shares it, and the pairing -- downstream (HashStabilization) is by adjacency. ORDER BY created_at ASC, rowid ASC diff --git a/backend/src/LibDB/Releases.fs b/backend/src/LibDB/Releases.fs index b21dc04e2c..959b074b25 100644 --- a/backend/src/LibDB/Releases.fs +++ b/backend/src/LibDB/Releases.fs @@ -245,10 +245,95 @@ let steps : List = print $" release: added `removed` to {List.length rows} stored conflict(s)" } + // What BUILTINS each item calls. New table, so `CREATE TABLE IF NOT EXISTS` in the schema + // would reach an existing store only because the bootstrap replays it, which it does not + // promise to. Named here so the store records having got it. Empty until the next fold + // rebuilds it, which is correct: it is derived. + { name = "20260912_000001_package_builtin_deps" + run = + fun () -> + Sql.query + "CREATE TABLE IF NOT EXISTS package_builtin_deps ( + item_hash TEXT NOT NULL, + builtin_name TEXT NOT NULL, + builtin_version INTEGER NOT NULL)" + |> Sql.executeStatementSync + + Sql.query + "CREATE UNIQUE INDEX IF NOT EXISTS idx_package_builtin_deps_unique + ON package_builtin_deps(item_hash, builtin_name, builtin_version)" + |> Sql.executeStatementSync } + // NEW STEPS GO ABOVE THIS LINE -- `scripts/migrations/new` appends here, and edits nothing else. ] +/// The format the store says it was written in, if it says. +/// +/// Absent means a store older than the stamp, which is every store built before seeds carried one. +/// That is not an error: it predates the field, and its ops are format 1 by construction. +let storedFormat () : Option = + if not (tableExists "store_meta") then + None + else + Sql.query "SELECT value FROM store_meta WHERE key = 'format'" + |> Sql.execute (fun read -> read.string "value") + |> Result.unwrap + |> List.tryHead + |> Option.bind (fun v -> + match System.UInt32.TryParse v with + | true, n -> Some n + | false, _ -> None) + + +/// Say so when a store was written by a NEWER build's format, and stamp one that carries no format +/// yet. +/// +/// The asymmetry is the point. A store BEHIND this build is the migrator's job (it can read an old +/// layout, because every historical reader stays in the binary). A store AHEAD of it cannot be read +/// by trying harder: the layout is one this binary has never seen. +/// +/// SAID, not raised, and the wording matters more than usual, because there is no working command +/// left to recover WITH. The projections hold blobs in the newer layout too, so this build dies on +/// its first package lookup -- which includes resolving the name of the command you typed. So the +/// note names the FILE to move, not a verb to run: a `mv` needs no working binary. +/// +/// `dark store rollback` covers the other case, and the likelier one: you upgraded, you are still +/// on the build that did it, and you want it undone. +let noteFormatSkew () : unit = + match storedFormat () with + | Some n when n > LibSerialization.Binary.BaseFormat.currentVersion -> + System.Console.Error.WriteLine( + $"note: this store is format {n} and this build reads " + + $"{LibSerialization.Binary.BaseFormat.currentVersion}, so its ops cannot be read. Upgrade " + + $"the binary. If this store was upgraded here, the copy from before that is at " + + $"{Sqlite.currentDbPath}.pre-v{n} -- move it back over {Sqlite.currentDbPath}." + ) + | _ -> () + + +/// `noteFormatSkew`, plus the stamp for a store that carries none. +/// +/// Stamping is a WRITE, so it belongs here in the migration path rather than on every open: a store +/// that has run this once carries the stamp from then on. +let private checkFormat () : unit = + noteFormatSkew () + + match storedFormat () with + | Some n when n > LibSerialization.Binary.BaseFormat.currentVersion -> () + | _ -> + // Stamp it, so from here every store says what it is. `INSERT OR REPLACE` rather than a + // conditional: the value is the same whether the row was missing or already right. + Sql.query + "CREATE TABLE IF NOT EXISTS store_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + |> Sql.executeStatementSync + + Sql.query "INSERT OR REPLACE INTO store_meta (key, value) VALUES ('format', @v)" + |> Sql.parameters + [ "v", Sql.string (string LibSerialization.Binary.BaseFormat.currentVersion) ] + |> Sql.executeStatementSync + + let private alreadyRun () : Set = if not (tableExists "system_migrations_v0") then Set.empty @@ -323,6 +408,8 @@ let applySchemaIndexes (schemaSql : string) : unit = /// makes it safe to run against a store of any age, so a new step that skips those has nothing /// checking it. let runPending () : unit = + checkFormat () + // A step's name is its identity in `system_migrations_v0`, so two steps sharing one would run as // one and record as one, silently. Refused here, where every store passes on startup, because no // test constructs this list. diff --git a/backend/src/LibDB/Seed.fs b/backend/src/LibDB/Seed.fs index 3d330221a6..868bf9c7f7 100644 --- a/backend/src/LibDB/Seed.fs +++ b/backend/src/LibDB/Seed.fs @@ -35,6 +35,10 @@ module Permission = LibExecution.Permissions /// context for whatever the command was asked to do, not an event worth repeating before every answer. let mutable private warnedAboutUnreadableOps = false +/// Whether this process has already said that the store's format is ahead of this build's. Once, +/// for the same reason: it is context for the command, not an event. +let mutable private warnedAboutFormatSkew = false + // --------------------- // Export @@ -42,24 +46,39 @@ let mutable private warnedAboutUnreadableOps = false /// Export a seed database to the given output path: copy the full source DB, then strip everything /// that belongs to the machine that built it rather than to the package set (see the DELETEs below). -let export (outputPath : string) : Task = +/// A seed, optionally cut at a COMMIT rather than at now. +/// +/// `Some commit` keeps the ops that commit or one of its ancestors names, and drops the rest, so +/// two people fetching the same commit get the same bytes however far the source has moved since. +/// That immutability is what makes the seed cacheable and the pin reproducible. +/// +/// Ancestry through `commits.parent`, the same walk `revert` uses: a commit names a point in +/// history, not a set of ops. +let exportAt (outputPath : string) (upToCommit : string option) : Task = task { - let sourcePath = LibConfig.Config.dbPath - if System.IO.File.Exists outputPath then System.IO.File.Delete outputPath - // Checkpoint WAL before copying to ensure all data is in the main file - let sourceConnStr = $"Data Source={sourcePath};Mode=ReadOnly;Cache=Private" - use sourceConn = new SqliteConnection(sourceConnStr) - sourceConn.Open() - use checkpointCmd = sourceConn.CreateCommand() - checkpointCmd.CommandText <- "PRAGMA wal_checkpoint(TRUNCATE);" - checkpointCmd.ExecuteNonQuery() |> ignore - sourceConn.Close() - - System.IO.File.Copy(sourcePath, outputPath) - - let connStr = $"Data Source={outputPath};Mode=ReadWriteCreate;Cache=Private" + // Through SQLite's online-backup API, never a file copy. + // + // The store this runs against is LIVE -- a server cuts a seed while serving, and the process + // holding it has a WAL open -- and `data.db` alone is not the store while recent writes sit in + // `data.db-wal`. The previous version checkpointed the source first and then copied the file, + // which cannot work on a long-lived process: the checkpoint went through a ReadOnly connection, + // so the moment there was actually a WAL to fold in it failed with a disk I/O error. + // + // It also reads `Sqlite.connString` rather than the config path, so a test that repoints LibDB + // at its own store exports THAT store rather than the default one. + match Backup.toFile outputPath with + | Error e -> + Exception.raiseInternal $"could not snapshot the store to cut a seed: {e}" [] + | Ok() -> () + + // `Pooling=False`: a pooled connection outlives its `Close`, so on a process that cuts more than + // one seed the second cut is handed a handle to a file the first one has since deleted and + // replaced, and fails with "attempt to write a readonly database". A cut opens one connection and + // happens rarely; pooling buys it nothing. + let connStr = + $"Data Source={outputPath};Mode=ReadWriteCreate;Cache=Private;Pooling=False" use conn = new SqliteConnection(connStr) conn.Open() @@ -77,6 +96,7 @@ let export (outputPath : string) : Task = DELETE FROM package_values; DELETE FROM package_functions; DELETE FROM package_dependencies; + DELETE FROM package_builtin_deps; DELETE FROM deprecations; -- The builder's BRANCHES are not canon. A branch's ops live in `package_ops` at effective = 0 and are @@ -136,14 +156,103 @@ let export (outputPath : string) : Task = """ cleanCmd.ExecuteNonQuery() |> ignore + // Cut at a commit: drop every op the commit's history does not name, and every commit outside + // that history. Runs AFTER the clean above, so it only ever narrows what that already kept. + match upToCommit with + | None -> () + | Some commit -> + use cutCmd = conn.CreateCommand() + cutCmd.CommandText <- + """ + CREATE TEMP TABLE seed_ancestry AS + WITH RECURSIVE ancestry(h) AS ( + SELECT hash FROM commits WHERE hash = $commit + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent <> '' + ) + SELECT h FROM ancestry; + + DELETE FROM package_ops + WHERE commit_hash IS NULL OR commit_hash NOT IN (SELECT h FROM seed_ancestry); + + DELETE FROM commits WHERE hash NOT IN (SELECT h FROM seed_ancestry); + + DROP TABLE seed_ancestry; + """ + cutCmd.Parameters.AddWithValue("$commit", commit) |> ignore + cutCmd.ExecuteNonQuery() |> ignore + + // `cut_at` names a commit even when the caller asked for no cut, so EVERY seed says what it is + // a cut of. Read back after the cut, so it is the tip of what the file actually holds rather + // than the tip of the store it came from. + let cutAt = + use tipCmd = conn.CreateCommand() + tipCmd.CommandText <- + "SELECT hash FROM commits ORDER BY created_at DESC, rowid DESC LIMIT 1" + match tipCmd.ExecuteScalar() with + | null -> "" + | tip -> string tip + + // The stamp every seed and every store carries, so a store can say which cut it came from and + // which build made it. Written here because export is the only thing that knows. + // + // `format` is the op-blob layout version, which is what a migrator keys on: a store two + // formats behind needs two steps, and a store from a NEWER format has to be refused rather + // than misread. + use stampCmd = conn.CreateCommand() + stampCmd.CommandText <- + """ + CREATE TABLE IF NOT EXISTS store_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + DELETE FROM store_meta; + INSERT INTO store_meta (key, value) VALUES + ('format', $format), + ('cut_at', $cutAt), + ('kernel', $kernel), + ('at', $at); + """ + stampCmd.Parameters.AddWithValue( + "$format", + string LibSerialization.Binary.BaseFormat.currentVersion + ) + |> ignore + stampCmd.Parameters.AddWithValue("$cutAt", cutAt) |> ignore + stampCmd.Parameters.AddWithValue("$kernel", LibConfig.Config.buildHash) + |> ignore + stampCmd.Parameters.AddWithValue( + "$at", + System.DateTime.UtcNow.ToString( + "o", + System.Globalization.CultureInfo.InvariantCulture + ) + ) + |> ignore + stampCmd.ExecuteNonQuery() |> ignore + use vacuumCmd = conn.CreateCommand() vacuumCmd.CommandText <- "VACUUM;" vacuumCmd.ExecuteNonQuery() |> ignore + // Out of WAL before anyone gets the file. A seed is SHIPPED -- copied, served over HTTP, + // embedded in a binary -- and in WAL mode the `.db` on its own is not the whole database, so + // whether it is complete depends on when a checkpoint happened to run. `journal_mode=DELETE` + // folds the WAL back in and removes it, which makes the one file the whole seed. + // `LibDB.Sqlite` puts a store back into WAL at open, so nothing downstream loses it. + use settleCmd = conn.CreateCommand() + settleCmd.CommandText <- + "PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE;" + settleCmd.ExecuteNonQuery() |> ignore + conn.Close() } +/// A seed of main as it stands now. +let export (outputPath : string) : Task = exportAt outputPath None + + // --------------------- // Grow // --------------------- @@ -364,6 +473,7 @@ let projectionTables : List = "package_values" "locations" "package_dependencies" + "package_builtin_deps" "deprecations" // Folded from `Decision` ops; nothing else writes it. Being here is what makes it genuinely derived // rather than a second source of truth about the same decisions. @@ -496,6 +606,37 @@ module ValueEvaluationError = | None -> e.message | Some(PT.Hash h) -> $"Value {h} ({e.location}): {e.message}" +/// The values this store may EVALUATE: not the ones somebody else pushed here. +/// +/// Folding an op is inert -- it deserializes and writes projection rows, and runs no guest code. +/// Evaluating a `val` is not: it executes the body. On a client that distinction does not arise, +/// because everything in its store is either its own or something it chose to pull. On a SERVER it +/// is the whole difference between holding somebody's code and running it, and `op_owners` already +/// records which ops arrived by push. +/// +/// So: a value with a pushed binding and no local one is folded, browsable and servable in a seed, +/// and never executed here. Whoever fetches it evaluates it on their own machine, under their own +/// policy, which is where that decision belongs. +/// +/// "and no local one" matters. Content is shared, so a value this store authored can also arrive by +/// push from a peer who wrote the same thing; anything locally bound is still evaluated. +/// +/// `op_owners` is empty on an instance, so both EXISTS clauses are false there and this selects +/// exactly what it selected before. +let private evaluableValues = + """ + pv.rt_dval IS NULL + AND NOT ( + EXISTS (SELECT 1 FROM locations hl + WHERE hl.item_hash = pv.hash + AND hl.op_id IN (SELECT op_id FROM op_owners)) + AND NOT EXISTS (SELECT 1 FROM locations ll + WHERE ll.item_hash = pv.hash + AND ll.op_id NOT IN (SELECT op_id FROM op_owners)) + ) + """ + + /// Evaluate all package values that have NULL rt_dval, under `authority`. /// Multi-pass: values may depend on other values, so we retry until convergence. let evaluateAllValues @@ -535,11 +676,11 @@ let evaluateAllValues let! unevaluatedValues = Sql.query - """ + $""" SELECT pv.hash, pv.pt_def, l.owner, l.modules, l.name FROM package_values pv LEFT JOIN locations l ON l.item_hash = pv.hash AND l.unlisted_at IS NULL - WHERE pv.rt_dval IS NULL + WHERE {evaluableValues} """ |> Sql.executeAsync (fun read -> let hash = Hash(read.string "hash") @@ -657,6 +798,14 @@ let growIfNeeded : Task = task { use _span = Telemetry.span "seed.growIfNeeded" [] + + // Every process that opens the store passes through here, which is the only place a skew + // between the store's format and this build's is certain to be noticed. `Releases.runPending` + // says it too, but a shipped binary reaches that only when it has an embedded seed to unpack. + if not warnedAboutFormatSkew then + warnedAboutFormatSkew <- true + Releases.noteFormatSkew () + let! appliedCount = Telemetry.timeTask "seed.applyOps" [] (fun () -> applyUnappliedOps ()) // The fold above reads effective=1 only, so branch-scoped Decisions (a branch's propagation @@ -670,8 +819,10 @@ let growIfNeeded // NULL forever, and a NULL `rt_dval` reads as "value not found". Evaluate whenever any value is // unevaluated so the store self-heals on startup. let! hasUnevaluatedValues = + // The same predicate `evaluateAllValues` selects with, or a server would take this branch on + // every startup for hosted values it is never going to evaluate. Sql.query - "SELECT EXISTS(SELECT 1 FROM package_values WHERE rt_dval IS NULL) AS has_null" + $"SELECT EXISTS(SELECT 1 FROM package_values pv WHERE {evaluableValues}) AS has_null" |> Sql.executeRowAsync (fun read -> read.int64 "has_null") |> Task.map (fun n -> n > 0L) if appliedCount > 0L then diff --git a/backend/src/LibDB/Sqlite.fs b/backend/src/LibDB/Sqlite.fs index 3080c94d63..d1d52e9e1b 100644 --- a/backend/src/LibDB/Sqlite.fs +++ b/backend/src/LibDB/Sqlite.fs @@ -13,6 +13,17 @@ open Prelude let private connStringFor (path : string) : string = $"Data Source={path};Mode=ReadWriteCreate;Cache=Private;Pooling=true" +/// A connection to a one-shot FILE beside the store: a backup being written, a seed being cut, a +/// file being restored from. +/// +/// Unpooled, which is the whole difference. A pooled connection outlives its `Close`, so a process +/// that touches the same path twice -- a server cutting a seed at one commit, then at another, into +/// a file it deleted in between -- is handed a handle to the file that is no longer there and fails +/// with "attempt to write a readonly database" or a disk I/O error. The live store wants pooling; +/// a file opened once does not. +let private fileConnStringFor (path : string) : string = + $"Data Source={path};Mode=ReadWriteCreate;Cache=Private;Pooling=False" + let private defaultConnString = connStringFor LibConfig.Config.dbPath /// The store this process is actually reading and writing, which is `LibConfig.Config.dbPath` except @@ -55,7 +66,7 @@ module Backup = /// Snapshot the live store into `target`, creating it. let toFile (target : string) : Result = - copy connString (connStringFor target) + copy connString (fileConnStringFor target) /// Replace the live store's contents with `source`'s. /// @@ -67,7 +78,7 @@ module Backup = if not (System.IO.File.Exists source) then Error $"no file at {source}" else - copy (connStringFor source) connString + copy (fileConnStringFor source) connString module Sql = diff --git a/backend/src/LibDB/StoreUpgrade.fs b/backend/src/LibDB/StoreUpgrade.fs new file mode 100644 index 0000000000..f0f8adad10 --- /dev/null +++ b/backend/src/LibDB/StoreUpgrade.fs @@ -0,0 +1,227 @@ +/// Moving a STORE from one op-log format to the next. +/// +/// After the flip there is no `packages/` to rebuild a store from, so a format change has to carry +/// the store forward in place. That divides into two cases with very different costs, and telling +/// them apart is the first thing this module does: +/// +/// FORMAT-ONLY the BINARY layout changed (`LibSerialization/Binary/*`). Op ids are derived from +/// the DECODED op by `Hashing.computeOpRowId`, not from its bytes, so nothing's +/// identity moves. The migration is a blob rewrite: decode with the old reader, +/// re-encode with the new writer, leave every id alone, re-fold the projections. +/// That is what this module does. +/// +/// IDENTITY the HASHING changed (`LibSerialization/Hashing/*`). Every op id, item hash and +/// commit hash moves, every reference inside an op has to be remapped, and the +/// commit chain has to be rebuilt parent-first. Not built; see the note at the +/// bottom of this file for what it needs. +/// +/// The distinction is not a judgement call, and this refuses rather than guessing: it recomputes +/// each op's id from the decoded op and compares it with the id the store has. If they disagree, +/// the store's ids were minted by a different hashing and the blob rewrite would be a lie. +module LibDB.StoreUpgrade + +open System.Threading.Tasks +open FSharp.Control.Tasks + +open Prelude + +open Fumble +open LibDB.Sqlite + +module PT = LibExecution.ProgramTypes +module BS = LibSerialization.Binary.Serialization +module BaseFormat = LibSerialization.Binary.BaseFormat +module Hashing = LibSerialization.Hashing.Hashing + + +type Report = + { + from : uint32 + to_ : uint32 + /// Ops decoded and written back out. + rewritten : int + /// Ops this build cannot read at all: a peer's newer format, stored inert. Left EXACTLY as + /// they are. Re-encoding is impossible and dropping them would lose work a later build can + /// still apply, which is the promise the log makes. + unreadable : int + /// Where the pre-migration store was copied to. + backup : string + } + + +let private storedFormat () : uint32 = + Releases.storedFormat () |> Option.defaultValue 1u + + +/// Where the pre-migration copy goes: beside the store, named for the version being moved TO. +/// +/// Named for the target and not the source so it reads as "the store from before v2", which is +/// what someone rolling back is looking for. +let backupPathFor (target : uint32) : string = + $"{Sqlite.currentDbPath}.pre-v{target}" + + +/// Move this store to the format this build writes. +/// +/// The order is the whole safety argument. The backup lands FIRST, through SQLite's own backup API, +/// so the rollback target exists before anything is touched. The rewrite is then one transaction: +/// a store half-converted is a store where some ops are v1 and some v2 with nothing recording +/// which, and there is no reader that can sort that out afterwards. +/// +/// Projections are dropped rather than converted. They are a cache over the log, and re-folding is +/// cheap next to getting it wrong. +let upgrade () : Task> = + task { + let from = storedFormat () + let target = BaseFormat.currentVersion + + if from = target then + return Error $"this store is already format {target}; nothing to do" + elif from > target then + // `Releases.runPending` refuses this at open, so reaching here means someone called directly. + return + Error + $"this store is format {from} and this build writes {target}. A store from a NEWER \ + format cannot be read by trying harder; upgrade the binary instead." + else + + let backup = backupPathFor target + + match Sqlite.Backup.toFile backup with + | Error e -> return Error $"could not back the store up to {backup}: {e}" + | Ok() -> + + // Read the whole log first, outside the write transaction: the decode is the part that can + // throw, and it must not do so with the rewrite half-applied. + let! rows = + Sql.query "SELECT id, op_blob FROM package_ops ORDER BY rowid" + |> Sql.executeAsync (fun read -> (read.uuid "id", read.bytes "op_blob")) + + let rewrites = ResizeArray() + let mutable unreadable = 0 + let mutable identityMoved : Option = None + + for (id, blob) in rows do + match BS.PT.PackageOp.tryDeserialize id blob with + | None -> unreadable <- unreadable + 1 + | Some op -> + // The line between the two cases, checked per op rather than assumed. An id derived from + // the decoded op must still be the id the store filed it under; if it is not, the hashing + // moved and this is not the migration that store needs. + if Hashing.computeOpRowId op <> id then + if identityMoved = None then identityMoved <- Some id + else + rewrites.Add(id, BS.PT.PackageOp.serialize id op) + + match identityMoved with + | Some id -> + return + Error + $"op {id} re-derives a different id than the store filed it under, so this store's ids \ + were minted by a different hashing than this build uses. That is an identity-changing \ + migration, which re-mints the whole log; this only rewrites blobs. The store is \ + untouched and a copy is at {backup}." + | None -> + + // `executeTransactionSync`, not hand-written BEGIN/COMMIT around separate calls: connections + // are POOLED, so a `BEGIN` and the statements after it are not guaranteed to be on the + // same one, and the transaction would silently cover nothing. + // + // The stamp goes in the SAME transaction as the bytes it describes. Outside it, a crash + // between the two leaves a store whose blobs and whose claim about them disagree, which + // is worse than either failure alone. + let statements = + [ ("CREATE TABLE IF NOT EXISTS store_meta \ + (key TEXT PRIMARY KEY, value TEXT NOT NULL)", + [ [] ]) + ("INSERT OR REPLACE INTO store_meta (key, value) VALUES ('format', @v)", + [ [ "v", Sql.string (string target) ] ]) ] + // Only when there is something to rewrite. An empty log is a real state (a store whose + // ops have all been cut away) and a statement with no parameter sets is not worth + // asking the driver to reason about. + @ (if rewrites.Count = 0 then + [] + else + [ ("UPDATE package_ops SET op_blob = @blob WHERE id = @id", + rewrites + |> Seq.map (fun (id, blob) -> + [ "blob", Sql.bytes blob; "id", Sql.uuid id ]) + |> List.ofSeq) ]) + + let mutable failure : Option = None + + try + statements |> Sql.executeTransactionSync |> ignore> + with e -> + failure <- Some e.Message + + match failure with + | Some why -> + return Error $"the rewrite failed and nothing was written: {why}" + | None -> + + // Outside the transaction: re-folding runs the playback path, which opens its own connections. + let! _ = Seed.rebuildProjections () + + return + Ok + { from = from + to_ = target + rewritten = rewrites.Count + unreadable = unreadable + backup = backup } + } + + +/// Put back the copy `upgrade` made on its way to . +/// +/// Contents, not the file, through the same backup API: connections already open keep working and +/// see the restored data. Anything already read into memory is still the NEW store, so the caller +/// has to say to restart -- the same caveat `LocalStore.restoreFrom` carries. +let rollback (target : uint32) : Task> = + task { + let backup = backupPathFor target + + if not (System.IO.File.Exists backup) then + return + Error + $"no pre-v{target} copy at {backup}. A rollback can only undo an upgrade this store \ + actually ran." + else + match Sqlite.Backup.fromFile backup with + | Error e -> return Error $"could not restore {backup}: {e}" + | Ok() -> return Ok backup + } + + +// --------------------- +// The identity-changing case, and what it needs +// --------------------- +// +// NOT BUILT. Written down because the shape is settled and the cost is not, and because the next +// person to need it should not have to rediscover why it is bigger than it looks. +// +// It is reached by a change under `LibSerialization/Hashing/`, which moves every derived id at +// once: op ids (`Hashing.computeOpRowId`), item hashes (the content address an `AddFn` carries), +// and commit hashes (derived over message, author, stamp, PARENT and the sorted ids of the ops the +// commit names). So: +// +// 1. walk the log in order, decoding each op with the old reader +// 2. rewrite every hash REFERENCE inside it through the remap built so far -- a `SetName` points +// at an item hash, a `Decision` at the versions it pins +// 3. re-encode, re-derive the id, record `old -> new` +// 4. rebuild `commits` PARENT FIRST, since a commit's id depends on its parent's; every commit +// downstream of the first change moves even if its own contents did not +// 5. rewrite every table that stores an id or a hash as a foreign key: `op_owners`, `op_branches`, +// `sync_pushed`, `seed_ops`, `commits.parent`, `package_ops.commit_hash` +// 6. drop the projections and re-fold +// +// Two things make it worth the care rather than the speed: +// +// - it is DETERMINISTIC by construction, and that is the property that matters. Ids are content +// hashes, so two machines re-minting the same log independently arrive at the same ids, and a +// push after the migration dedups to nothing. Test it by re-minting two copies and diffing every +// id, which is what 8.F.f asks for. +// - it fails QUIETLY if any of step 5 is missed. A dangling `op_branches` row does not error; a +// branch just silently loses part of its own frontier. `SCM.StoreHealth` already reports exactly +// that class, so it is the check to run after, not a new one to write. diff --git a/backend/src/LibExecution/PackageRefs.fs b/backend/src/LibExecution/PackageRefs.fs index c7d980fee3..d6972e0c80 100644 --- a/backend/src/LibExecution/PackageRefs.fs +++ b/backend/src/LibExecution/PackageRefs.fs @@ -137,6 +137,106 @@ let setHashes (hashes : Map) : unit = hashGeneration <- hashGeneration + 1 +/// Say something once, however many refs trip over it. Public because the resolver that needs it +/// lives in `LibDB`, on the other side of the dependency. +let sayOnce (msg : string) : unit = warn msg + +/// The hash this BUILD pins for a fn ref, which is what "the signature this build expects" means: +/// the pinned version is in the store too, because content is never deleted, so a candidate can be +/// compared against it without pinning anything further. +let pinnedFnHash (modules : string list) (name : string) : string option = + let fqn = $"""fn/{String.concat "." modules}.{name}""" + getHashes () |> Map.tryFind fqn + +/// The hash this BUILD pins for a TYPE ref. Same role as `pinnedFnHash`: the declaration this +/// build was compiled against, to compare a candidate rebinding with. +let pinnedTypeHash (modules : string list) (name : string) : string option = + let fqn = $"""type/{String.concat "." modules}.{name}""" + getHashes () |> Map.tryFind fqn + +/// How a ref resolves against the live store, when it does. +/// +/// Installed by whoever owns the store, because `LibDB` depends on `LibExecution` and not the +/// other way round: `PackageRefs` cannot read `locations` itself. `None` until installed, and +/// `None` for a name the store does not bind. +/// +/// Both kinds go through the store now, and the net is the same shape for each: compare the +/// candidate against the PINNED version -- signatures for a fn, the declaration for a type -- and +/// fall back to the pin, loudly, on a mismatch. +/// +/// Types took longer to get here because of a claim that turned out to be half right. A `DRecord` +/// the kernel builds does carry its type's hash, so a store whose version of that type has a +/// different SHAPE would hand Dark a value it cannot typecheck. That is what the declaration +/// check is for. What the claim missed is which direction the coupling runs: F# never reads the +/// type name it receives -- all 44 `fromDT` conversions wildcard it -- so the hash is write-only, +/// used to tag values rather than to recognise them. Which makes it a lookup by name that happens +/// to be cached in a file, not a contract that has to be. +/// +/// What this buys, and it is the point: a type a branch has just authored has NO pin to compare +/// against, so it resolves from the store directly. That is what lets F# on a git branch reference +/// package code authored on a dark branch. +let mutable resolveFnByName : (string list -> string -> string option) = + fun _ _ -> None + +/// See `resolveFnByName`. Separate hook because the check differs: a type is compared by its +/// DECLARATION, not by a signature. +let mutable resolveTypeByName : (string list -> string -> string option) = + fun _ _ -> None + +/// Bumped whenever the store could have rebound a name, by whoever owns the store. +/// +/// The ref closures below cache the store's answer against this, which turns a resolved ref into +/// one int compare and no allocation. That is not a micro-optimisation: these sit under Option and +/// Result construction, and asking the store per call measured at 46% more allocation on the +/// reference workload. Caching inside the closure rather than inside the store lookup is what +/// removes the last of it -- a cache one layer down still pays a key allocation per call. +let mutable private storeGeneration = 0 + +/// Drop every ref's memo of what the store said. `LibDB.Caching` calls this on every fold, which +/// is the only moment a name's binding can move. +let invalidateStoreResolution () : unit = storeGeneration <- storeGeneration + 1 + +/// ON by default: the store is the source, and these seventeen are the places that matters most. +/// Edit the pretty-printer and the binary you already have starts using it. +/// +/// Safe to default because the resolver checks the candidate's SIGNATURE against the pinned +/// version and refuses a mismatch, loudly, falling back to the pin -- so a wrong edit degrades +/// rather than bricks the CLI. `DARK_REFS_BY_NAME=0` turns it off anyway, which is the escape +/// hatch for a store broken in some way the signature check does not catch. +let private byNameEnabled : Lazy = + lazy (System.Environment.GetEnvironmentVariable "DARK_REFS_BY_NAME" <> "0") + +let private currentStoreGeneration () : int = storeGeneration + +/// Can this ref be resolved right now, from the store or the pin, WITHOUT raising? +/// +/// The ref closures are lazy, so an unresolvable ref is found whenever some code path happens to +/// reach it -- which can be much later than the build, in a command unrelated to whatever made it +/// unresolvable. This is the same lookup with the raise removed, so a caller can ask about every +/// ref at once and answer the real question: does this kernel agree with this package set? +let tryResolve + (kind : string) + (modules : string list) + (name : string) + : Option = + let fromStore = + if byNameEnabled.Force() then + match kind with + | "fn" -> resolveFnByName modules name + | "type" -> resolveTypeByName modules name + | _ -> None + else + None + + match fromStore with + | Some hash -> Some hash + | None -> + let fqn = $"""{kind}/{String.concat "." modules}.{name}""" + match getHashes () |> Map.tryFind fqn with + | Some hash when hash <> "" -> Some hash + | _ -> None + + /// Shared body of `Type.p` and `Fn.p`: a closure resolving `/.` /// against the hash file. Resolution is cached once per hash generation: the answer /// cannot change while the generation is stable, and resolving per call costs an @@ -152,29 +252,79 @@ let private makeRef : unit -> string = let mutable cachedGen = -1 let mutable cached = "" + // The store's answer, against `storeGeneration` rather than the hash file's: a rebinding moves + // the store without touching the file. `ValueNone` means "asked, and the store had nothing", + // which is worth remembering too -- most refs on a store that does not bind them would otherwise + // pay a lookup per call. + let mutable storeGen = -1 + let mutable storeCached = ValueNone fun () -> - let gen = currentGeneration () - if gen = cachedGen then - cached - else - let fqn = $"""{kind}/{String.concat "." modules}.{name}""" - let h = getHashes () - match Map.tryFind fqn h with - | Some hash -> - record hash - cachedGen <- gen - cached <- hash - hash - | None -> - if Map.isEmpty h then - "" // Hash file not yet populated (CI before reload-packages) + // `record` is deliberately NOT on the hit path. It is a `Map.add` into the calling module's + // `_lookup`, so calling it per resolution allocates a map node per Option construction: 0.8 MB + // on the reference workload, measured, which is all of what this path costs. The generator + // only needs to have seen each hash once. + let fromStore = + if byNameEnabled.Force() then + let gen = currentStoreGeneration () + if gen = storeGen then + storeCached else - // A non-empty file missing this ref is stale: a ref was added, or an older - // binary regenerated it in place (`growIfNeeded` rewrites it). - Exception.raiseInternal - $"PackageRefs: {kind} hash not found. The hash file is stale; regenerate it with `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages`" - [ "fqn", fqn ] + let answer = + match kind with + | "fn" -> resolveFnByName modules name + | "type" -> resolveTypeByName modules name + | _ -> None + |> ValueOption.ofOption + storeGen <- gen + storeCached <- answer + match answer with + | ValueSome hash -> record hash + | ValueNone -> () + answer + else + ValueNone + + match fromStore with + | ValueSome hash -> hash + | ValueNone -> + + let gen = currentGeneration () + if gen = cachedGen then + cached + else + let fqn = $"""{kind}/{String.concat "." modules}.{name}""" + let h = getHashes () + match Map.tryFind fqn h with + | Some hash -> + record hash + cachedGen <- gen + cached <- hash + hash + | None -> + if Map.isEmpty h then + "" // Hash file not yet populated (CI before reload-packages) + else + // Neither the store nor the pin has it, and for a type there is nothing to degrade + // to -- a value has to be tagged with something. So this raises, and the message has + // to carry the whole situation, because the person reading it is usually mid-way + // through exactly the workflow this arc exists to support: F# that names package code + // which has not arrived yet. + let dotted = $"""Darklang.{String.concat "." modules}.{name}""" + + let message = + $"This build needs the {kind} `{dotted}`, and neither the package store nor " + + "`package-ref-hashes.txt` has it.\n" + + " Authoring it yourself: author it on your branch, then " + + "`scripts/run-local-exec refs generate`.\n" + + " Somebody else's, on a branch: `dark branch import ` then " + + "`dark switch `.\n" + + " Should be on main: your store is behind the kernel -- `dark pull`, or " + + "re-fetch the pinned package set." + + Exception.raiseInternal + message + [ "fqn", fqn; "kind", kind; "name", dotted ] module Type = @@ -604,3 +754,11 @@ let kernelHash () : string = |> sha.ComputeHash |> System.Convert.ToHexString |> fun s -> s.ToLowerInvariant().Substring(0, 16) + +/// Every ref the kernel declares, as (kind, modules, name). Populated at module init, so touching +/// this forces the whole table into existence. +let allRefs () : List = + let types = + Type._lookup |> Map.toList |> List.map (fun ((m, n), _) -> ("type", m, n)) + let fns = Fn._lookup |> Map.toList |> List.map (fun ((m, n), _) -> ("fn", m, n)) + types @ fns diff --git a/backend/src/LibExecution/package-ref-hashes.txt b/backend/src/LibExecution/package-ref-hashes.txt new file mode 100644 index 0000000000..0e7838bd0e --- /dev/null +++ b/backend/src/LibExecution/package-ref-hashes.txt @@ -0,0 +1,206 @@ +fn/Cli.Terminal.renderValue|20a0dff54d9a7e4526e17aa4efe30b98c22df9a897858ddc6e5507bfcfef0724 +fn/Cli.executeCliCommand|a4b840437b7c9b9b426441a3c6efd4a1468af9d822b278ef57befc165f9279a9 +fn/Internal.Test.parseSingleTestFromFile|408997505554a133d6f611ba95fcf82fe9f92c6c1b2164ddf1211c6a10caca75 +fn/LanguageTools.Parser.CliScript.parseForCli|941945f41a5765708ca908fb58e6129ab5843fbf3159e3b10cce2f703bcdf7a0 +fn/LanguageTools.Parser.Parse.parsePTExpr|70bea7c1a45c75f6185e1d3db5677e18b66adf91641cb442ea484e01d3b50ee4 +fn/LanguageTools.Parser.Parse.parsePTExprInContext|5c46e13fb98ec3d1d4b9956611de2b2884e97a9a964dce451c52f11f8e1cfa0c +fn/LanguageTools.Parser.Parse.parsePTSourceFileWithOps|578620c01e3e5f277fb36bf66219acba1b2e05ac3e31afc03a565e814c36641f +fn/PrettyPrinter.ProgramTypes.sourceFile|007d064ece21498a303caab46c3c40752dd30b4d11b76958b27ad5380579bfdc +fn/PrettyPrinter.ProgramTypes.sourceFileAtWidth|daadd234444f41787473b3edd814048e6648cbf307828213cbece6c7e89dd72d +fn/PrettyPrinter.RuntimeTypes.Dval.valueTypeName|d239f580e296aee0a1e2c34a6790386734168f298d0d8092d032a6baccb1e7e1 +fn/PrettyPrinter.RuntimeTypes.RuntimeError.toErrorMessage|7fd47bcec6e8f46519bd2794157aabceea5512da9fce19297b6e82d142d477fa +fn/PrettyPrinter.RuntimeTypes.RuntimeError.toString|cdc00e456e722be5dc507e18733ab004593af7ab7bdd5e8f943b47924824cb2c +fn/PrettyPrinter.RuntimeTypes.dval|887ac1c5b2c2bd2c0c33fe31cfd01bf8ba2084d39e2836bd2998dd0eb4a7ae0c +fn/PrettyPrinter.RuntimeTypes.fnName|21abe108476196d96790b3d40ec1d3889a2e9dc4fdce43241e5543230579627d +fn/PrettyPrinter.RuntimeTypes.typeReference|f734f6d47fca616c1926b13d09a23efb5d620b5a214f096f1c3a36e51ba8f2c8 +fn/Stdlib.HttpClient.request|dc2c6f30499e790d516015edc4d3664ae806960bae0d11148e3bb2a3bfcd9e60 +fn/Stdlib.HttpClient.stream|d6ecec7169b4515ffc41f4e26e6164b362466fb34b2976c6cff2bcb4072c736b +type/Cli.ExecutionError.ExecutionError|e9efb8b1850160b0eaaaa51fa5a0658cc8090c52bef576fe5cf822f863b023ec +type/Cli.ExecutionError.PermissionDenied|1eb2bddcaab09f01c5b050cd44c12eecc3f11764f301ce472efacec55579c1f2 +type/Cli.ExecutionError.Unhandled|aefa4db626bbf85788e5896f4759f5582746137940d41282aabddf8f004403b5 +type/Cli.Scripts.Script|dae5fc6163a0ebe22c6ef1f55ec7b77d455041a914e6364191eec0f214644c88 +type/DarkPackages.Stats|9d63bb09dcae3852c23cda15e84803477a5002d6be54a6d106aec9bf9d5ec6c7 +type/LanguageTools.AtRestTypeChecker.AmbiguousSubject|557a52c2e6050ce51e81e7300b252708c09907e99afed237ffa7eac4c1275f83 +type/LanguageTools.AtRestTypeChecker.Context|17e12adf761418d93bd727e9cafe37787e42897a920a5acaf1f2d9a175da9137 +type/LanguageTools.AtRestTypeChecker.DuplicateSite|964cbef482ab0ba87a061926f3a7c1aa33b16275624c30040094ca62be41eabe +type/LanguageTools.AtRestTypeChecker.Issue|f349e4beb6f2f61882b89ceaa86617adf6cd6f1f52e41308497f86061df40e06 +type/LanguageTools.AtRestTypeChecker.IssueCode|72b4b5fdffaa9c8c82cd273e2483f8fd68d47e765f881b8c20bd7df08bf98acd +type/LanguageTools.AtRestTypeChecker.ItemReport|1ab924b41a1c80b6a72e7d7c4f8feaa13467d5cb0c1c5c8d121597705e1f6208 +type/LanguageTools.AtRestTypeChecker.NameRef|024f7978c58b0080920438aae4c893b9647f1f01b64079d72a2ea6dba44f4542 +type/LanguageTools.AtRestTypeChecker.Report|e52f47898b9038e701d031e122c7ac731158aa2d1656c253f493ada50ffdd9b3 +type/LanguageTools.AtRestTypeChecker.Site|8ee5e516873622675089d638224913ff15b9e46adf70b284c0996b2b43ab70bf +type/LanguageTools.AtRestTypeChecker.StaticType|553b72aa15f3bbd26eb0d7b28281053b95d47cf74a3cb8d05f62a40214d4239b +type/LanguageTools.AtRestTypeChecker.UntrustedBuiltin|aefe439cf486f046d864c03e5b34442bd01fd85b3f77d8045abeaea099757bd7 +type/LanguageTools.AtRestTypeChecker.Verdict|c17cea9c7be860439544eb6d3eb33d378e825030a50ba40407f1f56a5b1b3c30 +type/LanguageTools.BuiltinFunction|f85a8fe29057d6cece428f186740cc5bacdb8d4856c70d09bf07c1078fbb75b9 +type/LanguageTools.BuiltinFunctionParameter|41fe90a5ee71d99032303646efc46d68bbfb7d81a894f0f888b5d252abe7fc4a +type/LanguageTools.BuiltinFunctionPurity|871e4962a7f55a23f35c21043f6759a09aa382972f3b5ca48d279d764695920e +type/LanguageTools.BuiltinValue|a4d157300a079b280f5eeb324b1192550d0eb68b2cf9a92f6b5fa0b46acc0550 +type/LanguageTools.Parser.CliScript.PTCliScriptModule|6081179541a51989bd857570ea0e3621db192bcf57526bee27c86c9a10f33910 +type/LanguageTools.Parser.CliScript.ParseError|6808e3ffbad45a9484f7d362d5b3f55ed7996f741470253a7178b1e108ce9b0f +type/LanguageTools.Parser.Point|267284360b92e3b7af64acd4bead9f7b11699b5df9b51c29e8c62ea702fbdcd0 +type/LanguageTools.Parser.Range|a6ba33909336f9302fcd6d9df47c9be9dc4f85ffd2cb08509d9c991287aa2d97 +type/LanguageTools.Permissions.ApprovalFailure|703e5f0bd3e7527b0295e1b5f797b5413a0d4b2e5a467d5e3e390d56701cff1d +type/LanguageTools.Permissions.Effect|7afb64a44090ba40d3c03f7a75175507007e6c0c4305b24d35badbea8b2e9444 +type/LanguageTools.Permissions.HostRule|cccfab0b6458e3c8e70ffc9849288ca2ea99b63133f87c0293b2b0d46c73af6d +type/LanguageTools.Permissions.HttpRule|d970fa7107d29a7b7edee09daeef0a0a2dfa1bee01e5b81ea69e0fb9d4e22fcc +type/LanguageTools.Permissions.Policy|a44fd0606df64bf8eb9caefddac00dd2a99e7793494b02fa34389bcc79c59514 +type/LanguageTools.Permissions.ProcessRule|17339a7f671e8ba31f33c33fdfb8e6e61a24357d984b8d7af90a1e239d1a808d +type/LanguageTools.Permissions.Rule|9e050e7ba9b5ca5f8080aef78893b89cb8ced303b9a73a63f66e045162e8ea4c +type/LanguageTools.Permissions.Scope|c070e01ea190d8013836dae8b361eaec47f0faff1f0086a9b30ad2c52f62ba1a +type/LanguageTools.ProgramTypes.BinaryOperation|94a439540f7cd066316843e3124648fec75d8e650b7256bb08d428ec0f9d1b5e +type/LanguageTools.ProgramTypes.BranchEventKind|40eb9edeb4d3cd4943f77a9a5491459571949aed52eaa8f9200cffdcd1686290 +type/LanguageTools.ProgramTypes.DB|f86fa6609526793025bff7d6028d83db5126e5c5b32bd9a1cde2526465751c65 +type/LanguageTools.ProgramTypes.DecisionKind|2251b3ac77f0db7a0a7bc6d3b2b85a8ea988b9d8033f11c8444a3d69f2b5790a +type/LanguageTools.ProgramTypes.Deprecation|47b77911f56bc81a6a82dfe81c07d11c88d132efdbef9c4dc8e8807f23de59b2 +type/LanguageTools.ProgramTypes.DeprecationKind|3513452dffa50fed6f0b9273aaf44dbda3e3aa166d9be38f84fba9a93d449ac2 +type/LanguageTools.ProgramTypes.DocPart|edaec509f09809494c45a1122074996158f00ad03c13865eb9cc18cc8647e7b2 +type/LanguageTools.ProgramTypes.Expr|51d1c6d465118e886efbf45f4a6abaa1282b52fc159df96cea2976cec73676ea +type/LanguageTools.ProgramTypes.FQFnName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.ProgramTypes.FQFnName.FQFnName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.ProgramTypes.FQTypeName.FQTypeName|7369292d3e4858b74948d8fab5915d9c3ae77deea0b6fd6b5d7463331142d0b9 +type/LanguageTools.ProgramTypes.FQTypeName.Package|e5d0c6269fdf0c50210733af27f0fed1b5ce593ac2830f0f4b62b723d2a1d2c6 +type/LanguageTools.ProgramTypes.FQValueName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.ProgramTypes.FQValueName.FQValueName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.ProgramTypes.Hash|596e678425be94c61f2bab9d1dd380691cd415167b4b9ed766af4df8ab0ccc2f +type/LanguageTools.ProgramTypes.Infix|8e630370f404b2345b6bb76e6163a1fe5c3ef7c379319d3e38ed5b80dc1e42df +type/LanguageTools.ProgramTypes.InfixFnName|85cdca8d06c978f066f0aaa59b920f53d2f16e2359d1021aebdd44d3b8a08523 +type/LanguageTools.ProgramTypes.ItemKind|642ec69245ff338e71dae980d8c7b1cac623b6b49fdd92375df3f90d06cc862f +type/LanguageTools.ProgramTypes.LetPattern|f957b698ed40cc0c0f4c38159ee7732206d87c84440008c91a5b90b318ccf5a3 +type/LanguageTools.ProgramTypes.LocatedItem|2921046a667c6002f86a7f94bd710bf693f9273e0e8daad8580925af13cab173 +type/LanguageTools.ProgramTypes.MatchCase|7949491033140cd482b2b2aa87c7c8428d6597a02c2dde9abe9152579651c9bc +type/LanguageTools.ProgramTypes.MatchPattern|7b53dddbddbf377ed473e774b41653e1bdca1fd12e667374c37e01bbed8acde6 +type/LanguageTools.ProgramTypes.NameResolution|f0455b86c93c628fad3c05a12096b0d0bdf952dced2cf66716f390dc2673c95a +type/LanguageTools.ProgramTypes.NameResolutionError|f4d41a91096f025fe7d0db4123ef4d31bf47a0602c55da47036f0d5640434f53 +type/LanguageTools.ProgramTypes.PackageFn.PackageFn|0cc514f99b65b8c6ca917880ae86966ad4fcfd256d59940fa9eb606317a3357a +type/LanguageTools.ProgramTypes.PackageFn.Parameter|179565d8649ec92edf9652d1acbc5640890871813c9b29e4298b58e3522ade75 +type/LanguageTools.ProgramTypes.PackageLocation|952f921bae69c1df53ffde67755738bd3071fa97165be6d1e30f1d8c3ee9c706 +type/LanguageTools.ProgramTypes.PackageOp|c5d23318efdf091d000c85b6fec0825d7c5166a39ec156877bb8f3cafe86bce0 +type/LanguageTools.ProgramTypes.PackageType.PackageType|deef75c738b32ba37d01397948c3413cc176f029c7db71323100c352c6699690 +type/LanguageTools.ProgramTypes.PackageValue.PackageValue|ba279c227b51e5fee8ba5c11be7861827b8c6a4ae8f8430407fb743d2e57ffe7 +type/LanguageTools.ProgramTypes.PipeExpr|087d322b1a7e063842a2d0722f1006284a8ff29fbeb90bc0017ed4a21f1b1488 +type/LanguageTools.ProgramTypes.PropagateRepoint|1f8f3258922f4e484c83bd72b003193ae5e43b7123472d3d1d2eb1e441087b8a +type/LanguageTools.ProgramTypes.PropagationPolicy|851ca4c25f20f9c92552ae5e997b9a6c95c0df415ea886bac754e9c820814cf9 +type/LanguageTools.ProgramTypes.Reference|c08a67fb0309f39d09ffce5ec91c05c2c4e0ad99318be2bb9d4bf3e070dd2289 +type/LanguageTools.ProgramTypes.ResolvedName|0951d47e4d55de37859945115cb9bc9be5be53a4bf7516f09c7bb2b6dd6cafaf +type/LanguageTools.ProgramTypes.Search.EntityType|7824e28c37f8b4f994c3f3752e682fb58229e66743c83ae7a6e4f29515cb4f6a +type/LanguageTools.ProgramTypes.Search.SearchDepth|364316241bbd776a630ac9c3f92927d3a634ef4446b9d45258a8436a254781ec +type/LanguageTools.ProgramTypes.Search.SearchQuery|8f81372f03d7148e91328db2cef2d0d85100ddf8ed4042dfc5e3f598c85b5e46 +type/LanguageTools.ProgramTypes.Search.SearchResults|751f8f7eebb10ce34c92278e99a9686bc43a476a448f467eeed11d0042d1eb44 +type/LanguageTools.ProgramTypes.StringSegment|58e0c47bb114d85fad882c54475952c9f4b355279b455e8043f2a2fc73b44218 +type/LanguageTools.ProgramTypes.TypeDeclaration.Definition|003a2de1f7448255e6fd849823be6e80ddb4942199e6b18c72af541bf4dd823c +type/LanguageTools.ProgramTypes.TypeDeclaration.EnumCase|3fe93214c933cdb751e62fd53c1dbeac17c8ffca4d09112babcadbf93c7590a1 +type/LanguageTools.ProgramTypes.TypeDeclaration.EnumField|62c877e2401adbd4328c46f113b6d00e7d17461e281f6f43869212de243831bc +type/LanguageTools.ProgramTypes.TypeDeclaration.RecordField|179565d8649ec92edf9652d1acbc5640890871813c9b29e4298b58e3522ade75 +type/LanguageTools.ProgramTypes.TypeDeclaration.TypeDeclaration|068e22041cea13c341085e83de7d4d94211b77cee6ff034a9df4f9c11d47a0d8 +type/LanguageTools.ProgramTypes.TypeReference|d3c3e692deee3fd94d133ae4a92e5754c8adf0bcd563142a10999d39b4b36962 +type/LanguageTools.RuntimeTypes.Applicable|6cca0bb6ce4f7fed1fe32e5ad57b1ee738706d644114cb4bbe5e11a4feae9b38 +type/LanguageTools.RuntimeTypes.ApplicableLambda|b95064f7830ecd768363dd3d2664244081d12fe421a694e2c90be9eb40e3683d +type/LanguageTools.RuntimeTypes.ApplicableNamedFn|06b91d3d07a600b573f8bd28fbbcdc4e662b520e5077080efe3dcaa7bdf63f58 +type/LanguageTools.RuntimeTypes.Dval|f3cf95edd1bfe13a32c69120d061d8677b22eb8897985988f26a5c8e4011aa55 +type/LanguageTools.RuntimeTypes.FQFnName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.RuntimeTypes.FQFnName.FQFnName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.RuntimeTypes.FQTypeName.FQTypeName|7369292d3e4858b74948d8fab5915d9c3ae77deea0b6fd6b5d7463331142d0b9 +type/LanguageTools.RuntimeTypes.FQTypeName.Package|e5d0c6269fdf0c50210733af27f0fed1b5ce593ac2830f0f4b62b723d2a1d2c6 +type/LanguageTools.RuntimeTypes.FQValueName.Builtin|bd575864526cda832d5e94fa3e827d98681766f1ecb8885e0b5dac1af0cde23a +type/LanguageTools.RuntimeTypes.FQValueName.FQValueName|73e8c054475b7f2bed90297e81b85e17c069b338406f8ccf9a1b70a040b9161c +type/LanguageTools.RuntimeTypes.Hash|596e678425be94c61f2bab9d1dd380691cd415167b4b9ed766af4df8ab0ccc2f +type/LanguageTools.RuntimeTypes.Instruction|2ddd25bc868e04b2f502f22edbefe4318a1e4698b187d3b1a9f7caf2dcec8fa2 +type/LanguageTools.RuntimeTypes.Instructions|cbf1ab8b258406546f4a879cdfeec802854af3cba80983d49bd856d45face991 +type/LanguageTools.RuntimeTypes.KnownType|096e7ddd17ad9900e2905265353ada42982d0f1a35dfa39da78751aa5d8bbdaa +type/LanguageTools.RuntimeTypes.LambdaImpl|b140c92e6e43ebcb68a6959347769de5b641c7149ecaa7c702ad5536ff5d2147 +type/LanguageTools.RuntimeTypes.LetPattern|f8b3270bed302c8e06dcb7b8a5e932ce26b404b2167d1d3109158066852c6702 +type/LanguageTools.RuntimeTypes.MatchPattern|c32d33ebbe9cb02c0447b2799e4acfa285325e84c024f9717d9bd439b57e4d89 +type/LanguageTools.RuntimeTypes.NameResolution|3abcaf537e21e80c102fcc20ee62566341c528426e7c7f064b6136f6c88eaba0 +type/LanguageTools.RuntimeTypes.NameResolutionError|f4d41a91096f025fe7d0db4123ef4d31bf47a0602c55da47036f0d5640434f53 +type/LanguageTools.RuntimeTypes.RuntimeError.Applications.Error|f6b584457e9e5df4af217353347c018df6c9f022b412296150ac08387ea2bd60 +type/LanguageTools.RuntimeTypes.RuntimeError.Bools.Error|ebbfa65c0688980f9ee83311f7cea51e567bd74fabb180302826a51f8c5639c2 +type/LanguageTools.RuntimeTypes.RuntimeError.CLIs.Error|da3f52e13faf9cb8c060d9bde7ef7f26d46c45af91292d4ee41ff732ee40f68f +type/LanguageTools.RuntimeTypes.RuntimeError.Dicts.Error|29c76bcdde500809134fdd9e9cc25553c2bf410b7592c29a695a08c0399c5162 +type/LanguageTools.RuntimeTypes.RuntimeError.Enums.Error|cf5599570dad0c6653cb791ec3c3c8489cdc394667f3c7ca6a8fc9493966191e +type/LanguageTools.RuntimeTypes.RuntimeError.Error|872d08a657bc317393453c9bf5ad4f6ad5763741076137054f3d91fc0aa9b825 +type/LanguageTools.RuntimeTypes.RuntimeError.Ints.Error|a3c5852ebe977e62763b25487608bce2ed1903a832e035993c99fb45b5a56fdf +type/LanguageTools.RuntimeTypes.RuntimeError.Jsons.Error|d43d739fab1d85693d4dced787167c1c05ffae26873d6a683c3b96a2fb19726b +type/LanguageTools.RuntimeTypes.RuntimeError.Lets.Error|2befad20e7f6306338a5c730921d9d031d6b4b0b1785fa6443e18ca10c74f067 +type/LanguageTools.RuntimeTypes.RuntimeError.Lists.Error|a698587423995026c5e61cdbaeb2894a94e81a0a88bebe3da1a87a84274f151c +type/LanguageTools.RuntimeTypes.RuntimeError.Matches.Error|4b1e5ad3647a3dc2aa2f7d493b7637335604f5d9566fb3af4f4cca5bfca46d18 +type/LanguageTools.RuntimeTypes.RuntimeError.Records.Error|1a0f2e292b56b519e4d969f413144c16f3ea7cbe73b1056cab8d37c48b26275e +type/LanguageTools.RuntimeTypes.RuntimeError.Statements.Error|4240e113859d32841d783d78e3169f5a8ae8b87c3fe6b169d9c52ec3d2da9650 +type/LanguageTools.RuntimeTypes.RuntimeError.Strings.Error|2182dbf5bf844104dc70780c1fef6866618e0f96a6a0c764716112fb1d4fe07f +type/LanguageTools.RuntimeTypes.RuntimeError.Unwraps.Error|32ef16d276d78a349fabbbe2f7872b9aa85ba6742f25dbbdb5b2f0477e55cfaa +type/LanguageTools.RuntimeTypes.StringSegment|9b4999992350c36a25dcd53a3fb0c92dc6449c534713b392c74bf2c3cab989e2 +type/LanguageTools.RuntimeTypes.TypeReference|611535d15626223e1245fa4023f81691430a58963c5e0410bf52659614ec8496 +type/LanguageTools.RuntimeTypes.ValueType|98dd5d3be603f70536815aca3ae3db82a5788336f3254237e9d5e48bdd2695f2 +type/LanguageTools.Sign|c8276aff8c52a06bbfacac24d654f3bc3d4a2a37bb0b9ba0ef98d2b99ca56dca +type/LanguageTools.WrittenTypes.BinaryOperation|94a439540f7cd066316843e3124648fec75d8e650b7256bb08d428ec0f9d1b5e +type/LanguageTools.WrittenTypes.Expr|72db9e5ad59e416e141cfee8075f6b27cd44a2b11e315a508951411a667339f4 +type/LanguageTools.WrittenTypes.FnDeclaration.FnDeclaration|3cf6af515e205f67127fd5725bca0dd4f0b1e07ce234acd3d490c826d613cf8e +type/LanguageTools.WrittenTypes.FnDeclaration.NormalParameter|2dcf9acdca3ca0abeeddb4d7cfe65195dbaa17cfb3cde417f4fd8ddb673bc8fa +type/LanguageTools.WrittenTypes.FnDeclaration.Parameter|bf0f75061470b8faa1494e67e3c4f0a25ae3d2f3b91542265f938a5db7cd64ea +type/LanguageTools.WrittenTypes.FnDeclaration.UnitParameter|7b1f2b8d5d41696eb8a33a3b77fcad6287a4af6ef4d3e9da98a9aa4512ba5e91 +type/LanguageTools.WrittenTypes.FnIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.Infix|8e630370f404b2345b6bb76e6163a1fe5c3ef7c379319d3e38ed5b80dc1e42df +type/LanguageTools.WrittenTypes.InfixFnName|85cdca8d06c978f066f0aaa59b920f53d2f16e2359d1021aebdd44d3b8a08523 +type/LanguageTools.WrittenTypes.LetPattern|a01a8aee1ae846263a89c26bd4bedb013403069beaf1bc6b4bb7731e7c2375c1 +type/LanguageTools.WrittenTypes.MatchCase|f05617d9c57f11337cafaeb401d4da7504870bc0a634b24776de9e530b5144fd +type/LanguageTools.WrittenTypes.MatchPattern|528c1badff3131b6910b03a3091b0ebe465874dbd9dac928b63f25ec5f1f8b02 +type/LanguageTools.WrittenTypes.ModuleDeclaration.Declaration|c89618597f1c0dad4b0cfa53ca652505826a409931728eb89db3f39b5c38ac22 +type/LanguageTools.WrittenTypes.ModuleDeclaration.ModuleDeclaration|1a418dd008179924b912513c94be30868d2eb4919a039c64a6a364a68c554a61 +type/LanguageTools.WrittenTypes.ModuleIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.ParsedFile|07382e181d7d13690ef13b4f95f2ba8b07e7140332ce2af981d7a7296ba64e99 +type/LanguageTools.WrittenTypes.PipeExpr|c885f6f5219f56d346406b636d0d7948517f8f2cdb0f80135707b23d41eeac70 +type/LanguageTools.WrittenTypes.QualifiedFnIdentifier|0f68e49a14653866fe015d68714149c50cd572ab8aaf9f5c0e66cece4e4f3856 +type/LanguageTools.WrittenTypes.QualifiedTypeIdentifier|e813835428f56f4a33a22ca382385c82509d255644639bae0daec48f94c26398 +type/LanguageTools.WrittenTypes.SourceFile.SourceFile|630e8cc6e7bed1a47eac5add8a5cdc1c2f3e5cf41ed220a258710f29669f4554 +type/LanguageTools.WrittenTypes.SourceFile.SourceFileDeclaration|53a59a7e8a6a4e670834cb26ca70aa8475fc401aa447e2e85f32d434fd98f5f9 +type/LanguageTools.WrittenTypes.StringSegment|3b069928919b9a8ea9df0efc533b908daded5e5f4650c5886aa5fd319a2849eb +type/LanguageTools.WrittenTypes.TypeDeclaration.Definition|240935955526e9e0c62f6901bbd25bb1a4887596cd0c51d5382f6fc42ba4d732 +type/LanguageTools.WrittenTypes.TypeDeclaration.EnumCase|89ece486356b4be56caabfabbddcf7cc4eea6403a3510c2f16855a6708d094fa +type/LanguageTools.WrittenTypes.TypeDeclaration.EnumField|9b73a8dfadc90d77e9b5707318ab34d659c8cd9a66e6cabe96a8ed425bb86a3a +type/LanguageTools.WrittenTypes.TypeDeclaration.RecordField|e57cc695dd2e30a35ec8351b28b14f8e7b71ec9be5466e9af3096ce0f9ea49af +type/LanguageTools.WrittenTypes.TypeDeclaration.TypeDeclaration|ec050095398e9b4d8e564c269a9855981401110218de1c340bfdb6f6c8647d05 +type/LanguageTools.WrittenTypes.TypeIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.TypeReference.Builtin|e1a6e53933d0ba4d11f08c592bc7b5c997c791c614e9451ddb9278257bc529f7 +type/LanguageTools.WrittenTypes.TypeReference.TypeReference|df116cfc0b6f7d492401ec365cfec5aa580a31a3486731f4d99bf86114b2af6a +type/LanguageTools.WrittenTypes.ValueDeclaration.ValueDeclaration|74051a4d9394cf0bed20c5fa0eeee6603a1aa0d55eba59ca57c6a83ef0cdfbf9 +type/LanguageTools.WrittenTypes.ValueIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/LanguageTools.WrittenTypes.VariableIdentifier|a308efb9d71cb57411fb0cb0f5d162eb80173a814235cc6913833cb129932dec +type/PrettyPrinter.RuntimeTypes.RuntimeError.ErrorMessage|2314408c9017bab823414db19433005893a4d09e6235a41011e3ab1f5ae79818 +type/Stdlib.AltJson.Json|a3b6038ebf54eb71804f0ad7b96b883e442ccf4d01af8ac220c6981fa3530305 +type/Stdlib.AltJson.ParseError.ParseError|8dc77b20b13969bee07766b01d9afa75c78fbfe780443c621742fa2021831abe +type/Stdlib.Cli.ExecutionOutcome|1a4a0d68fa0df09a91283de7e194709cdd210793d8e517b44e26090caeaf031a +type/Stdlib.Cli.FileSystem.FileError|76b5a147c2369e90d08ae51df41920e46e38c0d5b5828539542049cbc63bd188 +type/Stdlib.Cli.OS.OS|b7c0e15d61d6652c5be3cdc9360637524332b72aec228c3a771e2f3f88a8a1fe +type/Stdlib.Cli.Posix.Error|a55af955adffd4fd5aa31b9c0f097a496a161b9514a18922a54943b95aaec4b0 +type/Stdlib.Cli.Stdin.Key.Key|788e35c44bf7b0546cfce602a673dcc38ee2a2e84680e6e8df39f5c281325085 +type/Stdlib.Cli.Stdin.KeyRead.KeyRead|12da87b8fc16efa28f98572b629ca38be3729f94d928e2c60e17242d03f11f9d +type/Stdlib.Cli.Stdin.Modifiers.Modifiers|5473100a8e7309f3438b5ef92a99fba6518d937b3de96d2c3e4169a21e097100 +type/Stdlib.Float.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Stdlib.Http.Request|e8c423f68d1d0743b84b7ac20b131372c646a5ce534c8438c8801d18e6274362 +type/Stdlib.Http.Response|4d686467d89ac2a7af562f68fc4d9048d12774ffd4827a6dde5efd69cc127a74 +type/Stdlib.HttpClient.BadHeader|dbfe4487bd474fdf0ffccaf4d8f546785e1037becc1f5527696e053e5199f1b8 +type/Stdlib.HttpClient.BadUrlDetails|1dbb355daa5a3c3531e71a0682e0aa20ccf6751d136abcfdcca5def779c21eb5 +type/Stdlib.HttpClient.RequestError|4e899f22b33061bc75ced7d558833eb454908a5ea4f414980c31c778289dcfcf +type/Stdlib.HttpClient.Response|4d686467d89ac2a7af562f68fc4d9048d12774ffd4827a6dde5efd69cc127a74 +type/Stdlib.HttpClient.StreamResponse|9a509af8b0b16226d61a9f4bea0ffa533c36a889606c161799398b92c22dab78 +type/Stdlib.Int.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Stdlib.Int128.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int16.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int32.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int64.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Int8.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Json.ParseError.JsonPath.Part.Part|a19fcef1b58b4010b122abe9bef2401d38085229b44a085eb27e35f60652f3dc +type/Stdlib.Json.ParseError.ParseError|7e14f06f970c01da29e51d928202774b414e098b0376f49bc2234a8b857fae5f +type/Stdlib.Option.Option|c58cf283d0e5d6634a76fa30ea5b0d29e615e1c4fe1cc2bfca27077cd72bc072 +type/Stdlib.Result.Result|03549ecc7eb39c974d3f7181c87a3ee9337fc91307ecb2c24351b0df4809ce15 +type/Stdlib.Sqlite.Value|41c352765ba7273fab25c56327f91a554550dc3db8e1417a6fcf2e4da5e586e7 +type/Stdlib.UInt128.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt16.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt32.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt64.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.UInt8.ParseError|ab0a98d41aabb3799b3209709a093f3e0ccc6ec79853bb333bfe7cafc8813333 +type/Stdlib.Uuid.ParseError|e342cc59c22ca41ab61c5bb375c04c6dbb31592d5a79491593a3b2bc522fb86b +type/Tracing.FnCall|a117302204173443b563b8ae07857467463bd14c77bec3cf12653bab5a4b04ce +type/Tracing.InputVar|e75e7d0c6e9202b9ec72823bdb3dbc668a22d61d4b3468dd3f1e729b814d461d +type/Tracing.TraceData|542e314e375133a017fb9f27733a0ede6a167af4d70edf1c64aaa9aa0145c76e +type/Tracing.TraceSummary|d7623097e2da2eb7216d8b77fa6877d586ed369da09b5bea50eca6c5576b5a6f diff --git a/backend/src/LibSerialization/Binary/BaseFormat.fs b/backend/src/LibSerialization/Binary/BaseFormat.fs index 38cef2b554..e8207c922f 100644 --- a/backend/src/LibSerialization/Binary/BaseFormat.fs +++ b/backend/src/LibSerialization/Binary/BaseFormat.fs @@ -3,19 +3,34 @@ module LibSerialization.Binary.BaseFormat open System -/// v1 is the format the op-log substrate ships with; nothing older ever existed in -/// the wild (pre-v1 stores were rebuilt from `.dark` source each build). Bump on every -/// wire-layout change, keeping a readV1 beside the new writer: from here, stores -/// cannot be rebuilt from text. -[] -let CurrentVersion = 1u +/// The format this build WRITES. v1 is the format the op-log substrate ships with; nothing older +/// ever existed in the wild (pre-v1 stores were rebuilt from `.dark` source each build). Bump on +/// every wire-layout change, keeping a readV1 beside the new writer: from here, stores cannot be +/// rebuilt from text. +/// +/// `DARK_FORMAT_VERSION` overrides it, and exists for one reason: there is only one format so far, +/// so the store migrator has nothing to migrate between and no way to be exercised. With it set, +/// this build writes the version named and reads every version up to it -- a SYNTHETIC bump whose +/// layout happens to be identical, which makes a migration across it a pure blob rewrite. That is +/// the format-only case, and having it mechanised and tested before the first real bump is the +/// point. Never set in production; `LibDB.StoreUpgrade` is what uses it. +/// +/// A plain `let` rather than a `[]` for that reason. Nothing pattern-matches on it. +let currentVersion : uint32 = + match System.Environment.GetEnvironmentVariable "DARK_FORMAT_VERSION" with + | null + | "" -> 1u + | s -> + match System.UInt32.TryParse s with + | true, n when n >= 1u -> n + | _ -> 1u /// Binary file header structure (8 bytes) type BinaryHeader = { // The blob's format version. Passed to version-dispatched readers (makeDeserializerV) so a new // binary can decode an OLD layout by branching on it: the keystone of any future format - // migration. Bump `CurrentVersion` on the next wire-layout change and add the matching readVN. + // migration. Bump `currentVersion` on the next wire-layout change and add the matching readVN. Version : uint32 // 4 bytes - format version DataLength : uint32 } // 4 bytes - payload size @@ -43,8 +58,17 @@ module Varint = module Validation = let validateVersion (version : uint32) = - // Reject formats from other versions rather than guessing how to parse them. - if version <> CurrentVersion then + // OLDER is fine, NEWER is not, and the asymmetry is the whole point of a versioned header. + // + // A blob from an older format can be read: every historical reader stays in the binary, and + // `makeDeserializerV` hands the version to the reader so it can branch. That is how a store + // moves forward without being rebuilt from text, which after the flip is the only way it can + // move at all. + // + // A blob from a NEWER format cannot be read by trying harder -- the layout is one this binary + // has never seen -- so it is refused rather than guessed at. `LibDB.Releases` refuses the whole + // STORE for the same reason, before anything gets as far as a blob. + if version = 0u || version > currentVersion then raise (BinaryFormatException(UnsupportedVersion version)) let validateDataLength (expected : uint32) (actual : uint32) = diff --git a/backend/src/LibSerialization/Binary/Serialization.fs b/backend/src/LibSerialization/Binary/Serialization.fs index b582bf64a9..861da616b8 100644 --- a/backend/src/LibSerialization/Binary/Serialization.fs +++ b/backend/src/LibSerialization/Binary/Serialization.fs @@ -58,7 +58,7 @@ let makeSerializer<'T, 'ID> use finalWriter = new BinaryWriter(finalStream) let header = - { Version = CurrentVersion; DataLength = uint32 payloadBytes.Length } + { Version = currentVersion; DataLength = uint32 payloadBytes.Length } Header.write finalWriter header finalWriter.Write(payloadBytes) @@ -72,7 +72,7 @@ let makeSerializer<'T, 'ID> /// every historical readVN alongside one current writer; that is what lets a new binary decode an OLD /// blob. /// -/// No reader dispatches on the version yet, because `CurrentVersion` is still 1: v1 is the first +/// No reader dispatches on the version yet, because `currentVersion` is still 1: v1 is the first /// format whose blobs outlive the binary that wrote them, since before it every store was rebuilt /// from `.dark` on each build. This exists so the first layout change has somewhere to go. let makeDeserializerV<'T, 'ID> diff --git a/backend/src/LocalExec/LocalExec.fs b/backend/src/LocalExec/LocalExec.fs index 7fafa055a2..6c1d52a49a 100644 --- a/backend/src/LocalExec/LocalExec.fs +++ b/backend/src/LocalExec/LocalExec.fs @@ -58,9 +58,11 @@ module HandleCommand = // work. let! _ = LibDB.Inserts.commitAllAsBaseline "package reload (baseline)" - // Generate hash file BEFORE evaluating values, so that PackageRefs - // lookups resolve correctly during value evaluation. - do! LibDB.PackageRefsGenerator.generate () + // Hashes in MEMORY before evaluating values, so that PackageRefs lookups resolve during + // it. Not written to disk: the file moves when the PIN moves, not on every reload, which + // is what stops two package-touching branches conflicting in it. `scripts/packages/pin` + // and `refs generate` write it. + do! LibDB.PackageRefsGenerator.refreshInMemory () LibExecution.PackageRefs.reloadHashes () // Evaluate all values now that all definitions are in the DB @@ -109,10 +111,13 @@ module HandleCommand = return Error $"Migration failed: {ex.Message}" } - let exportSeed (outputPath : string) : Ply> = + let exportSeed + (outputPath : string) + (upToCommit : string option) + : Ply> = uply { try - do! LibDB.Seed.export outputPath + do! LibDB.Seed.exportAt outputPath upToCommit let size = System.IO.FileInfo(outputPath).Length / 1024L / 1024L print $"Seed exported to {outputPath} ({size} MB)" return Ok() @@ -120,6 +125,206 @@ module HandleCommand = return Error $"Export failed: {ex.Message}" } + /// Stand on the branch this rundir is on, the way the CLI does before it runs anything. + /// + /// Not global to LocalExec: the fill path deliberately refills MAIN from disk, and doing that + /// while standing on a branch would be wrong. Only the commands that ask a question ABOUT the + /// current branch select it. + let selectStoredBranch () : Ply = + uply { + match! LibDB.BranchSelection.select None None with + | Ok selection -> + LibDB.PackageManager.selectBranch ( + selection.branchId + |> Option.defaultValue LibExecution.ProgramTypes.BranchId.Main + ) + | Error _ -> () + } + + /// The git branch this tree is checked out on, if it is a git tree at all. + /// + /// Read out of `.git/HEAD` rather than by shelling out: this runs inside the build, and a + /// process spawn for one line of text is not worth it. A detached HEAD answers `None`, which is + /// right -- there is no branch NAME to line up with. + let private gitBranchName () : Option = + try + // Walk UP looking for `.git`, rather than assuming the rundir sits directly inside the + // repo. It does for the dev rundir and does not for a test's, and the difference is silent: + // you get no branch name and no error. + let rec findGitHead (dir : System.IO.DirectoryInfo) : string option = + if isNull (box dir) then + None + else + let candidate = System.IO.Path.Combine(dir.FullName, ".git", "HEAD") + if System.IO.File.Exists candidate then + Some candidate + else + findGitHead dir.Parent + + let head = + match findGitHead (System.IO.DirectoryInfo LibConfig.Config.runDir) with + | Some h -> h + | None -> "" + + if head <> "" && System.IO.File.Exists head then + let text = (System.IO.File.ReadAllText head).Trim() + let prefix = "ref: refs/heads/" + + if text.StartsWith prefix then Some(text.Substring prefix.Length) else None + else + None + with _ -> + None + + + /// Does this kernel agree with the package set in front of it? + /// + /// Direction one of the two-way interface: every name the kernel references has to resolve, in + /// the store as seen from the current branch or in the pin. Direction two -- every builtin the + /// package set calls existing in this kernel -- needs the store to record builtin edges, which + /// it does not yet. + /// + /// Asked all at once, and at BUILD time, because the ref closures are lazy: an unresolvable ref + /// is otherwise found whenever some code path happens to reach it, which can be a different day + /// and an unrelated command. The case this exists for is checking out somebody's git branch + /// without their package work: the F# in your tree names things your store has never heard of, + /// and you should be told that then, in one list, rather than one at a time by whatever runs + /// first. + let checkRefs () : Ply> = + uply { + do! selectStoredBranch () + + // Every hash this store actually HOLDS content for. A ref resolving to a hash is not the + // same as that hash naming anything: a pin can name content the store no longer has, and + // that is precisely the failure this check exists to catch -- it does not error at runtime, + // it renders blank. Asked once as a set rather than per ref. + let! knownTypes = + Sql.query "SELECT hash FROM package_types" + |> Sql.executeAsync (fun read -> read.string "hash") + + let! knownFns = + Sql.query "SELECT hash FROM package_functions" + |> Sql.executeAsync (fun read -> read.string "hash") + + let known = Set.union (Set.ofList knownTypes) (Set.ofList knownFns) + + let unresolved = + LibExecution.PackageRefs.allRefs () + |> List.filter (fun (kind, modules, name) -> + match LibExecution.PackageRefs.tryResolve kind modules name with + | None -> true + | Some hash -> not (Set.contains hash known)) + + // Direction two: every builtin the package set calls has to exist in THIS kernel. Recorded + // by the fold in `package_builtin_deps`, which is what makes a store able to say which + // kernel it needs -- the check that used to answer this grepped `.dark` text off disk and + // stops being possible the day packages come from a seed. + let kernelBuiltins = + let b = Builtins.all () + Set.union + (b.fns.Values + |> Seq.map (fun f -> (f.name.name, f.name.version)) + |> Set.ofSeq) + (b.values.Values |> Seq.map (fun v -> (v.name.name, 0)) |> Set.ofSeq) + + let! calledBuiltins = + Sql.query + "SELECT DISTINCT builtin_name, builtin_version FROM package_builtin_deps" + |> Sql.executeAsync (fun read -> + (read.string "builtin_name", read.int "builtin_version")) + + let missingBuiltins = + calledBuiltins |> List.filter (fun b -> not (Set.contains b kernelBuiltins)) + + // An EMPTY table is not a pass. `package_builtin_deps` is a projection, so a store that got + // the table from a release step without re-folding has no rows, and the builtin half of the + // check would report success having asked nothing. Saying so is the difference between this + // check and one that quietly stops covering what it was written for. + if List.isEmpty calledBuiltins then + return + Error( + "this store records no builtin calls at all, so the builtin half of this check asked " + + "nothing. `package_builtin_deps` is a projection: re-fold the log to fill it " + + "(`scripts/build/reload-packages`, or any migration that drops projections)." + ) + elif List.isEmpty unresolved && List.isEmpty missingBuiltins then + let n = List.length (LibExecution.PackageRefs.allRefs ()) + print ( + $"All {n} kernel refs resolve, and all {List.length calledBuiltins} builtins this " + + "package set calls exist in this kernel." + ) + return Ok() + elif List.isEmpty unresolved then + let lines = + missingBuiltins + |> List.sort + |> List.map (fun (n, v) -> $" Builtin.{n} (v{v})") + |> String.concat "\n" + + return + Error( + $"this package set calls {List.length missingBuiltins} builtin(s) this kernel does " + + $"not have:\n{lines}\n\nA builtin was removed or renamed out from under package " + + "code that calls it. Land the package change that stops calling it, move the pin " + + "forward, and only then remove the builtin." + ) + else + let lines = + unresolved + |> List.sortBy (fun (kind, m, n) -> (kind, m, n)) + |> List.map (fun (kind, modules, name) -> + $""" {kind} Darklang.{String.concat "." modules}.{name}""") + |> String.concat + " +" + + let! hint = + uply { + // If git is on a branch and a dark branch of the same name exists, that is almost + // certainly where the missing items are -- so say the command rather than the + // category. The coupling made visible at the one moment it matters, instead of a + // rule somebody has to have read. + match gitBranchName () with + | None -> return "" + | Some git -> + let! darkBranch = LibDB.Branches.liveIdForName git + + match darkBranch with + | Some _ when LibDB.PackageManager.currentBranchId () = BranchId.Main -> + return + $"\n\ngit is on `{git}` and there is a dark branch called `{git}`, " + + $"but you are on dark main. Try `dark switch {git}`." + | _ -> return "" + } + + return + Error( + $"{List.length unresolved} kernel ref(s) do not resolve against this package set:\n" + + lines + + $"\n\nThis kernel and this package set do not agree. Usually that means the " + + "F# in your tree names package code your store does not have: import the branch " + + "bundle that goes with it, or move to the branch that has it." + + hint + ) + } + + /// Write `package-ref-hashes.txt` from whatever store this rundir has. + /// + /// The kernel's entry points are pinned BY HASH, so a binary needs that file before it can resolve + /// anything. The fill path writes it as a side effect of reloading `packages/`; this is the same + /// step on its own, for a store that arrived as a SEED and has no `packages/` to reload. That is + /// the only thing standing between a fetch-at-pin build and a working binary. + let generateRefs () : Ply> = + uply { + try + do! selectStoredBranch () + do! LibDB.PackageRefsGenerator.generate () + LibExecution.PackageRefs.reloadHashes () + return Ok() + with ex -> + return Error $"Generating package refs failed: {ex.Message}" + } + let listMigrations () : Ply> = uply { try @@ -183,7 +388,26 @@ let main (args : string[]) : int = | [ "export-seed"; outputPath ] -> handleCommand $"Exporting seed to {outputPath}" - (HandleCommand.exportSeed outputPath) + (HandleCommand.exportSeed outputPath None) + + // Cut at a commit, so what a pin fetches is fixed by the commit rather than by when it asked: + // the same commit yields the same OPS however far the store has moved since, and ids are derived + // from op content, so two stores built from it agree. Not byte-identical -- the stamp records + // which build cut it and when -- and nothing needs it to be. + | [ "export-seed"; outputPath; commit ] -> + handleCommand + $"Exporting seed at {commit} to {outputPath}" + (HandleCommand.exportSeed outputPath (Some commit)) + + | [ "refs"; "check" ] -> + handleCommand + "checking the kernel's refs against this package set" + (HandleCommand.checkRefs ()) + + | [ "refs"; "generate" ] -> + handleCommand + "writing package-ref-hashes.txt from this store" + (HandleCommand.generateRefs ()) | [ "pm-sweep-blobs" ] -> handleCommand @@ -206,7 +430,9 @@ let main (args : string[]) : int = print " reload-packages" print " migrations run" print " migrations list" - print " export-seed " + print " export-seed [commit]" + print " refs generate" + print " refs check" print " pm-sweep-blobs" print " bench" print " bench-render" diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin b/backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin new file mode 100644 index 0000000000..a89b241afe Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/packageLocation.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin b/backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin new file mode 100644 index 0000000000..3cf2323cc5 Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/packageOp.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin new file mode 100644 index 0000000000..041cb1aeeb Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageFn.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin new file mode 100644 index 0000000000..c8857a8cd4 Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageType.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin new file mode 100644 index 0000000000..60e6e38de9 Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/ptPackageValue.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin new file mode 100644 index 0000000000..326d8c4b0d Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/rtDval.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin new file mode 100644 index 0000000000..8b02fda7f5 Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/rtInstructions.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageFn.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageFn.bin new file mode 100644 index 0000000000..455765d65f Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageFn.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageValue.bin b/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageValue.bin new file mode 100644 index 0000000000..ae3e5f97ae Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/rtPackageValue.bin differ diff --git a/backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin b/backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin new file mode 100644 index 0000000000..3b971412c9 Binary files /dev/null and b/backend/testfiles/serialization-artifacts/corpus/v1/toplevel.bin differ diff --git a/backend/tests/Tests/CliPackages.Tests.fs b/backend/tests/Tests/CliPackages.Tests.fs index c771333df4..037077c8f4 100644 --- a/backend/tests/Tests/CliPackages.Tests.fs +++ b/backend/tests/Tests/CliPackages.Tests.fs @@ -815,6 +815,222 @@ let aVersionMovedAndMovedBackKeepsTheLastNaming = }) +/// `remove` is the only thing that writes an `Unbind`, and the claim it prints is the +/// interesting one: the name ends, the content does not, so callers go on working. That is only +/// true because a reference points at content rather than at a name, which is the property worth +/// a test rather than a comment. +/// `grep` searches BODIES, which is the half `search` does not do. Scoped to one module on +/// purpose: unscoped it renders every live item, and the point of the scope argument is that a +/// test, like a person, usually knows roughly where to look. +/// A revert is a rebinding, not a recovery: every version is still in the store, so pointing the +/// name back at an old hash is the whole operation. Which is why it is symmetric. +let revertPutsANameBackAndIsSymmetric = + instanceTest + "revert restores the version a commit held, and reverting twice is a no-op" + (fun state -> + task { + do! start state + do! fn state "Tests.Rv.f" "() : Int64 = 111L" + do! commit state "rv one" + + let! log = runCliPlain state [ "log" ] + // The newest commit is first, and that is the one holding 111. + let first = + log.Split('\n') + |> Array.tryPick (fun line -> + System.Text.RegularExpressions.Regex.Match(line, "[0-9a-f]{8}") + |> fun m -> if m.Success then Some m.Value else None) + + let commitOfOne = + match first with + | Some c -> c + | None -> Tests.failtestf "no commit hash in `dark log`:\n%s" log + + do! fn state "Tests.Rv.f" "() : Int64 = 222L" + do! commit state "rv two" + do! evals state "Tests.Rv.f ()" "222" "the newer version is live" + + do! run state [ "revert"; "Tests.Rv.f"; commitOfOne ] + do! evals state "Tests.Rv.f ()" "111" "and the revert put the old one back" + + do! + shows + state + [ "revert"; "Tests.Rv.f"; commitOfOne ] + "already holds" + "reverting to where it already is says so rather than authoring a no-op" + do! discardAll state + }) + +/// The three ways to get it wrong, which is where a two-argument command earns its errors. +let revertRefusesWhatItCannotFind = + instanceTest + "revert names what it could not find, for the name and for the commit" + (fun state -> + task { + do! start state + do! + shows + state + [ "revert"; "Tests.Rv.nothingHere"; "abc12345" ] + "nothing here is named" + "an unknown name" + do! fn state "Tests.Rv.known" "() : Int64 = 1L" + do! commit state "rv known" + do! + shows + state + [ "revert"; "Tests.Rv.known"; "ffffffffff" ] + "no single commit here starts with" + "an unknown commit" + do! shows state [ "revert" ] "usage: dark revert" "bare prints usage" + do! discardAll state + }) + +let grepFindsSourceAndNotJustNames = + instanceTest + "grep matches a function body, and reports name and line" + (fun state -> + task { + do! start state + // A string LITERAL, not a comment: grep reads rendered source, and the renderer prints + // the AST, which keeps `///` docs and drops `//` asides. + do! fn state "Tests.Grep.needle" "() : String = \"findmeplease\"" + do! fn state "Tests.Grep.other" "() : String = \"something else\"" + do! commit state "grep fixture" + + do! + shows + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "Tests.Grep.needle:" + "the hit names the item" + do! + shows + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "findmeplease" + "and shows the matching line" + do! + lacks + state + [ "grep"; "findmeplease"; "Tests.Grep" ] + "Tests.Grep.other" + "an item whose body does not match is not reported" + do! discardAll state + }) + +/// The cache is the feature, so the second run has to agree with the first. Keyed by content +/// hash, which is why it never needs invalidating. +let grepAgreesWithItselfOnceCached = + instanceTest "a second grep, reading the cache, finds the same thing" (fun state -> + task { + do! start state + do! fn state "Tests.GrepTwice.f" "() : String = \"cachedtoken\"" + do! commit state "grep cache fixture" + + do! + shows + state + [ "grep"; "cachedtoken"; "Tests.GrepTwice" ] + "Tests.GrepTwice.f:" + "cold" + do! + shows + state + [ "grep"; "cachedtoken"; "Tests.GrepTwice" ] + "Tests.GrepTwice.f:" + "warm" + do! discardAll state + }) + +/// A search tool you cannot trust a negative answer from is worse than none, so "no hits" and +/// "could not read it" must not look alike. +/// The bug the SCM silo's guard exists to stop, in grep's shape: `locations` is main's +/// projection, so an enumeration that read it directly would search MAIN from a branch and report +/// nothing wrong. This is the test that the overlay is actually consulted. +let grepSeesTheBranchYouAreStandingOn = + instanceTest "grep finds a branch's own work, and main does not" (fun state -> + task { + do! start state + do! switch state "grep-branch" + do! fn state "Tests.GrepBranch.only" "() : String = \"branchonlytoken\"" + + do! + shows + state + [ "grep"; "branchonlytoken"; "Tests.GrepBranch" ] + "Tests.GrepBranch.only:" + "the branch sees its own work" + + do! onMain state + do! + shows + state + [ "grep"; "branchonlytoken"; "Tests.GrepBranch" ] + "no live item's source contains" + "main does not" + + do! archiveBranches state [ "grep-branch" ] + }) + +let grepSaysWhenItFindsNothing = + instanceTest "grep says so when nothing matches" (fun state -> + task { + do! start state + do! + shows + state + [ "grep"; "zzz-not-in-any-source-zzz"; "Darklang.Stdlib.Option" ] + "no live item's source contains" + "an honest empty answer" + do! shows state [ "grep" ] "usage: dark grep" "bare prints usage" + }) + +let removeEndsTheNameAndLeavesTheCallersAlone = + instanceTest "remove ends a name, and what called it still runs" (fun state -> + task { + do! start state + do! fn state "Tests.Rm.leaf" "() : Int64 = 4242L" + do! fn state "Tests.Rm.caller" "() : Int64 = Tests.Rm.leaf ()" + do! commit state "add leaf and caller" + + do! evals state "Tests.Rm.caller ()" "4242" "the caller works to begin with" + + do! run state [ "remove"; "Tests.Rm.leaf"; "-y" ] + + do! + shows + state + [ "view"; "Tests.Rm.leaf" ] + "Not found" + "the name holds nothing now" + do! + evals + state + "Tests.Rm.caller ()" + "4242" + "but the caller still runs, because it references content and not a name" + do! discardAll state + }) + +/// Bare and wrong-argument shapes, which is where this kind of command goes wrong: a confirming +/// verb that cannot find its target must refuse rather than ask about nothing. +let removeRefusesWhatIsNotThere = + instanceTest + "remove refuses a name that holds nothing, and refuses to run bare" + (fun state -> + task { + do! start state + do! + shows + state + [ "remove"; "Tests.Rm.nothingHere"; "-y" ] + "nothing here is named" + "an unknown name is named back" + do! shows state [ "remove" ] "usage: dark remove" "bare prints usage" + }) + let renameIsVisibleToEverythingThatReads = instanceTest "a renamed item is readable at its new name, by every reader" @@ -843,8 +1059,85 @@ let renameIsVisibleToEverythingThatReads = }) +/// `dark edit ` is the replacement for opening a `.dark` file, so the properties that +/// matter are the ones a file gave you for free: everything applies together, and nothing you did +/// not ask to remove goes away. +/// +/// Driven through the FILE form, which needs no terminal. The editor form is the same code past +/// the point where it has a file. +let editModuleAppliesTogetherAndRemovesNothing = + instanceTest + "editing a module applies every declaration at once, and omitting one leaves it alone" + (fun state -> + task { + do! start state + + let write (contents : string) : string = + let path = System.IO.Path.GetTempFileName() + ".dark" + System.IO.File.WriteAllText(path, contents) + path + + do! + run + state + [ "module" + "/Tests.EditMod" + write "let one (): Int64 = 1L\nlet two (): Int64 = 2L\n" ] + + // One changed, one untouched, applied as a batch. + do! + run + state + [ "edit" + "Tests.EditMod" + write "let one (): Int64 = 111L\nlet two (): Int64 = 2L\n" ] + + do! + shows + state + [ "eval"; "Tests.EditMod.one ()" ] + "111" + "the edited one landed" + do! + shows + state + [ "eval"; "Tests.EditMod.two ()" ] + "2" + "its sibling is untouched" + + // Leaving a declaration OUT must not end it. Guessing the other way is where this command + // would lose work: one mis-parse that dropped an item from the render would unbind a name. + do! + shows + state + [ "edit"; "Tests.EditMod"; write "let one (): Int64 = 222L\n" ] + "left alone, not in your file" + "the omitted declaration is named back" + + do! shows state [ "eval"; "Tests.EditMod.two ()" ] "2" "and is still bound" + + // A declaration that does not parse lands NOTHING, not the half that did. + do! + run + state + [ "edit" + "Tests.EditMod" + write "let one (): Int64 = 999L\nlet bad (): Int64 = @@@\n" ] + + do! + shows + state + [ "eval"; "Tests.EditMod.one ()" ] + "222" + "a batch with one bad declaration applies none of it" + + do! discardAll state + }) + + let tests : List = - [ lsNamesWhatIsThere + [ editModuleAppliesTogetherAndRemovesNothing + lsNamesWhatIsThere treeShowsDescendants viewPrintsSource viewRefusesWhatIsNotThere @@ -862,6 +1155,14 @@ let tests : List = deleteRefusesWhatIsNotThere deprecateAndUndeprecate renameIsVisibleToEverythingThatReads + removeEndsTheNameAndLeavesTheCallersAlone + removeRefusesWhatIsNotThere + grepFindsSourceAndNotJustNames + grepAgreesWithItselfOnceCached + grepSaysWhenItFindsNothing + grepSeesTheBranchYouAreStandingOn + revertPutsANameBackAndIsSymmetric + revertRefusesWhatItCannotFind aDocOnlyEditKeepsTheVersionAndStillLands aFieldsDocEditLands anEnumCasesDocEditLands diff --git a/backend/tests/Tests/CliScm.Tests.fs b/backend/tests/Tests/CliScm.Tests.fs index 54246982f6..b6596877fb 100644 --- a/backend/tests/Tests/CliScm.Tests.fs +++ b/backend/tests/Tests/CliScm.Tests.fs @@ -1658,12 +1658,16 @@ let private editChangesAnItemWithoutRetypingIt = "usage: dark edit" "a bare `edit` says how to use it" + // A module is EDITABLE now (`Packages.Edit.editModule`), so what this asserts is the + // shape without a terminal: it points at the file form rather than spawning an editor + // into a pipe. The old expectation here was "a module is refused", which is how this + // test earned its place -- it caught the behaviour change the day it happened. do! shows state [ "edit"; "Tests.Edit" ] - "is a module" - "a module is refused, and named as the reason" + "applies a file instead" + "a module with no terminal points at the file form" do! shows diff --git a/backend/tests/Tests/MultiInstance.Tests.fs b/backend/tests/Tests/MultiInstance.Tests.fs index 14fe3fbc1a..6b2826bf95 100644 --- a/backend/tests/Tests/MultiInstance.Tests.fs +++ b/backend/tests/Tests/MultiInstance.Tests.fs @@ -691,14 +691,14 @@ let hostedOpsAreNotThisStoresDraft = [ "id", Sql.string (string hostedId) ] Expect.equal stillThere 1L "and a discard leaves it where it is" + // Folded, not inert. A server that cannot fold cannot see or serve what it hosts, so it + // folds now -- and `op_owners`, not `effective`, is what keeps a peer's push out of this + // store's draft, which is the assertion above. let! effective = Sql.query "SELECT effective AS e FROM package_ops WHERE id = @id" |> Sql.parameters [ "id", Sql.string (string hostedId) ] |> Sql.executeRowAsync (fun read -> read.int64 "e") - Expect.equal - effective - 0L - "still inert: a relay serves what it is handed, it does not run it" + Expect.equal effective 1L "and it is folded like any other arriving op" }) diff --git a/backend/tests/Tests/OpTransport.Tests.fs b/backend/tests/Tests/OpTransport.Tests.fs index 016ec8b9ab..0b6a6ef9b1 100644 --- a/backend/tests/Tests/OpTransport.Tests.fs +++ b/backend/tests/Tests/OpTransport.Tests.fs @@ -114,21 +114,28 @@ let foldQuarantinesPoison = /// /// This asserts the flag rather than the consequence because the consequence needs a second process. let relayStoreDoesNotQueueForFolding = - testTask "storeOpsWithOwner: hosted ops are never queued for this store's fold" { + testTask "storeOpsWithOwner: a hosted op is folded, but is not this store's draft" { do! cleanup () let id = "fada0000-0000-0000-0000-0000000000ef" let! n = Inserts.storeOpsWithOwner "someone" [ (id, "00", ts) ] Expect.equal n 1L "stored the op" + // It used to be stored `effective = 0` so no fold would ever touch it. A server has to be + // able to see and serve what it hosts, so it is folded now like any other arriving op. let! effective = effectiveOf id - Expect.equal - effective - 0L - "hosted ops are stored effective=0, so growIfNeeded never folds them into this store's main" + Expect.equal effective 1L "queued for the fold" let! applied = appliedOf id - Expect.equal applied 0L "and they are not pretended to have been applied" + Expect.equal applied 0L "but not pretended to have been applied already" + + // The half that must NOT change. `effective` was carrying two meanings -- "fold this" and + // "this is mine" -- and only the first was given up. An op a peer pushed here is their work, + // so it is not in this store's draft and a discard must leave it alone. + let! draft = LibDB.Queries.getDraftOps () + let inDraft = + draft |> List.exists (fun op -> string (Inserts.computeOpHash op) = id) + Expect.isFalse inDraft "a hosted op is not this store's draft" do! cleanup () } diff --git a/backend/tests/Tests/OpsProjections.Tests.fs b/backend/tests/Tests/OpsProjections.Tests.fs index 4dee582610..bdc58f376e 100644 --- a/backend/tests/Tests/OpsProjections.Tests.fs +++ b/backend/tests/Tests/OpsProjections.Tests.fs @@ -321,7 +321,7 @@ let durableReleaseCarriesForward = let registryCoversProjections = // The COUNT is in the name on purpose: adding a projection to the registry without adding it here // is exactly the drift this catches. - test "the projection registry covers exactly the 7 regenerable projections" { + test "the projection registry covers exactly the 8 regenerable projections" { Expect.equal (List.sort Seed.projectionTables) (List.sort @@ -330,6 +330,7 @@ let registryCoversProjections = "package_values" "locations" "package_dependencies" + "package_builtin_deps" "deprecations" "propagation_policy" ]) "the registry's tables are exactly Seed.export's stripped projections (incl. deprecations)" diff --git a/backend/tests/Tests/Serialization.Binary.Tests.fs b/backend/tests/Tests/Serialization.Binary.Tests.fs index 22c27ddeb2..3b195366fb 100644 --- a/backend/tests/Tests/Serialization.Binary.Tests.fs +++ b/backend/tests/Tests/Serialization.Binary.Tests.fs @@ -316,11 +316,222 @@ module ConsistentSerializationTests = }) +/// The ABI regression net: blobs written by PAST builds, kept forever, read by this one. +/// +/// Different question from the golden files above, and the difference is the whole point. A golden +/// pins "the current writer still produces these bytes", and is REGENERATED whenever a format change +/// is intended -- so the moment the format moves, the old bytes are gone and nothing checks that +/// they can still be read. This corpus is never regenerated. One file per (format version, type), +/// committed once and then only read. +/// +/// What it asserts is the migrator's core operation: take a blob a previous build wrote, decode it +/// with whatever reader claims to understand that version, write it back out with TODAY'S writer, +/// and decode THAT. The two decoded values must be equal. A reader that mis-decodes an old layout +/// fails here even when the bytes it goes on to produce are perfectly self-consistent, which is +/// exactly the failure a migration cannot afford: the store is the only copy of the ops. +module Corpus = + module BaseFormat = LibSerialization.Binary.BaseFormat + + /// One serialized type, and the two things a corpus needs of it: what this build would write, and + /// whether a stored blob survives a trip through today's writer. + /// + /// `reread` hides the type, which is why this is a record of closures rather than a generic: the + /// kinds have nothing in common at the type level and the test only ever asks these two questions. + type Kind = + { + name : string + /// What this build writes for each test value, in order. The generator's input. + blobs : unit -> List + /// Decode `blob`, re-encode it with today's writer, decode that, and compare. Raises if the + /// blob cannot be decoded at all, which is itself the failure. + reread : byte[] -> bool + } + + let private kindEq + (name : string) + (ser : 'T -> byte[]) + (deser : byte[] -> 'T) + (eq : 'T -> 'T -> bool) + (values : List<'T>) + : Kind = + { name = name + blobs = fun () -> values |> List.map ser + reread = fun blob -> let v = deser blob in eq v (deser (ser v)) } + + let private kind name ser deser values = kindEq name ser deser (=) values + + /// Everything that goes INTO a store, which is what a migration has to carry forward. The op log + /// first, because it is canonical and the rest is projection. + let kinds : List = + [ kind + "packageOp" + (BS.PT.PackageOp.serialize "corpus") + (BS.PT.PackageOp.deserialize "corpus") + Values.ProgramTypes.packageOps + kind + "ptPackageType" + (fun (t : PT.PackageType.PackageType) -> + BS.PT.PackageType.serialize t.hash t) + (BS.PT.PackageType.deserialize "corpus") + Values.ProgramTypes.packageTypes + kind + "ptPackageValue" + (fun (v : PT.PackageValue.PackageValue) -> + BS.PT.PackageValue.serialize v.hash v) + (BS.PT.PackageValue.deserialize "corpus") + Values.ProgramTypes.packageValues + kind + "ptPackageFn" + (fun (f : PT.PackageFn.PackageFn) -> BS.PT.PackageFn.serialize f.hash f) + (BS.PT.PackageFn.deserialize "corpus") + Values.ProgramTypes.packageFns + kind + "packageLocation" + (BS.PT.PackageLocation.serialize "corpus") + (BS.PT.PackageLocation.deserialize "corpus") + Values.ProgramTypes.packageLocations + kind + "toplevel" + (fun (tl : PT.DB.T) -> BS.PT.Toplevel.serialize tl.tlid tl) + (BS.PT.Toplevel.deserialize 0UL) + Values.ProgramTypes.toplevels + // The RT side is stored too: `package_values.rt_dval` and the compiled fn bodies. A migration + // that carried the op log forward and left these unreadable would look like it worked. + // + // NaN needs its own arm, as it does in the roundtrip tests above: `DFloat nan = DFloat nan` is + // false, so plain equality reports a perfectly good decode as a mis-decode. + kindEq + "rtDval" + (BS.RT.Dval.serialize "corpus") + (BS.RT.Dval.deserialize "corpus") + (fun a b -> + match a, b with + | RT.DFloat f1, RT.DFloat f2 when + System.Double.IsNaN f1 && System.Double.IsNaN f2 + -> + true + | _ -> a = b) + (Values.RuntimeTypes.dvals ()) + kind + "rtPackageValue" + (fun (v : RT.PackageValue.PackageValue) -> + BS.RT.PackageValue.serialize v.hash v) + (BS.RT.PackageValue.deserialize "corpus") + Values.RuntimeTypes.packageValues + kind + "rtPackageFn" + (fun (f : RT.PackageFn.PackageFn) -> BS.RT.PackageFn.serialize f.hash f) + (BS.RT.PackageFn.deserialize "corpus") + Values.RuntimeTypes.packageFns + kind + "rtInstructions" + (BS.RT.Instructions.serialize "corpus") + (BS.RT.Instructions.deserialize "corpus") + Values.RuntimeTypes.instructions ] + + /// One file per (version, kind), holding every blob for it: `[count][len][bytes]...`, all + /// little-endian uint32. A file per blob would be seven hundred files for the Dval set alone, + /// which makes the corpus unreadable as a diff and unpleasant to carry. + let private frame (blobs : List) : byte[] = + use stream = new System.IO.MemoryStream() + use w = new System.IO.BinaryWriter(stream) + w.Write(uint32 (List.length blobs)) + blobs + |> List.iter (fun b -> + w.Write(uint32 b.Length) + w.Write b) + w.Flush() + stream.ToArray() + + let private unframe (data : byte[]) : List = + use stream = new System.IO.MemoryStream(data) + use r = new System.IO.BinaryReader(stream) + let count = r.ReadUInt32() |> int + [ for _ in 1..count -> r.ReadBytes(r.ReadUInt32() |> int) ] + + let private fileFor (v : uint32) (k : Kind) = $"corpus/v{v}/{k.name}.bin" + + /// Write this build's blobs for the CURRENT version, for any kind not already stored. + /// + /// Never overwrites. A corpus entry is a historical fact about what some build actually wrote, and + /// regenerating one turns the net into a mirror -- which is precisely how the golden files above + /// stop covering an old format. Deliberately changing what a version's bytes are means deleting + /// the file by hand, and having to justify it. + let generate () : unit = + let v = BaseFormat.currentVersion + + System.IO.Directory.CreateDirectory(Config.serializationDir + $"corpus/v{v}") + |> ignore + + kinds + |> List.iter (fun k -> + let f = fileFor v k + if not (File.fileExists Config.Serialization f) then + File.writefileBytes Config.Serialization f (frame (k.blobs ())) + print $" corpus: wrote {f}") + + let tests = + // Every version from 1 up to this build's, since every one of them is a layout this binary + // claims to read. A version with no file fails rather than being skipped quietly. + [ 1u .. BaseFormat.currentVersion ] + |> List.map (fun v -> + testList + $"v{v}" + (kinds + |> List.map (fun k -> + test k.name { + let f = fileFor v k + + Expect.isTrue + (File.fileExists Config.Serialization f) + $"v{v} has stored {k.name} blobs ({f}). Write missing ones with \ + DARK_CONFIG_SERIALIZATION_GENERATE_TEST_DATA=y and COMMIT them: a format this \ + build claims to read with no stored bytes is a claim nothing checks." + + let stored = File.readfileBytes Config.Serialization f |> unframe + + stored + |> List.iteri (fun i blob -> + Expect.isTrue + (k.reread blob) + $"{k.name}[{i}] written at v{v} still decodes to the same value through this \ + build's writer") + + // At the CURRENT version there is a second, stronger thing to say: this build must + // still produce these exact bytes. That is what catches a layout change made without + // bumping `currentVersion`, which is the one format bug that corrupts silently -- two + // builds both calling themselves v1 and disagreeing about what a v1 blob means. + // + // A PREFIX, so a new test value can be appended without destroying the stored bytes + // for a version. Inserting one in the middle breaks this on purpose: the corpus is + // positional, and history is what it is for. + if v = BaseFormat.currentVersion then + let current = k.blobs () + + Expect.isGreaterThanOrEqual + (List.length current) + (List.length stored) + $"{k.name} still has at least the {List.length stored} test values its corpus was \ + written from. Removing one leaves stored bytes nothing can be compared against; \ + append rather than insert or delete." + + List.zip stored (List.truncate (List.length stored) current) + |> List.iteri (fun i (storedBlob, currentBlob) -> + Expect.equal + currentBlob + storedBlob + $"this build writes {k.name}[{i}] exactly as v{v} did. If the layout genuinely \ + changed, bump BaseFormat.currentVersion and keep a readVN -- do not rewrite \ + this file.") + }))) + + let generateTestFiles () = // Enabled in dev so we can see changes as git diffs // Disabled in CI so changes will fail the tests if Config.serializationGenerateTestData then ConsistentSerializationTests.generateTestFiles () + Corpus.generate () () @@ -351,4 +562,6 @@ let tests = RT.closureAccessIsStripped RT.namedFnAccessIsStripped ] - testList "consistent serialization" ConsistentSerializationTests.testTestFiles ] + testList "consistent serialization" ConsistentSerializationTests.testTestFiles + + testList "stored blobs from past formats" Corpus.tests ] diff --git a/config/dev b/config/dev index db17fd6bd0..a1f0819f15 100644 --- a/config/dev +++ b/config/dev @@ -27,8 +27,13 @@ DARK_CONFIG_TRACE_SAMPLING_RULE_DEFAULT=sample-all # Turn it on for the one command you want a trace from. DARK_CONFIG_TRACE_DETAIL=off -# Package source: "disk" (reload from .dark files) or "seed" (fetch from R2) -# Default: disk (set to "seed" to use the seed-based flow) +# Package source: "disk" (reload from .dark files) or "seed" (fetch a built store). +# +# Still "disk", deliberately, and not yet the default it is headed for. In seed mode the build +# planner routes a `packages/**.dark` edit NOWHERE, which is correct -- the store is the source -- +# but it means the only way to change package code is the authoring verbs, and `dark edit +# ` does not exist yet. Flipping this before it does would leave no way to edit packages +# at all. `scripts/build/reload-packages` still works by hand if you opt in early. DARK_CONFIG_PACKAGES_SOURCE=disk ###################################### diff --git a/docs/package-workflow.md b/docs/package-workflow.md new file mode 100644 index 0000000000..ef03dffcf6 --- /dev/null +++ b/docs/package-workflow.md @@ -0,0 +1,210 @@ +# Working on Dark when the packages live in a store + +How to add a builtin, use it from Dark, reference a new package type or function from F#, try all +of it locally, and get it to everybody else. + +The one-line version: **the store is the source of truth for Dark code, git is the source of truth +for F#, and a git branch carries the package work its F# depends on so the two merge as one thing.** + +Read `docs/dev-setup` first if the container is not up. `dark docs packages` is the short version of +this from inside the CLI. + +--- + +## Where a build's packages come from + +Two answers, and `package-set.txt` at the root says which: + + commit unset built from `packages/` on disk, by reloading it + commit fetched as a SEED from a package server, at that commit + +`scripts/build/prepare-package-set` is the one place that answers the question, and CI's +package-reloading jobs go through it. It also checks that the kernel and the package set agree +before letting the build continue. + +It ships `unset` today, which is the reloading behaviour that has always been there. Everything +below works in both modes except where it says otherwise. + +--- + +## Adding a builtin and using it from Dark + +A builtin is F#; the Dark that calls it is a package item. They are two sides of one change and +they belong in one PR. + +1. Add the fn to the `fns` list in the right `Builtins/Libs/.fs`. `AGENTS.md` says how to + pick the subproject and why you return structured Dark values rather than strings. +2. Wrap it, once, in a Dark package fn. `tests/builtin` enforces exactly one wrapper and at least + one caller: a builtin with two `Builtin.x` references fails, and so does one with none. +3. `scripts/dev/build`. The build now ends with: + + All 206 kernel refs resolve, and all 734 builtins this package set calls + exist in this kernel. + + That second half is the check that matters here. It reads `package_builtin_deps`, which the fold + fills from the real call graph -- not a grep over text, which is what it replaces and which stops + being possible once `packages/` is gone. + +**Removing or renaming a builtin is two steps, and the check enforces the order.** Land the package +change that stops calling it, move the pin forward, and only then remove the builtin. Doing it the +other way round fails the build with: + + this package set calls 1 builtin(s) this kernel does not have: + Builtin.someBuiltinWeDeleted (v0) + +Adding one is safe in a single step, because an older package set simply does not call it. + +--- + +## Referencing a new package type or function from F# + +This is the case that needs a branch, and the only one that does. + +F# names package items through `PackageRefs`. Those resolve **from the store, by name**, with the +hash in `package-ref-hashes.txt` as a fallback and a shape check in between: a candidate whose +signature (fn) or declaration (type) does not match what this build was compiled against is refused, +loudly, and the pinned version is used. A type nobody has pinned yet -- one your branch has just +authored -- has nothing to compare against, so the store's answer is taken. That is what makes this +work at all. + + git checkout -b add-foo + dark branch add-foo # same name; they travel together + + dark type /Darklang.LanguageTools.Foo '{ n: Int64 }' + # authored on the dark branch. Invisible from main, and from everyone else. + + # ...add `let foo = p [] "Foo"` in PackageRefs.fs and use it... + + scripts/dev/build # compiles; regenerates refs; runs `refs check` + scripts/packages/bundle export # ~1KB of JSON: package-branch.json + git commit -a # F# + package-ref-hashes.txt + the bundle + +The bundle is how your package work reaches anyone else. Export is explicit, like `git add`. + +**Order matters in one small way:** author the type before the F# that uses it RUNS. Adding the ref +and building is fine -- a ref is a lazy closure -- but the first code path that reaches an +unresolvable one raises. + +### What your coworker does + +Nothing special. They check out the branch and build. + + git checkout add-foo + scripts/dev/build # prepare-package-set imports the bundle automatically + +Importing is automatic because checking out the git branch IS asking for that branch's package +code, and it is idempotent -- ops are content addressed, so a second import says "up to date". + +**If they are on the wrong dark branch, the build tells them, completely, at build time:** + + 1 kernel ref(s) do not resolve against this package set: + type Darklang.LanguageTools.Foo + + git is on `add-foo` and there is a dark branch called `add-foo`, + but you are on dark main. Try `dark switch add-foo`. + +That is deliberate: the refs are lazy, so without this you would find out whenever some unrelated +command happened to reach that code path. + +### Reviewing it + + scripts/packages/bundle show --source + +Prints what the bundle changes, with the full declarations, by importing into a throwaway store. +Reviewing somebody's branch should not mean taking their ops into the store you work in. + +--- + +## Trying it locally before it goes anywhere + +Everything above is local already. Your store is yours: there are no restrictions on what you can +author or rebind on your own machine, including under `Darklang.*`. + +To try the two-machine shape without a server, a second store is enough: + + dark branch export add-foo /tmp/b.json # from one + dark branch import /tmp/b.json # into the other + +To try it against a real server, run one: + + DARK_MATTER_WRITE_SECRET= dark serve Darklang.Matter.router --port 9090 + +`scripts/testing/gates seed-serving` and `gates server-folds` do exactly this and are the worked +examples if you want to see the whole thing driven end to end. + +--- + +## Getting it to everybody else + + dark push your own namespace, straight to the server's main + dark branch push anything else, including `Darklang.*` + +**Your namespace is yours to publish to freely.** Log in, write a function, push it. No branch, no +PR, no waiting. + +**`Darklang.*` is reviewed**, and the server refuses a direct push that binds it: + + 403: this server does not accept pushes that bind Darklang.Stdlib.List.foo + into its main. That namespace is reviewed: `dark branch push` instead, and + it lands on main when the change is merged. + +A branch push is always accepted, because a branch is isolated, nobody runs it, and review is what +moves it to main. That is also why an abandoned branch costs nothing: close the PR and the bundle +goes with it. + +The server FOLDS what you push, so it shows up in `/m`, `/p` and in seeds. It never RUNS it: a +pushed `val` is folded, browsable and servable, and evaluated only on the machine that fetches it. + +--- + +## The pin, and when it moves + +`package-set.txt` names the commit a build's packages come from. It moves when somebody re-pins: + + scripts/packages/pin head --url # pin to what it has now + scripts/packages/pin --show + scripts/packages/pin --unset # back to building from packages/ + +Every pin is checked before it is written, so a commit that does not resolve is refused then rather +than at the next CI run. + +Re-pinning also regenerates `package-ref-hashes.txt`, and that is the only thing that does. +Ordinary package work leaves it alone, which is what stops two branches that both touch packages +conflicting in a 206-line generated file. The diff a pin bump produces is the point: one reviewable +list of every kernel identity that moved. + +A pinned seed is cached per machine under `~/.darklang/seeds`, so the network is needed once per +pin, not once per clone. + +--- + +## When the op-log format changes + +A store does not get rebuilt from text any more, so it has to be carried forward in place. + + dark store format, which commit it was cut at, which build cut it + dark store upgrade move the op log to this build's format + dark store rollback put back the copy `upgrade` took + +`upgrade` copies the store first and does the rewrite in one transaction. It only rewrites BLOBS: a +change that moves content hashes is a much larger migration and it refuses rather than half-doing +it. It is a no-op until the first real format bump. + +If you are running a build that reads an OLDER format than your store, it says so and names the file +to move back -- a `mv` needs no working binary, and by then you do not have one. + +--- + +## Things that will bite + +- **`dark log` on a branch shows that branch's OPS, not main's commits.** Ask `dark --branch main + log` when you want commits. +- **A bundle left behind after its branch merges** is how somebody imports work from weeks ago. + Going back to dark main removes it; `bundle export` on main will clean up a stale one. +- **Your main can drift ahead of the pin** if you `dark pull`. Then your branch may rest on commits + nobody else has, and your bundle will not be enough for them. `bundle export` warns when it sees + this. +- **Two branches adding the same name with different shapes** both merge, last-writer-wins picks + one, and the loser's F# then references a name whose declaration is not what it compiled against. + The compatibility check catches it at the second merge, so main goes red rather than silently + wrong -- but it is a merge conflict git cannot see. diff --git a/package-set.txt b/package-set.txt new file mode 100644 index 0000000000..af8a09c3bb --- /dev/null +++ b/package-set.txt @@ -0,0 +1,20 @@ +# The package set this kernel is pinned to. +# +# `scripts/packages/pin` writes this file; `scripts/fetch-seed`, CI and the dev container read it. +# Its whole job is to make "which packages" a decision recorded in git rather than whatever the +# tree happened to contain when a build ran. +# +# There is deliberately no URL here. A pin says WHICH package set, not where to get it: two +# mirrors serving the same commit are interchangeable, and a URL in the repo is a deployment +# detail that ages badly and belongs to whoever runs the server. The address comes from +# `DARK_SEED_URL` in the environment, or `--url` on the command line. +# +# commit the SCM commit on the package server that this kernel was tested against. `unset` +# means there is no pin yet, and everything that reads this file falls back to +# building the package set from `packages/` -- which is what happens today. +# format the op-blob layout the seed at that commit is written in. A fetch that gets a +# different one refuses rather than folding it, because a layout this binary has +# never seen cannot be read by trying harder. + +commit unset +format 1 diff --git a/packages/darklang/cli/docs/command.dark b/packages/darklang/cli/docs/command.dark index 2d0abcf072..9843dc0696 100644 --- a/packages/darklang/cli/docs/command.dark +++ b/packages/darklang/cli/docs/command.dark @@ -150,7 +150,7 @@ module Topics = DocTopic { name = "relay" - description = "The optional sync server: setup, the write secret, smoke tests" + description = "The optional sync server: setup, the write secret, serving a seed" aliases = [ "matter", "server", "sync-server" ] context = "cli" content = Docs.Relay.content diff --git a/packages/darklang/cli/docs/packages.dark b/packages/darklang/cli/docs/packages.dark index 8f75a93265..2eb41b5de8 100644 --- a/packages/darklang/cli/docs/packages.dark +++ b/packages/darklang/cli/docs/packages.dark @@ -34,4 +34,37 @@ or search queries. Write `Stdlib.List.map` (preferred) or the full source name ## Stdlib vs Builtin Stdlib: packages/darklang/stdlib/*.dark Builtins: F# in backend/src/Builtins/ - Use `builtins` to list all builtins""" + Use `builtins` to list all builtins + +## Where a build's package set comes from + +Two answers, and `package-set.txt` at the repo root says which: + + commit unset built from `packages/` on disk, by reloading it + commit fetched as a SEED from a package server, at that commit + +The second is the direction this is going: the server's store is the source of +truth, a seed of it is what a fresh container and CI fetch, and `packages/` goes +away. Today it ships `unset`, so nothing has changed yet. + + scripts/packages/pin head pin to what that server has now + scripts/packages/pin pin to a named commit + scripts/packages/pin --show what is pinned + scripts/packages/pin --unset back to building from `packages/` + +The server address is not in the repo: it comes from `DARK_SEED_URL` or +`--url`. A pin says WHICH package set, not where to get it. + + scripts/build/prepare-package-set get it, whichever of the two it is + scripts/fetch-seed --url just the fetch + +## What a store says about itself + + store format, which commit it was cut at, which build cut it + store upgrade move the op log to the format this build writes + store rollback put back the copy `upgrade` took on its way to format + +`upgrade` copies the store first and rewrites in one transaction. It only +rewrites blobs: a change that moves content hashes is a much larger migration, +and it refuses rather than half-doing it. It is a no-op until the first format +bump.""" diff --git a/packages/darklang/cli/docs/relay.dark b/packages/darklang/cli/docs/relay.dark index ad23f4c38a..1dce35db05 100644 --- a/packages/darklang/cli/docs/relay.dark +++ b/packages/darklang/cli/docs/relay.dark @@ -4,11 +4,22 @@ let content () : String = """# Running your own sync server (the relay) Syncing is OPTIONAL and opt-in: an instance is complete without one, and -connecting is one `dark sync setup`. The relay is the sync server: a dumb -store-and-forward for the op log that holds ops and hands them back out. It never folds or runs what it hosts, so pushing code to it -cannot change what it serves, and the worst a bad relay can do is withhold. -Losing one loses availability, not history -- every client holds the full log -it has seen, and the next push repopulates a wiped relay. +connecting is one `dark sync setup`. The server holds the op log and hands it +back out: `/sync/pull` pages it, `/seed/*.db` cuts it as a file. Losing one +loses availability, not history -- every client holds the full log it has seen, +and the next push repopulates a wiped server. + +It FOLDS what you push, into main, like any other instance. That is what makes +`/m`, `/p` and the seed routes show hosted packages rather than only what the +binary shipped with. It does not RUN them: a pushed `val` is folded, browsable +and servable in a seed, and never evaluated on the server -- whoever fetches it +evaluates it on their own machine, under their own policy. + +Folding still binds NAMES, though, last-writer-wins over the whole store, and +the server resolves its own router through those names. So anyone who can push +here can change what this server serves. Who may bind what is not answered yet; +for now the write secret is the whole of the answer, so treat push access as +trust. ## Standing one up @@ -27,11 +38,33 @@ Clients connect with: curl https:///ping answers curl https:///sync/head op count, as JSON + curl https:///seed/meta head commit, format, counts curl -X POST https:///sync/push 401 without the secret The 401 from another machine is the smoke test worth doing once: it proves the secret is required from the outside, not just missing locally. +## Serving the package set as a file + + /seed/latest.db the store as of now + /seed/.db the store as of that commit and its ancestors + /seed/meta the same four fields without the megabytes + /seed/meta?commit= ...about that commit, or 404 if it is unknown + +A seed is the op log as a file: what a fresh install folds to become this +store, and what CI fetches instead of parsing text. `latest.db` resolves to +the head and is served as the cut AT that head, so it is the same thing as +asking for that commit by name. + +Cut at a commit, always, because that is what makes a pin reproducible: the +same commit yields the same ops however far the store has moved since. Cuts +are cached beside the store and never need invalidating, commits being +immutable and ops append-only. A short commit prefix resolves like git's; an +ambiguous one is refused rather than guessed at. + +Clients fetch with `scripts/fetch-seed --url `, which takes the commit +from `package-set.txt` if that tree has a pin. See `dark docs packages`. + ## Two things not to do to a relay - Don't run client commands (`dark status`, `dark fn`, ...) against its @@ -48,7 +81,8 @@ the secret is required from the outside, not just missing locally. are idempotent, run `dark sync` a conflict after pull not data loss -- both versions are kept; `dark conflicts` - "store was written by a newer build" upgrade dark; the store is fine + "this store is format N" upgrade dark, or move the + `.pre-vN` copy back; `dark store` Reads are public by design. If that's wrong for your relay, put it behind something that terminates auth; the relay itself only gates writes and diff --git a/packages/darklang/cli/exportSeed.dark b/packages/darklang/cli/exportSeed.dark index 07258de478..fa2d61783b 100644 --- a/packages/darklang/cli/exportSeed.dark +++ b/packages/darklang/cli/exportSeed.dark @@ -8,7 +8,7 @@ let execute (state: AppState) (args: List): AppState = Stdlib.printLine (help state) state | [outputPath] -> - match Builtin.pmSeedExport outputPath with + match Stdlib.LocalStore.seedTo outputPath (Stdlib.Option.Option.None) with | Ok _ -> Stdlib.printLine $"Seed exported to {outputPath}" state diff --git a/packages/darklang/cli/packages/edit.dark b/packages/darklang/cli/packages/edit.dark index 73b24d91c3..b0d77e14a0 100644 --- a/packages/darklang/cli/packages/edit.dark +++ b/packages/darklang/cli/packages/edit.dark @@ -78,15 +78,13 @@ let execute (state: AppState) (args: List) : AppState = match itemLocation location with | None -> - Stdlib.printLine - (Stdlib.Cli.UI.Colors.error - $"{name} is a module. `dark edit` takes one fn, type or value.") - - Stdlib.printLine - (Stdlib.Cli.UI.Colors.dimText - " `dark module /Some.Module ` edits a whole module at once.") + match location with + | Module modulePath -> editModule state modulePath fileArg + | _ -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {name}.") - state + state | Some loc -> match fileArg with @@ -203,6 +201,13 @@ let help (_state: AppState) : String = [ "" " dark edit open $EDITOR on the declaration, apply on save" " dark edit apply that file instead (no editor, for scripts)" + " dark edit the whole module at once, same two shapes" + "" + " A MODULE opens every declaration directly under it, and applies them in one" + " batch: all of them are validated first, so a half-finished edit lands nothing" + " rather than half. Leaving a declaration OUT does not end it -- the ones you" + " omitted are named back to you and left alone. `dark remove ` ends a" + " name, explicitly and in one op." "" " The file holds one declaration, the same form `dark module` takes." " `dark view --raw` prints exactly that form, so reading an item," @@ -225,3 +230,201 @@ let complete (args: List) : List = Packages.View.complete state args + + +// --------------------- +// Editing a whole module +// --------------------- +// +// `dark edit ` is the replacement for opening a `.dark` file, and after the flip there is +// no file to open. It renders every declaration directly under the module, hands you $EDITOR, and +// applies the result through `dark module` -- which validates ALL of them before saving ANY, so a +// half-finished edit lands nothing rather than half. +// +// It does NOT delete by omission, and that is deliberate rather than unfinished. `dark module` +// adds and updates; leaving a declaration out of the file does not unbind it. Guessing the other +// way is where this command would lose work: one mis-parse that dropped an item from the render +// would quietly end a name. So omissions are REPORTED and left alone, and `dark remove` stays the +// way a name ends -- one op, confirmed, and visible in `dark status`. + +/// Every declaration directly under , rendered, in a stable order. +/// +/// Types first, then values, then functions, each alphabetical. Order has to be deterministic or +/// a render-edit-save round trip churns the file for nobody's benefit; kind-then-name is the order +/// the module listing already uses, so what you edit looks like what you browsed. +let moduleSource + (branchId: Uuid) + (modulePath: List) + : (String * List) = + let results = Packages.Query.allDirectDescendants branchId modulePath + + // Lambdas here take no type annotation -- Dark does not allow one -- so `wrap` carries the kind + // and the item shape is whatever `LocatedItem` gives: `.location` and `.entity`. + let render (wrap: LanguageTools.ProgramTypes.PackageLocation -> Packages.PackageLocation) (items: List<'a>) : List<(String * String)> = + items + |> Stdlib.List.map (fun i -> i.location) + |> Stdlib.List.sortBy (fun loc -> loc.name) + |> Stdlib.List.filterMap (fun loc -> + match Packages.View.rawSource branchId (wrap loc) with + | Ok src -> Stdlib.Option.Option.Some((loc.name, src)) + | Error _ -> Stdlib.Option.Option.None) + + let rendered = + Stdlib.List.append + (render (fun l -> Packages.PackageLocation.Type l) results.types) + (Stdlib.List.append + (render (fun l -> Packages.PackageLocation.Value l) results.values) + (render (fun l -> Packages.PackageLocation.Function l) results.fns)) + + let names = rendered |> Stdlib.List.map (fun r -> let (n, _) = r in n) + let body = rendered |> Stdlib.List.map (fun r -> let (_, s) = r in s) + + (Stdlib.String.join body "\n\n", names) + + +/// `dark edit `: every declaration under it, in $EDITOR, applied atomically. +let editModule + (state: AppState) + (modulePath: List) + (fileArg: Stdlib.Option.Option) + : AppState = + let branchId = state.currentBranchId + let dotted = Stdlib.String.join modulePath "." + let moduleArg = "/" ++ dotted + + let (before, namesBefore) = moduleSource branchId modulePath + + // Reports which declarations were in the module and are not in , having applied + // nothing about them. `dark module` adds and updates; it does not unbind, and this command + // deliberately does not guess -- one mis-parse that dropped an item from the render would + // otherwise quietly end a name. + // Only after an apply that LANDED. A parse error lands nothing at all, and saying "left alone" + // about it reads as though the rest went in. + let reportOmissions (source: String) : Unit = + let gone = + namesBefore + |> Stdlib.List.filter (fun n -> + Stdlib.Bool.not (Stdlib.String.contains source n)) + + if gone == [] then + () + else + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" left alone, not in your file: {Stdlib.String.join gone ", "}") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + " editing a module does not end a name. `dark remove ` does, explicitly.") + + match fileArg with + | Some file -> + match Stdlib.Cli.File.readText file with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot read {file}: {e.message}") + + state + | Ok source -> + let result = Packages.ModuleCommand.execute state [ moduleArg, file ] + + let _ = + if result.exitCode == 0 then reportOmissions source else () + + result + | None -> + + if namesBefore == [] then + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"{dotted} has no declarations directly under it.") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + " `dark ls` shows what is there; a module of submodules is edited one level down.") + + state + else + + match Stdlib.Cli.Tui.TerminalSupport.current () with + | Unavailable reason -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot open an editor: {reason}.") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" `dark edit {dotted} ` applies a file instead, with no terminal.") + + state + | Available -> + + match Stdlib.Cli.Dir.createTemp () with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {dotted}: no temp directory ({e.message})") + + state + | Ok dir -> + let path = $"{dir}/{Stdlib.String.join modulePath "."}.dark" + + match Stdlib.Cli.File.writeText path before with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Cannot edit {dotted}: could not write {path} ({e.message})") + + state + | Ok _ -> + let editor = editorCommand () + + match Stdlib.Cli.Process.runInteractive editor [ path ] with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Could not run `{editor}`: {e.message}") + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" Set $EDITOR, or edit {path} and run `dark edit {dotted} {path}`.") + + state + | Ok code when code != 0 -> + Stdlib.printLine + $"`{editor}` exited {Stdlib.Int.toString code}; nothing applied." + + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText $" Your edit is still at {path}.") + + state + | Ok _ -> + match Stdlib.Cli.File.readText path with + | Error e -> + Stdlib.printLine + (Stdlib.Cli.UI.Colors.error $"Could not read back {path}: {e.message}") + + state + | Ok after -> + if (Stdlib.String.trim after) == (Stdlib.String.trim before) then + let _ = Stdlib.Cli.Dir.deleteRecursive dir + Stdlib.printLine (Stdlib.Cli.UI.Colors.success "no change.") + state + else + let result = Packages.ModuleCommand.execute state [ moduleArg, path ] + + let _ = + if result.exitCode == 0 then reportOmissions after else () + + // Did anything land? Asked against what the module WAS, for the same reason the + // single-item path does: the store pretty-prints, so comparing against what you typed + // says "no" almost always. + // + // The temp file is the only copy of an interactive edit, so it survives a rejection + // and the path is printed. A whole module is a lot to lose. + let (now, _) = moduleSource branchId modulePath + + if (Stdlib.String.trim now) != (Stdlib.String.trim before) then + let _ = Stdlib.Cli.Dir.deleteRecursive dir + result + else + Stdlib.printLine + (Stdlib.Cli.UI.Colors.dimText + $" nothing landed; your edit is still at {path}") + + result diff --git a/packages/darklang/cli/packages/grep.dark b/packages/darklang/cli/packages/grep.dark new file mode 100644 index 0000000000..b5d9f372b7 --- /dev/null +++ b/packages/darklang/cli/packages/grep.dark @@ -0,0 +1,258 @@ +module Darklang.Cli.Packages.Grep + +// `dark grep ` -- search the SOURCE of what is live on your branch. +// +// `dark search` looks at names. This looks at bodies, which is the one you want when you remember +// what code DID and not what it was called. After the flip there is no `packages/` tree to reach +// for instead, so this is the replacement for it. +// +// What it searches is RENDERED source, printed from the stored AST, so `///` docs are in and `//` +// asides are not -- the parser does not keep them. Worth knowing before concluding a string is +// absent. +// +// The source is not stored anywhere. The store holds ops; text is produced by the pretty-printer, +// about 9ms an item across roughly six and a half thousand of them, so grepping by rendering +// everything on demand would take a minute. It is cached instead, keyed by CONTENT HASH, which +// never needs invalidating because content is immutable: an item is rendered once per hash, ever. +// +// The cache lives BESIDE the store rather than in it, the way `credentials.db` does. It is derived, +// it is per-install, and it has no business travelling in a seed or a sync -- `Seed.export` strips +// by a denylist, so a table added to the store would ride out in every seed until someone +// remembered to name it. + +/// Where the rendered-source cache lives: beside the store, never inside it. +let cachePath () : String = + let storeDir = Stdlib.Cli.Path.parent (Stdlib.LocalStore.path ()) + Stdlib.Cli.Path.join [ storeDir, "source-cache.db" ] + + +/// Create the cache if this install has never grepped. Cheap enough to call every time. +let ensureCache () : Unit = + let sql = + "CREATE TABLE IF NOT EXISTS source_cache_v0 (item_hash TEXT PRIMARY KEY, source TEXT NOT NULL)" + + match Stdlib.Sqlite.exec (cachePath ()) sql with + | Ok _ -> () + | Error e -> + Cli.printErr $"could not open the source cache at {cachePath ()}: {e}" + () + + +/// Every live name on this branch, as (hash, owner, modules, name, itemType). +/// +/// Through the SCM silo, NOT a direct read of `locations`: that table is main's projection, so a +/// branch's own work is not in it and a grep run from a branch would search main and say nothing +/// was wrong. `allLiveBindings` lays the chain overlay over main, which is the branch-aware answer. +let liveItems (branchId: Uuid) (scope: String) : List<(String * String * String * String * String)> = + SCM.PackageOps.allLiveBindings branchId + |> Stdlib.List.filterMap (fun b -> + let dottedModule = if b.modules == "" then b.owner else $"{b.owner}.{b.modules}" + + let inScope = + (scope == "") + || (dottedModule == scope) + || (Stdlib.String.startsWith dottedModule $"{scope}.") + + if inScope then + Stdlib.Option.Option.Some((b.hash, b.owner, b.modules, b.name, b.itemType)) + else + Stdlib.Option.Option.None) + + +/// Everything this install has already rendered, as hash -> source. +/// +/// One query, not one per item. A grep over the whole store asks about six and a half thousand +/// hashes, and asking SQLite six and a half thousand times costs more than the rendering the +/// cache exists to avoid. +let cachedAll () : Dict = + match Stdlib.Sqlite.query (cachePath ()) "SELECT item_hash, source FROM source_cache_v0" with + | Error _ -> Stdlib.Dict.empty + | Ok rows -> + rows + |> Stdlib.List.fold (Stdlib.Dict.empty) (fun acc row -> + match + (Stdlib.Sqlite.textField row "item_hash", Stdlib.Sqlite.textField row "source") + with + | (Some h, Some src) -> Stdlib.Dict.set acc h src + | _ -> acc) + + +let remember (hash: String) (source: String) : Unit = + let sql = + "INSERT OR REPLACE INTO source_cache_v0 (item_hash, source) VALUES (@p0, @p1)" + + let _ = Stdlib.Sqlite.execP (cachePath ()) sql [ hash, source ] + () + + +/// A dotted name, for reporting a hit. +let dotted (owner: String) (modules: String) (name: String) : String = + if modules == "" then $"{owner}.{name}" else $"{owner}.{modules}.{name}" + + +/// The source of one item, from the cache or by rendering it. +let sourceOf + (branchId: Uuid) + (warm: Dict) + (hash: String) + (owner: String) + (modules: String) + (name: String) + (itemType: String) + : Stdlib.Option.Option = + match Stdlib.Dict.get warm hash with + | Some s -> Stdlib.Option.Option.Some s + | None -> + let modulePath = + if modules == "" then [] else Stdlib.String.split modules "." + + let location = + LanguageTools.ProgramTypes.PackageLocation + { owner = owner; modules = modulePath; name = name } + + let wrapped = + match itemType with + | "fn" -> Packages.PackageLocation.Function location + | "type" -> Packages.PackageLocation.Type location + | _ -> Packages.PackageLocation.Value location + + match Packages.View.rawSource branchId wrapped with + | Ok s -> + let _ = remember hash s + Stdlib.Option.Option.Some s + | Error _ -> Stdlib.Option.Option.None + + +/// The matching lines of one item's source, as (lineNumber, text). +let matchesIn (source: String) (pattern: String) : List<(Int * String)> = + source + |> Stdlib.String.split "\n" + |> Stdlib.List.indexedMap (fun i line -> (i + 1, line)) + |> Stdlib.List.filter (fun (_i, line) -> Stdlib.String.contains line pattern) + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + let acceptedTheWait = + args |> Stdlib.List.any (fun a -> a == "--all") + + let positional = + args |> Stdlib.List.filter (fun a -> a != "--all") + + let (pattern, scope) = + match positional with + | [ p ] -> (p, "") + | [ p, sc ] -> (p, sc) + | _ -> ("", "") + + if pattern == "" then + Cli.printErr "usage: dark grep [module] [--all]" + + Cli.printHint + " a module scopes the search and is much quicker; --all accepts the wait on a cold cache." + + { state with exitCode = 1 } + else + + let _ = ensureCache () + let items = liveItems state.currentBranchId scope + let warm = cachedAll () + let total = Stdlib.List.length items + + // Said once, before the wait rather than after it: the first grep on an install renders + // everything, and a minute of silence reads as a hang. + let anyCached = + match items with + | (h, _, _, _, _) :: _ -> (Stdlib.Dict.get warm h) != Stdlib.Option.Option.None + | [] -> true + + // An unscoped grep against a cold cache renders everything, which is minutes. Refused rather + // than done silently: a command that hangs for five minutes without saying why is one people + // stop trusting, and this one is also run by the registry sweep, where it cost nine minutes + // of the suite before it asked first. + if (scope == "") && (Stdlib.Bool.not anyCached) && (Stdlib.Bool.not acceptedTheWait) then + Cli.printErr + $"searching everything means rendering {Stdlib.Int.toString total} items, which takes minutes on this install." + + Cli.printHint " `dark grep ` searches under one module and is quick." + Cli.printHint " `dark grep --all` does it anyway; the wait is paid once, then cached." + { state with exitCode = 1 } + else + + if Stdlib.Bool.not anyCached then + Cli.printHint + $" rendering {Stdlib.Int.toString total} items. This is paid once; later greps read the cache." + + // Rendered per item, keeping the failures rather than dropping them: an item whose source + // cannot be produced is NOT an item without matches, and a search tool that cannot tell you + // the difference is one you cannot trust a negative answer from. + let outcomes = + items + |> Stdlib.List.map (fun (hash, owner, modules, name, itemType) -> + (dotted owner modules name, + sourceOf state.currentBranchId warm hash owner modules name itemType)) + + let unreadable = + outcomes + |> Stdlib.List.filter (fun (_where, source) -> + match source with + | None -> true + | Some _ -> false) + + let hits = + outcomes + |> Stdlib.List.map (fun (where, source) -> + match source with + | None -> [] + | Some s -> + matchesIn s pattern + |> Stdlib.List.map (fun (lineNo, line) -> + $"{where}:{Stdlib.Int.toString lineNo}: {Stdlib.String.trim line}")) + |> Stdlib.List.flatten + + let unreadableCount = Stdlib.List.length unreadable + + if unreadableCount > 0 then + Cli.printErr + $"could not render {Stdlib.Int.toString unreadableCount} of {Stdlib.Int.toString total} items; they were NOT searched." + + unreadable + |> Stdlib.List.take 3 + |> Stdlib.List.map (fun (where, _s) -> $" {where}") + |> Stdlib.printLines + + match hits with + | [] -> + Cli.printOk $"no live item's source contains {pattern}." + state + | _ -> + hits |> Stdlib.printLines + Cli.printHint + $" {Stdlib.Int.toString (Stdlib.List.length hits)} line(s). `dark search` looks at names instead." + + state + + + +let help (_state: Cli.AppState) : String = + [ "" + " dark grep [module] [--all]" + "" + " Searches the SOURCE of everything live on your branch, and prints" + " name:line: for each hit. `dark search` searches names instead." + "" + " Plain substring matching, not a regex." + "" + " It searches RENDERED source, which is printed from the stored AST. So" + " `///` docs are searchable and `//` asides are not: those are not kept." + "" + " The first run on an install renders every item and caches the result" + " beside the store, keyed by content hash. Later runs read the cache, and" + " because content is immutable it never needs invalidating." ] + |> Stdlib.String.join "\n" + + +let complete + (_state: Cli.AppState) + (_args: List) + : List = + [] diff --git a/packages/darklang/cli/packages/remove.dark b/packages/darklang/cli/packages/remove.dark new file mode 100644 index 0000000000..f92131e8c4 --- /dev/null +++ b/packages/darklang/cli/packages/remove.dark @@ -0,0 +1,131 @@ +module Darklang.Cli.Packages.Remove + +// `dark remove ` -- take a name off the shelf. +// +// One op, `Unbind`, and it is the only thing that writes one. Until now a name could be bound and +// rebound but never ENDED, so a name authored by mistake lived forever and greeted strangers as a +// constraint. +// +// What it does not do is throw anything away. The content is untouched and stays in the store, +// reachable by hash, so everything that calls it goes on working: a reference points at content, +// not at a name. What ends is the name. +// +// Not `dark delete`, which is deprecation -- that leaves the name bound and marks it obsolete, so +// callers still resolve it and are told why. Deprecation is for something that still exists and +// should not be used; removal is for a name that should not exist. + +/// The op a removal authors. +/// +/// `previous = Some hash` records what the name held, for the same reason `SetName` carries one: +/// a merge on the other side wants to know what was replaced rather than guessing. +let ops + (location: LanguageTools.ProgramTypes.PackageLocation) + (reference: LanguageTools.ProgramTypes.Reference) + : List = + [ LanguageTools.ProgramTypes.PackageOp.Unbind( + location, + Stdlib.Option.Option.Some(LanguageTools.ProgramTypes.Reference.hash reference) + ) ] + + +/// `-y` / `--yes` anywhere in the arguments, and everything that is not a flag. +let parseArgs (args: List) : (Bool * List) = + let autoConfirm = + args |> Stdlib.List.any (fun a -> (a == "-y") || (a == "--yes")) + + let rest = + args + |> Stdlib.List.filter (fun a -> + Stdlib.Bool.not ((a == "-y") || (a == "--yes"))) + + (autoConfirm, rest) + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + let (autoConfirm, rest) = parseArgs args + + match rest with + | [ name ] -> + let here = state.packageData.currentLocation + + match Packages.Location.parseRelativeTo here name with + | Error e -> + Cli.printErr $"the name to remove: {e}" + { state with exitCode = 1 } + + | Ok location -> + let dotted = + PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location + + match Packages.Rename.referenceAt state.currentBranchId location with + | None -> + Cli.printErr $"nothing here is named {dotted}." + Cli.printHint " `dark ls` lists what is; `dark search ` looks wider." + { state with exitCode = 1 } + + | Some reference -> + // `confirmExplicit`, not `confirm`: a stray Return should not end a name. An unattended + // run says `-y`, which is the only thing that means yes without a person. + let proceed = + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + $"Remove the name {dotted}? Its content stays and its callers keep working. (y/n): ") + + if proceed then + match SCM.PackageOps.add state.currentBranchId (ops location reference) with + | Ok _ -> + Cli.printOk $"removed the name {dotted}." + + Cli.printHint + " the content is still in the store and its callers still resolve it: they reference content, not names." + + Cli.printHint " `dark status` shows the change; it travels when you sync." + + state + + | Error e -> + Cli.printErr $"couldn't remove: {e}" + { state with exitCode = 1 } + else + Cli.printOk "left it alone." + state + + | [] -> + Cli.printErr "usage: dark remove [-y]" + Cli.printHint " the name is relative to where you are (`dark nav` moves)." + { state with exitCode = 1 } + + | _ -> + Cli.printErr "usage: dark remove [-y]" + Cli.printHint " one name at a time, so the confirmation can say what it is ending." + { state with exitCode = 1 } + + +let help (_state: Cli.AppState) : String = + [ "" + " dark remove [-y]" + "" + " Ends a name. The content it held is untouched: it stays in the store," + " reachable by hash, and everything that calls it goes on working, because" + " a reference points at content rather than at a name." + "" + " Use it when the NAME was the mistake. For something that still exists and" + " should not be used, `dark delete` deprecates instead, which leaves the name" + " bound and tells callers why." + "" + " One `Unbind` op is authored, so `dark status` shows it and it travels when" + " you sync." + "" + " -y, --yes skip the confirmation; required for an unattended run" ] + |> Stdlib.String.join "\n" + + +let complete + (state: Cli.AppState) + (args: List) + : List = + match args with + | [] -> Packages.Nav.complete state args + | [ _ ] -> Packages.Nav.complete state args + | _ -> [] diff --git a/packages/darklang/cli/packages/revert.dark b/packages/darklang/cli/packages/revert.dark new file mode 100644 index 0000000000..a886e2ea0e --- /dev/null +++ b/packages/darklang/cli/packages/revert.dark @@ -0,0 +1,147 @@ +module Darklang.Cli.Packages.Revert + +// `dark revert ` -- put a name back to what it held at a past commit. +// +// Nothing is undone and nothing is recovered: every version an item ever had is still in the +// store, because content is content-addressed and never deleted. So a revert is a rebinding -- +// one `SetName`, pointing the name at a hash it used to point at. +// +// Which means it is symmetric. Reverting a revert is just another revert, and `dark log ` +// still shows every version including the one you moved away from. +// +// "At that commit" means the commit or any of its ANCESTORS, not the ops that commit happens to +// name. You give a commit to mean a point in time, and the version you want was usually bound by +// something earlier. + +/// The op a revert authors: bind the name to the old hash, recording what it replaced. +/// +/// `previous = Some current` is the lineage, and it is what stops a peer's merge from reading this +/// as two machines inventing the same name from nothing. +let ops + (location: LanguageTools.ProgramTypes.PackageLocation) + (target: LanguageTools.ProgramTypes.Reference) + (current: LanguageTools.ProgramTypes.Hash) + : List = + [ LanguageTools.ProgramTypes.PackageOp.SetName( + location, + target, + Stdlib.Option.Option.Some current + ) ] + + +/// The same kind of reference the name holds now, pointing at . A revert cannot change +/// what KIND of thing a name holds -- a fn cannot become a type -- so the current binding decides. +let referenceFor + (current: LanguageTools.ProgramTypes.Reference) + (hash: LanguageTools.ProgramTypes.Hash) + : LanguageTools.ProgramTypes.Reference = + match current with + | PackageFn _ -> LanguageTools.ProgramTypes.Reference.PackageFn hash + | PackageType _ -> LanguageTools.ProgramTypes.Reference.PackageType hash + | PackageValue _ -> LanguageTools.ProgramTypes.Reference.PackageValue hash + + +let execute (state: Cli.AppState) (args: List) : Cli.AppState = + match args with + | [ name, commit ] -> + let here = state.packageData.currentLocation + + match Packages.Location.parseRelativeTo here name with + | Error e -> + Cli.printErr $"the name to revert: {e}" + { state with exitCode = 1 } + + | Ok location -> + let dotted = + PrettyPrinter.ProgramTypes.PackageLocation.packageLocation location + + match Packages.Rename.referenceAt state.currentBranchId location with + | None -> + Cli.printErr $"nothing here is named {dotted}." + Cli.printHint " `dark log` shows what has been committed; `dark ls` what is live." + { state with exitCode = 1 } + + | Some current -> + let currentHash = LanguageTools.ProgramTypes.Reference.hash current + + let currentHashText = + LanguageTools.ProgramTypes.hashToString currentHash + + match SCM.PackageOps.resolveCommit commit with + | None -> + Cli.printErr $"no single commit here starts with {commit}." + Cli.printHint " `dark log` lists them; a longer prefix disambiguates." + { state with exitCode = 1 } + + | Some fullCommit -> + + match SCM.PackageOps.bindingAsOfCommit location fullCommit with + | None -> + Cli.printErr $"{dotted} was not bound at {commit} or anywhere before it." + Cli.printHint $" `dark log {name}` shows the commits that touched it." + + Cli.printHint + " a revert goes back to what the name held THEN, so one added later has nothing to go back to." + + { state with exitCode = 1 } + + | Some oldHash -> + if oldHash == currentHashText then + Cli.printOk + $"{dotted} already holds what it held at {commit} -- nothing to do." + + state + else + let target = + referenceFor current (LanguageTools.ProgramTypes.Hash.Hash oldHash) + + match + SCM.PackageOps.add + state.currentBranchId + (ops location target currentHash) + with + | Ok _ -> + let short = Stdlib.String.slice oldHash 0 8 + Cli.printOk $"reverted {dotted} to the version it held at {commit} ({short})." + + Cli.printHint + " nothing was thrown away: the version you moved off is still in the store, and `dark log` still lists it." + + Cli.printHint " `dark status` shows the change; it travels when you sync." + + state + + | Error e -> + Cli.printErr $"couldn't revert: {e}" + { state with exitCode = 1 } + + | _ -> + Cli.printErr "usage: dark revert " + Cli.printHint " reverts the name to what it held at that commit, or at any commit before it." + { state with exitCode = 1 } + + +let help (_state: Cli.AppState) : String = + [ "" + " dark revert " + "" + " Points a name back at the version it held at that commit. One `SetName`:" + " nothing is undone and nothing is recovered, because every version is still" + " in the store and a name is just a binding." + "" + " \"At that commit\" includes every commit before it, so you can name a point" + " in history rather than the exact commit that happened to rebind the name." + "" + " Symmetric: reverting a revert is another revert, and `dark log` keeps" + " showing every version either way." ] + |> Stdlib.String.join "\n" + + +let complete + (state: Cli.AppState) + (args: List) + : List = + match args with + | [] -> Packages.Nav.complete state args + | [ _ ] -> Packages.Nav.complete state args + | _ -> [] diff --git a/packages/darklang/cli/registry.dark b/packages/darklang/cli/registry.dark index e3b3f643ae..4d603a3253 100644 --- a/packages/darklang/cli/registry.dark +++ b/packages/darklang/cli/registry.dark @@ -31,6 +31,7 @@ module Registry = ("scripts", "Store, manage, and run Dark scripts", [], Cli.Scripts.execute, Cli.Scripts.help, Cli.Scripts.complete) ("view", "View details of a module, type, value, or fns", [], Packages.View.execute, Packages.View.help, Packages.View.complete) ("tree", "Display package hierarchy in tree format", [ "packages" ], Packages.Tree.execute, Packages.Tree.help, Packages.Tree.complete) + ("grep", "Search the source of what is live (`search` looks at names)", [], Packages.Grep.execute, Packages.Grep.help, Packages.Grep.complete) ("search", "Search modules, types, functions, and values", [], Packages.Search.execute, Packages.Search.help, Packages.Search.complete) ("typecheck", "Audit package declarations without changing them", [], Packages.Typecheck.execute, Packages.Typecheck.help, Packages.Typecheck.complete) ("deps", "Show dependencies and dependents", [ "dependencies" ], Cli.Deps.execute, Cli.Deps.help, Cli.Deps.complete) @@ -40,6 +41,8 @@ module Registry = ("module", "Define multiple package declarations", [], Packages.ModuleCommand.execute, Packages.ModuleCommand.help, Packages.ModuleCommand.complete) ("edit", "Change an item without retyping it", [], Packages.Edit.execute, Packages.Edit.help, Packages.Edit.complete) ("rename", "Give an item a different name (its content and callers are untouched)", [], Packages.Rename.execute, Packages.Rename.help, Packages.Rename.complete) + ("revert", "Put a name back to what it held at a past commit", [], Packages.Revert.execute, Packages.Revert.help, Packages.Revert.complete) + ("remove", "End a name (the content stays, and its callers keep working)", [], Packages.Remove.execute, Packages.Remove.help, Packages.Remove.complete) ("hash", "Show hash of a package item", [], Packages.Hash.execute, Packages.Hash.help, Packages.Hash.complete) // SCM commands ("status", "What's in your draft, in one line", [ "wip", "changes" ], Cli.Status.execute, Cli.Status.help, Cli.Status.complete) @@ -79,6 +82,7 @@ module Registry = ("db", "Manage databases", [], Packages.DB.execute, Packages.DB.help, Packages.DB.complete) ("outliner", "Interactive tree outliner", [ "tree-exp" ], Outliner.App.execute, Outliner.App.help, Outliner.App.complete) ("export-seed", "Export minimal seed.db from current database", [], ExportSeed.execute, ExportSeed.help, ExportSeed.complete) + ("store", "What this store is, and moving it between op-log formats", [], Store.execute, Store.help, Store.complete) ("traces", "List and view execution traces", [ "trace" ], Tracing.execute, Tracing.help, Tracing.complete) ("review", "Stage someone else's ops for op-level approval", [], Cli.Sync.executeReviewCmd, Cli.Sync.helpReview, Cli.Sync.complete) ("conflicts", "Review what a merge, rebase or sync auto-resolved", [], Conflicts.execute, Conflicts.help, Conflicts.complete) @@ -197,7 +201,7 @@ module Registry = // Command groups organized by category (shared between compact and detailed views) let commandGroups () : List<(String * List)> = - [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "hash", "db", "deprecate", "delete", "undeprecate" ]) + [ ("Packages", [ "nav", "ls", "view", "tree", "back", "search", "grep", "typecheck", "deps", "val", "fn", "type", "module", "edit", "rename", "remove", "revert", "hash", "db", "deprecate", "delete", "undeprecate" ]) ("Changes", [ "status", "commit", "commits", "show", "discard", "undo", "propagate", "conflicts", "constraints", "ack", "ops" ]) ("Branches", [ "branch", "branches", "switch", "merge", "rebase", "diff", "log", "resolve" ]) @@ -208,7 +212,7 @@ module Registry = ("Security", [ "permissions" ]) ("Network", [ "devices" ]) ("Apps", [ "apps" ]) - ("Utilities", [ "clear", "help", "docs", "quit", "builtins", "export-seed", "views", "workbench" ]) ] + ("Utilities", [ "clear", "help", "docs", "quit", "builtins", "export-seed", "store", "views", "workbench" ]) ] let formatCommandGroup (groupName: String) (commandNames: List) (commands: List) : String = let formattedCommands = diff --git a/packages/darklang/cli/store.dark b/packages/darklang/cli/store.dark new file mode 100644 index 0000000000..483db1cc37 --- /dev/null +++ b/packages/darklang/cli/store.dark @@ -0,0 +1,158 @@ +module Darklang.Cli.Store + +// `dark store` -- what this store IS, and moving it between op-log formats. +// +// Distinct from `dark backups`, which is about copies of a store, and from `dark status`, which is +// about the code in one. This is about the FILE: which format its ops are written in, which commit +// it was cut at, which build cut it. +// +// After the flip there is no `packages/` to rebuild a store from, so a format change has to carry +// the store forward in place. `upgrade` is that, and it is a no-op until the first real bump -- it +// exists now so that the bump is not also the first time the migration runs. + +let help (_state: AppState) : String = + [ "What this store is, and moving it between op-log formats." + "" + "Usage:" + " store what this store says about itself" + " store upgrade move the op log to the format this build writes" + " store rollback put back the copy `upgrade` took on its way to format " + "" + "`upgrade` copies the store first and rewrites in one transaction. It only rewrites" + "blobs: a change that moves content hashes is a larger migration, and it refuses" + "rather than half-doing it." + "" + "Both write, so both need --yes when nothing is watching." ] + |> Stdlib.String.join "\n" + + +/// `dark store` with no arguments: the stamp, as it stands. +let showInfo (state: AppState) : AppState = + let rows = SCM.StoreMeta.all () + + if rows == [] then + Stdlib.printLine + "This store carries no stamp. It predates one; `store upgrade` writes it." + + state + else + rows + |> Stdlib.List.iter (fun row -> + let (k, v) = row + let key = Stdlib.String.padEndToWidth k 8 + Stdlib.printLine $"{key} {v}") + + // Naming the verb here, because this is where someone looks after an upgrade went wrong and + // `dark backups` does not list the copy an upgrade takes. The verb knows its own filename; it + // says so plainly if there is no copy to put back. + let format = SCM.StoreMeta.get "format" + + if (format != "") && (format != "1") then + Stdlib.printLine "" + + Stdlib.printLine + $" `store rollback {format}` puts back the copy taken on the way here." + else + () + + state + + +let executeUpgrade (state: AppState) (args: List) : AppState = + let (autoConfirm, _rest) = Packages.Remove.parseArgs args + + // A whole-log rewrite, so it asks. `confirmExplicit` rather than `confirm`: a stray Return must + // not start one, and an unattended run says `-y`. + if + Stdlib.Bool.not ( + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + "Rewrite every op in this store into this build's format? A copy is taken first. (y/n): ") + ) + then + state + else + match Stdlib.LocalStore.upgrade () with + | Error e -> + Stdlib.printLine (View.formatError e) + state + | Ok report -> + Stdlib.printLine + $"Upgraded from format {Stdlib.Int64.toString report.from} to {Stdlib.Int64.toString report.to}." + + Stdlib.printLine + $"{Stdlib.Int64.toString report.rewritten} op(s) rewritten; the previous store is at {report.backup}." + + if report.unreadable > 0L then + Stdlib.printLine + $"{Stdlib.Int64.toString report.unreadable} op(s) this build cannot read were left as they are, so a later build can still apply them." + else + () + + Stdlib.printLine "Projections were re-folded. Nothing further to do." + state + + +let executeRollback (state: AppState) (args: List) : AppState = + match args with + | [] -> + Stdlib.printLine (View.formatError "Usage: store rollback ") + state + | target :: rest -> + match Stdlib.Int64.parse target with + | Error _ -> + Stdlib.printLine + (View.formatError + $"`store rollback` takes the format version that was upgraded TO, and \"{target}\" isn't a number") + + state + | Ok n -> + let (autoConfirm, _rest) = Packages.Remove.parseArgs rest + + if + Stdlib.Bool.not ( + Cli.Prompt.confirmExplicit + autoConfirm + (Stdlib.Cli.UI.Colors.warning + $"Replace this store with the copy taken before the move to format {target}? (y/n): ") + ) + then + state + else + match Stdlib.LocalStore.rollbackTo n with + | Error e -> + Stdlib.printLine (View.formatError e) + state + | Ok path -> + Stdlib.printLine $"Restored from {path}. Restart to pick it up." + state + + +let execute (state: AppState) (args: List) : AppState = + match args with + | [] -> showInfo state + | [ "help" ] -> + Stdlib.printLine (help state) + state + | "upgrade" :: rest -> executeUpgrade state rest + | "rollback" :: rest -> executeRollback state rest + | other -> + Stdlib.printLine + (View.formatError + $"`store` takes nothing, `upgrade` or `rollback`; not \"{Stdlib.String.join other " "}\"") + + Stdlib.printLine (help state) + state + + +let complete (_state: AppState) (args: List) : List = + match args with + | [] -> [ "upgrade", "rollback", "help" ] |> Stdlib.List.map Completion.simple + | [ partial ] -> + [ "upgrade", "rollback", "help" ] + |> Stdlib.List.filter (fun c -> Stdlib.String.startsWith c partial) + |> Stdlib.List.map Completion.simple + // `rollback` takes a format version, which only the store knows and only one of which is ever + // right, so there is nothing useful to offer past the subcommand. + | _ -> [] diff --git a/packages/darklang/scm/packageOpsBindings.dark b/packages/darklang/scm/packageOpsBindings.dark index cbbcd2b576..2a2f404596 100644 --- a/packages/darklang/scm/packageOpsBindings.dark +++ b/packages/darklang/scm/packageOpsBindings.dark @@ -233,6 +233,126 @@ let chainBindings (branchId: Uuid) : List = chainBindingEntries branchId |> Stdlib.List.filterMap (fun w -> w.binding) +/// The full hash of the commit names, or None when nothing or more than one does. +/// +/// People type what `dark log` printed, which is the short form. Ambiguity is None rather than a +/// guess: two commits sharing a prefix is rare and picking one silently would revert a name to +/// the wrong history. +/// +/// main-scoped: commits are main's history. A branch's own work is in its overlay, not here. +let resolveCommit (prefix: String) : Stdlib.Option.Option = + let sql = "SELECT hash FROM commits WHERE hash LIKE @p0 || '%' LIMIT 2" + + match Stdlib.Sqlite.queryP (Stdlib.LocalStore.path ()) sql [ prefix ] with + | Ok [ row ] -> Stdlib.Sqlite.textField row "hash" + | _ -> Stdlib.Option.Option.None + + +/// What was bound to AS OF : the newest binding written by any op in +/// that commit or one of its ancestors. `None` when the name was not bound anywhere in that history. +/// +/// Ancestry rather than "ops in that commit": you ask about a commit to mean a point in time, and +/// the name was usually bound by something earlier. `commits.parent` is the chain, and it exists +/// because a commit's id is derived over its parent, the way git's is. +/// +/// History is main's, so the read is main-scoped. A past binding is a fact about what was +/// committed, and a branch has not committed anything -- its own work is in its overlay, which +/// `chainBindingEntries` answers for and this deliberately does not. +let bindingAsOfCommit + (loc: LanguageTools.ProgramTypes.PackageLocation) + (commitHash: String) + : Stdlib.Option.Option = + let modules = Stdlib.String.join loc.modules "." + + // main-scoped: this asks what main's history HELD, including bindings long superseded, so it + // reads unlisted rows too -- which is the one question `locations` is the right table for. + let sql = + "WITH RECURSIVE ancestry(h) AS ( + SELECT @p0 + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent != '' + ) + SELECT l.item_hash + FROM locations l JOIN package_ops p ON p.id = l.op_id + WHERE l.owner = @p1 AND l.modules = @p2 AND l.name = @p3 + AND l.source != 'unbind' + AND p.commit_hash IN (SELECT h FROM ancestry) + ORDER BY l.origin_ts DESC, l.rowid DESC + LIMIT 1" + + match + Stdlib.Sqlite.queryP + (Stdlib.LocalStore.path ()) + sql + [ commitHash, loc.owner, modules, loc.name ] + with + | Ok (row :: _) -> Stdlib.Sqlite.textField row "item_hash" + | _ -> Stdlib.Option.Option.None + + +/// EVERY name live on , as bindings. The enumerating form of . +/// +/// Main's projection, then the chain overlay laid over it: a branch's SetName replaces main's entry +/// for that name, and a name the chain left unbound is dropped even though main still holds it. +/// +/// Here rather than at the call site because the overlay is the whole point. A caller that wanted +/// "everything live" and read `locations` would get MAIN's list while standing on a branch, and get +/// it plausibly -- which is the class of bug this silo exists to stop. +let allLiveBindings (branchId: Uuid) : List = + let mainRows = + // main-scoped: deliberately main's projection, as the BASE the chain overlay below replaces + // entries in. Reading it alone would be the bug; reading it as the floor is the design. + let sql = + "SELECT owner, modules, name, item_type, item_hash, COALESCE(origin_ts, '') AS origin_ts + FROM locations + WHERE unlisted_at IS NULL AND source != 'unbind'" + + match Stdlib.Sqlite.query (Stdlib.LocalStore.path ()) sql with + | Error _ -> [] + | Ok rows -> + rows + |> Stdlib.List.filterMap (fun row -> + let get (col: String) : Stdlib.Option.Option = + Stdlib.Sqlite.textField row col + + match + (get "owner", get "modules", get "name", get "item_type", get "item_hash") + with + | (Some o, Some m, Some n, Some t, Some h) -> + Stdlib.Option.Option.Some( + Conflicts.Binding + { owner = o + modules = m + name = n + itemType = t + hash = h + originTs = (get "origin_ts") |> Stdlib.Option.withDefault "" + author = "" + previous = Stdlib.Option.Option.None } + ) + | _ -> Stdlib.Option.Option.None) + + if SCM.Branch.isMain branchId then + mainRows + else + let words = chainBindingEntries branchId + + let key (o: String) (m: String) (n: String) : String = $"{o}|{m}|{n}" + + let spokenFor = + words + |> Stdlib.List.fold Stdlib.Dict.empty (fun acc w -> + Stdlib.Dict.set acc (key w.owner w.modules w.name) true) + + // Main's entries that the chain has NOT rebound or unbound, plus whatever the chain leaves bound. + let untouched = + mainRows + |> Stdlib.List.filter (fun b -> + (Stdlib.Dict.get spokenFor (key b.owner b.modules b.name)) == Stdlib.Option.Option.None) + + Stdlib.List.append untouched (words |> Stdlib.List.filterMap (fun w -> w.binding)) + + /// What resolves to on : the chain overlay first, then main's projection, /// which is where every chain ends. THE branch-aware read of a live binding. /// diff --git a/packages/darklang/scm/packageOpsCommits.dark b/packages/darklang/scm/packageOpsCommits.dark index ece0bfdb38..094746f102 100644 --- a/packages/darklang/scm/packageOpsCommits.dark +++ b/packages/darklang/scm/packageOpsCommits.dark @@ -72,6 +72,48 @@ let count () : Int64 = |> Stdlib.Option.withDefault 0L +/// What a seed cut at would carry: (ops, commits). `None` when no commit here +/// has that hash. +/// +/// The same ancestry walk `LibDB.Seed.exportAt` cuts with, and the same exclusion of branch ops, so +/// the numbers match what a fetch actually gets. Exists so a pin can be CHECKED without cutting one: +/// the answer is two counts, and the seed is twelve megabytes. +let seedShapeAt (commitHash: String) : Stdlib.Option.Option<(Int64 * Int64)> = + let sql = + "WITH RECURSIVE ancestry(h) AS ( + SELECT hash FROM commits WHERE hash = @p0 + UNION + SELECT c.parent FROM commits c JOIN ancestry a ON c.hash = a.h WHERE c.parent <> '' + ) + SELECT + (SELECT count(*) FROM package_ops + WHERE commit_hash IN (SELECT h FROM ancestry) + AND id NOT IN (SELECT op_id FROM op_branches)) AS ops, + (SELECT count(*) FROM ancestry) AS commits" + + match Stdlib.Sqlite.queryOneP (localDb ()) sql [ commitHash ] with + | None -> Stdlib.Option.Option.None + | Some row -> + match (Stdlib.Sqlite.intField row "ops", Stdlib.Sqlite.intField row "commits") with + // A hash nobody here has gives an empty ancestry, so zero commits. That is "not found", not a + // seed with nothing in it. + | (Some _, Some 0L) -> Stdlib.Option.Option.None + | (Some ops, Some commits) -> Stdlib.Option.Option.Some((ops, commits)) + | _ -> Stdlib.Option.Option.None + + +/// How many ops are COMMITTED on main, which is what a seed cut from here carries. +/// +/// Not : that counts the draft and every branch's delta too, and a seed carries neither. +let committedCountOnMain () : Int64 = + Stdlib.Sqlite.scalarInt + (localDb ()) + "SELECT count(*) AS n FROM package_ops + WHERE commit_hash IS NOT NULL AND id NOT IN (SELECT op_id FROM op_branches)" + "n" + |> Stdlib.Option.withDefault 0L + + /// Which ops a branch can SEE: main's log, plus that branch's own delta ops. Another branch's ops are in /// the same table but invisible from here -- that isolation is the point of the overlay. /// diff --git a/packages/darklang/scm/storeMeta.dark b/packages/darklang/scm/storeMeta.dark new file mode 100644 index 0000000000..d85e06b073 --- /dev/null +++ b/packages/darklang/scm/storeMeta.dark @@ -0,0 +1,42 @@ +module Darklang.SCM.StoreMeta + +// The `store_meta` silo: what a store says about ITSELF, as opposed to about the code in it. +// +// Written by the seed cut (`LibDB.Seed.exportAt`, the only thing that knows what it just made) and +// topped up at open by `LibDB.Releases`, so every store carries one whether it came from a seed or +// grew here. Four keys today: +// +// format the op-blob layout version. A reader that understands a different one must refuse. +// cut_at the commit this store was cut at, so a seed says what it is a cut of. +// kernel the build that cut it, for tracing a bad seed back to a binary. +// at when. +// +// Read-only from Dark: the values are facts about a file, and nothing in package code is in a +// position to know better than the thing that wrote them. + + +/// The value of , or "" when this store carries no such key. +/// +/// "" rather than an Option: a store older than the stamp has none of these, which every caller +/// treats the same way as "not recorded" -- the same choice `Stdlib.LocalStore.configGet` makes. +let get (key: String) : String = + match + Stdlib.Sqlite.queryOneP + (localDb ()) + "SELECT value FROM store_meta WHERE key = @p0" + [ key ] + with + | Some row -> Stdlib.Sqlite.textField row "value" |> Stdlib.Option.withDefault "" + | None -> "" + + +/// Everything this store says about itself, for `dark status` and the seed endpoints. +let all () : List<(String * String)> = + Stdlib.Sqlite.rowsOrEmptyP + (localDb ()) + "SELECT key, value FROM store_meta ORDER BY key" + [] + |> Stdlib.List.filterMap (fun row -> + match (Stdlib.Sqlite.textField row "key", Stdlib.Sqlite.textField row "value") with + | (Some k, Some v) -> Stdlib.Option.Option.Some((k, v)) + | _ -> Stdlib.Option.Option.None) diff --git a/packages/darklang/stdlib/localStore.dark b/packages/darklang/stdlib/localStore.dark index de5fea2d0f..e3dd811ad0 100644 --- a/packages/darklang/stdlib/localStore.dark +++ b/packages/darklang/stdlib/localStore.dark @@ -44,3 +44,70 @@ let backupTo (path: String) : Stdlib.Result.Result = /// memory is still the OLD store, though, so the caller should say to restart. let restoreFrom (path: String) : Stdlib.Result.Result = Builtin.localDbRestoreFrom path + + +/// Cut a SEED of this store to : the op log and its commits, with every per-instance +/// table stripped, so a fresh install that folds it agrees on every id. +/// +/// narrows it to that commit and its ancestors, which is what makes a pinned +/// fetch reproducible: the same commit yields the same OPS however far the store has moved since, +/// and ids come from op content, so every store built from it agrees. `None` takes everything +/// committed. +/// +/// The file carries a `store_meta` stamp saying which commit it was cut at, which format its op +/// blobs are in, and which build cut it. So a seed says what it is without anyone having to +/// remember where it came from. +let seedTo + (path: String) + (upToCommit: Stdlib.Option.Option) + : Stdlib.Result.Result = + Builtin.pmSeedExport path upToCommit + + +/// What an upgrade did. Structured rather than a sentence: a caller decides what to print, and +/// `unreadable > 0` is a thing a script might want to act on. +type UpgradeReport = + { /// The format the store was in. + from: Int64 + /// The format it is in now, which is the one this build writes. + to: Int64 + /// Ops decoded and written back out. + rewritten: Int64 + /// Ops this build cannot read at all: a peer's newer format, stored inert. Left exactly as + /// they are, because re-encoding is impossible and dropping them would lose work a later + /// build can still apply. + unreadable: Int64 + /// Where the pre-upgrade store was copied to, which is what puts back. + backup: String } + + +/// Move this store's op log to the format this build writes. +/// +/// A no-op until the first format bump: with nothing to migrate between it says so rather than +/// touching anything. The copy lands before any write, and the rewrite is one transaction, because +/// a half-converted store has some ops in one format and some in another with nothing recording +/// which -- and no reader can sort that out afterwards. +/// +/// Only rewrites BLOBS. A change that moves content hashes is a different and much larger +/// migration; this refuses rather than half-doing it. See `LibDB.StoreUpgrade`. +let upgrade () : Stdlib.Result.Result = + match Builtin.pmStoreUpgrade () with + | Error e -> Stdlib.Result.Result.Error e + | Ok parts -> + let (from, to, rewritten, unreadable, backup) = parts + + (UpgradeReport + { from = from + to = to + rewritten = rewritten + unreadable = unreadable + backup = backup }) + |> Stdlib.Result.Result.Ok + + +/// Put back the copy took on its way to . +/// +/// Contents, not the file, so connections already open keep working. Anything already read into +/// memory is still the upgraded store, so the caller should say to restart. +let rollbackTo (target: Int64) : Stdlib.Result.Result = + Builtin.pmStoreRollback target diff --git a/packages/darklang/sync/relay/protocol.dark b/packages/darklang/sync/relay/protocol.dark index 31ac57c11e..39fc82ebdf 100644 --- a/packages/darklang/sync/relay/protocol.dark +++ b/packages/darklang/sync/relay/protocol.dark @@ -131,6 +131,30 @@ let pushHandlerFn (req: HttpRequest) : HttpResponse = match SCM.Wire.wireDecode req.body with | Ok bundle -> + // What this server's MAIN will accept from a stranger. Your own namespace takes a plain push; + // the reviewed one does not, and a branch push is how that work arrives instead. 403 rather + // than 500: this is a refusal, and a client that reads it as a server fault retries forever. + let reserved = SCM.Wire.reservedBindingsIn bundle.ops + + if reserved != [] then + let shown = reserved |> Stdlib.List.take 5 |> Stdlib.String.join ", " + + let andMore = + let extra = (Stdlib.List.length reserved) - 5 + + if extra > 0 then + $" (and {Stdlib.Int.toString extra} more)" + else + "" + + Stdlib.Http.responseWithText + ($"this server does not accept pushes that bind {shown}{andMore} into its main. +" + ++ "That namespace is reviewed: `dark branch push` instead, and it lands on main when " + ++ "the change is merged. Your own namespace takes a plain `dark push`.") + 403 + else + match SCM.Wire.storeWithOwner owner bundle.ops bundle.commits with | Ok n -> // A JSON body, not a sentence: the client acts on this count, and a protocol a client @@ -255,3 +279,237 @@ let branchOwnersHandlerFn (req: HttpRequest) : HttpResponse = match branchAccessAllowed req with | Error why -> Stdlib.Http.responseWithText why 401 | Ok _ -> Stdlib.Http.responseWithText (SCM.PackageOps.relayListAllBranches ()) 200 + + +// --------------------- +// Seeds +// --------------------- +// +// A SEED is the op log as a file: what a fresh install folds to become this store, and what CI fetches +// instead of parsing `packages/` out of text. It is cut from the SAME log `/sync/pull` pages out, so the +// two can never disagree about what this relay holds. +// +// Cut AT A COMMIT, always, even when the caller asked for "latest": that is what makes a pin +// reproducible. Two machines asking for the same commit get the same OPS however far the store has +// moved in between, which is the property `scripts/packages/pin` rests on. The file is not +// byte-identical across servers -- the stamp says which build cut it and when -- and nothing needs +// it to be, since ids are derived from op content rather than from the file. + +val seedRoute = "/seed/:file" + + +/// Where a cut seed is cached: beside the store, named for the commit it was cut at. +/// +/// A cut is decided entirely by its commit -- commits are immutable and ops only ever append -- so a +/// file here never needs invalidating. Beside the store rather than inside it, like the source cache: +/// it is derived, it is per-install, and it has no business travelling inside a seed of its own. +let seedCachePath (commit: String) : String = + let storeDir = Stdlib.Cli.Path.parent (Stdlib.LocalStore.path ()) + Stdlib.Cli.Path.join [ storeDir, "seeds", $"{commit}.db" ] + + +/// The commit main is at here, or "" when nothing has been committed yet. +let headCommit () : String = + match SCM.Commits.recent 1L with + | c :: _ -> c.hash + | [] -> "" + + +/// How many cut seeds this server keeps. At roughly twelve megabytes each, eight is about a +/// hundred megabytes, and more than anybody browsing pins needs at once. +val seedCacheKeep = 8 + + +/// Drop the oldest cuts past `seedCacheKeep`. +/// +/// Not tidiness. `/seed/.db` is ANONYMOUS -- it has to be, because fetching a pinned +/// package set is the one thing a contributor with no access must be able to do -- and it keeps a +/// file per commit asked for. The commits are discoverable through the equally public +/// `/sync/pull`, so without a bound a stranger can walk a store's history and make the server keep +/// a twelve-megabyte file for every commit in it. +/// +/// Oldest by modified time, which is "least recently asked for" because a cut is rewritten only +/// when it is missing. Failures are ignored: this is housekeeping, and refusing to serve a seed +/// because a stale file would not delete would be the wrong trade. +let pruneSeedCache () : Unit = + let dir = Stdlib.Cli.Path.parent (seedCachePath "x") + + match Stdlib.Cli.Dir.list dir with + | Error _ -> () + | Ok entries -> + let cuts = + entries + |> Stdlib.List.filter (fun e -> Stdlib.String.endsWith e ".db") + |> Stdlib.List.filterMap (fun e -> + let full = Stdlib.Cli.Path.join [ dir, e ] + + match Stdlib.Cli.File.getModifiedTime full with + | Ok t -> Stdlib.Option.Option.Some((t, full)) + | Error _ -> Stdlib.Option.Option.None) + + let total = Stdlib.List.length cuts + + if total <= seedCacheKeep then + () + else + // Oldest first, then delete exactly the excess. Ascending rather than descending because + // every count here is an `Int` and negating one to reverse the sort is how this got an + // Int/Int64 mismatch the first time, which raised inside the handler and turned every seed + // request into a 500. + cuts + |> Stdlib.List.sortBy (fun c -> let (t, _) = c in t) + |> Stdlib.List.take (total - seedCacheKeep) + |> Stdlib.List.iter (fun c -> + let (_, path) = c + let _ = Stdlib.Cli.File.delete path + ()) + + +/// Cut a seed at , or hand back the one already cut there. +let cutSeedAt (commit: String) : Stdlib.Result.Result = + let path = seedCachePath commit + + if Stdlib.Cli.File.exists path then + Stdlib.Result.Result.Ok path + else + match Stdlib.Cli.Dir.createRecursive (Stdlib.Cli.Path.parent path) with + | Error e -> + Stdlib.Result.Result.Error + $"could not make the seed cache directory: {Stdlib.Cli.Posix.Error.toString e}" + | Ok _ -> + match Stdlib.LocalStore.seedTo path (Stdlib.Option.Option.Some commit) with + | Error e -> Stdlib.Result.Result.Error e + | Ok _ -> + let _ = pruneSeedCache () + Stdlib.Result.Result.Ok path + + +/// What a seed cut from this relay says about itself, so a pin can be checked without downloading +/// tens of megabytes to read four fields. +type SeedMeta = + { /// The commit the seed this describes would be cut at: the current head, or the resolved + /// `?commit=` when one was asked about. "" only when nothing is committed here at all. + head: String + /// The op-blob layout a seed from here is written in. A fetcher that reads a different one must + /// refuse rather than fold it. + format: String + /// The build that would cut it, for tracing a bad seed back to a binary. This server's own + /// build, not whatever its store says: the store's `kernel` records the build that cut the seed + /// it GREW from, which is a different fact and older. + kernel: String + /// What a seed cut at `head` would carry. A client comparing these with its own knows whether + /// a fetch would tell it anything. + ops: Int64 + commits: Int64 } + + +/// `/seed/meta`: the fields a pin needs, without the seed. +/// +/// `?commit=` asks about THAT commit instead of the head, which is how a pin is checked: it +/// answers 404 if the commit does not resolve here, and otherwise the shape the cut would have. +/// Two counts rather than twelve megabytes. +let seedMetaHandlerFn (req: HttpRequest) : HttpResponse = + let asked = + Stdlib.Http.Request.queryParam req "commit" |> Stdlib.Option.withDefault "" + + let resolved = + if asked == "" then + Stdlib.Result.Result.Ok(Stdlib.Option.Option.None) + else + match SCM.PackageOps.resolveCommit asked with + | Some hash -> Stdlib.Result.Result.Ok(Stdlib.Option.Option.Some hash) + | None -> Stdlib.Result.Result.Error $"no single commit here starts with \"{asked}\"" + + match resolved with + | Error why -> Stdlib.Http.responseWithText why 404 + | Ok at -> + let shape = + match at with + | Some hash -> SCM.PackageOps.seedShapeAt hash + | None -> + Stdlib.Option.Option.Some( + (SCM.PackageOps.committedCountOnMain (), + SCM.PackageOps.commitCountOnBranch SCM.Branch.mainBranchId) + ) + + match shape with + | None -> Stdlib.Http.responseWithText $"no commit here named \"{asked}\"" 404 + | Some counts -> + let (ops, commits) = counts + + let meta = + SeedMeta + { head = + match at with + | Some hash -> hash + | None -> headCommit () + format = SCM.StoreMeta.get "format" + kernel = Darklang.Cli.Installation.Helpers.buildHash () + ops = ops + commits = commits } + + Stdlib.Http.responseWithJson (Stdlib.Json.serialize meta) 200 + + +/// `/seed/latest.db` and `/seed/.db`: the store as a file, cut at a commit. +/// +/// `latest.db` resolves to the current head and is served as the cut AT that head, so it is the same +/// bytes as asking for that commit by name. A short commit prefix resolves like git's; an ambiguous +/// one is refused rather than guessed at, because guessing here pins the wrong package set. +let seedHandlerFn (req: HttpRequest) : HttpResponse = + let requested = + Stdlib.HttpServer.getPathParam req seedRoute "file" + |> Stdlib.Option.withDefault "" + + // `.db` is part of the route rather than a query param so a browser, `curl -O` and a CI cache all + // end up with a file named like what it is. + if Stdlib.Bool.not (Stdlib.String.endsWith requested ".db") then + Stdlib.Http.badRequest + "a seed is asked for as `/seed/latest.db` or `/seed/.db`" + else + + let asked = Stdlib.String.dropLast requested 3 + + let resolved = + if asked == "" then + // `/seed/.db` asks for nothing. Refused rather than read as an empty prefix, which matches + // every commit and so resolves to one whenever a store happens to hold exactly one. + Stdlib.Result.Result.Error + "a seed is asked for as `/seed/latest.db` or `/seed/.db`" + else if asked == "latest" then + let head = headCommit () + + if head == "" then + Stdlib.Result.Result.Error + "nothing has been committed here yet, so there is no seed to cut" + else + Stdlib.Result.Result.Ok head + else + // `resolveCommit`, not `SCM.Commits.byHashPrefix`: that one takes the newest on a tie, which is + // the right rule for a listing and the wrong one here. An ambiguous prefix would pin a package + // set nobody chose, and silently. + match SCM.PackageOps.resolveCommit asked with + | Some hash -> Stdlib.Result.Result.Ok hash + | None -> + Stdlib.Result.Result.Error + $"no single commit here starts with \"{asked}\"" + + match resolved with + | Error why -> Stdlib.Http.responseWithText why 404 + | Ok commit -> + match cutSeedAt commit with + | Error why -> Stdlib.Http.serverError why + | Ok path -> + match Stdlib.Cli.File.readBytes path with + | Error e -> + Stdlib.Http.serverError + $"cut the seed but could not read it back: {Stdlib.Cli.Posix.Error.toString e}" + | Ok bytes -> + // The commit is in the bytes too (`store_meta.cut_at`), so this header is a convenience for + // a fetcher that wants it without opening SQLite -- not the only record of it. + Stdlib.Http.responseWithHeaders + bytes + [ ("Content-Type", "application/vnd.sqlite3"), + ("X-Dark-Seed-Commit", commit), + ("Content-Disposition", "attachment; filename=\"seed.db\"") ] + 200 diff --git a/packages/darklang/sync/relay/server.dark b/packages/darklang/sync/relay/server.dark index bf235fc06a..15b02677a3 100644 --- a/packages/darklang/sync/relay/server.dark +++ b/packages/darklang/sync/relay/server.dark @@ -2,11 +2,16 @@ /// /// Run with: `dark serve Darklang.Matter.router --port 8080` /// -/// A relay is a DUMB store-and-forward for the op log: it holds ops and hands them back -/// out; it never folds or resolves them, so it never runs anyone's code. A client that -/// pulls for review sees every op before it lands, so the worst a relay can do is -/// withhold one. It reuses the SCM.Wire codec, so the relay's own store IS an op log like -/// any instance's. +/// The server holds the op log and hands it back out: `/sync/pull` pages it, `/seed/*.db` cuts +/// it as a file. It reuses the SCM.Wire codec, so the server's own store IS an op log like any +/// instance's. +/// +/// It also FOLDS what is pushed to it, into main, like any other instance. That is what makes +/// `/m`, `/p` and the seed routes show hosted packages rather than only what this binary shipped +/// with. It is also a deferred decision, not a settled one: names bind last-writer-wins over the +/// whole store, `Darklang.Matter.router` among them, so anyone who can push here can change what +/// this server resolves. The write secret is the only thing standing in front of that. See +/// `LibDB.Inserts` for what the old `effective = 0` defended and why it went. /// /// `handlers`, near the bottom, is the route table. module Darklang.Matter @@ -303,9 +308,9 @@ let ownersHandlerFn (_req: HttpRequest) : HttpResponse = page "Publishers" - ("

Publishers

Who owns the packages this relay serves. Code pushed here by " - ++ "other machines is held and handed back on request, but it is not folded into what the relay runs, " - ++ "so it does not appear here. Why.

" + ("

Publishers

Who owns the packages this server serves. Code pushed here " + ++ "is folded into main like anywhere else, so it shows up here as soon as it arrives. " + ++ "What that means.

" ++ $"
{rows}
") @@ -334,13 +339,12 @@ let statsHandlerFn (_req: HttpRequest) : HttpResponse = ++ $"
{opsStat}{fnStat}{typeStat}{valueStat}{edgeStat}
" ++ "

Two numbers, two different things

" ++ "

The op count is everything anyone has ever pushed here. The function, type and " - ++ "value counts are smaller and mean something else: they are what this relay itself RUNS and can " - ++ "therefore show you source for.

" - ++ "

A relay stores what you push without folding it into its own package set. That " - ++ "is deliberate, and it is the whole reason a relay is safe to point a machine at: pushing code here " - ++ "cannot change what this server runs, so nobody can push a new version of the relay to the relay. " - ++ "The cost is that hosted code is held, served back to whoever pulls it, and not browsable on these " - ++ "pages. What you can browse here is the package set this build ships with.

" + ++ "value counts are smaller and mean something else: they are what is LIVE, the latest version of " + ++ "each name rather than every version of it.

" + ++ "

Pushed code is folded into main here, the same as on your own machine, which " + ++ "is why you can browse it on these pages and fetch it as a seed. It also means a push can rebind " + ++ "a name this server itself resolves, including this page. Who may bind what is not answered yet; " + ++ "for now the write secret is the whole of the answer, so treat push access as trust.

" ++ "

What an op is

" ++ "

One recorded change: publishing a function, moving a name, deprecating " ++ "something. Every version of everything is still here, which is why the op count runs so far ahead " @@ -362,7 +366,10 @@ val handlers = post "/branch/push" Darklang.Matter.Relay.branchPushHandlerFn get "/branch/pull" Darklang.Matter.Relay.branchPullHandlerFn get "/branch/list" Darklang.Matter.Relay.branchListHandlerFn - get "/branch/owners" Darklang.Matter.Relay.branchOwnersHandlerFn ] + get "/branch/owners" Darklang.Matter.Relay.branchOwnersHandlerFn + // Before the `/seed/:file` route below it, which would otherwise swallow `meta` as a filename. + get "/seed/meta" Darklang.Matter.Relay.seedMetaHandlerFn + get Darklang.Matter.Relay.seedRoute Darklang.Matter.Relay.seedHandlerFn ] /// One server, two audiences. /// diff --git a/packages/darklang/sync/wire.dark b/packages/darklang/sync/wire.dark index d909b80cfe..774e684692 100644 --- a/packages/darklang/sync/wire.dark +++ b/packages/darklang/sync/wire.dark @@ -201,9 +201,25 @@ let localMaxRowid () : Int64 = |> Stdlib.Option.withDefault 0L -/// RELAY side of a push: store the pushed ops AND record who pushed them, so the owner can pull their -/// stuff back from any machine. Bulk (ops + ownership in one transaction) and store-only -- NO fold, since -/// a relay serves op blobs, not projections. Returns the count of newly-stored ops. +/// SERVER side of a push: store the pushed ops, record who pushed them so the owner can pull their +/// stuff back from any machine, and FOLD them into main. Returns the count of newly-stored ops. +/// +/// It stored without folding until 2026-09-12, which is why the package browser said "Nothing here" +/// for code every client had: the server's own projection never saw what it hosted. Folding is what +/// lets a seed be cut from the hosted set, and what makes pushing new code to a server able to change +/// what it runs. +/// The reserved names these ops would bind into this server's main, and that it will not accept. +/// Empty means the push is fine. +/// +/// Asked before storing, so a refusal can answer with a status that says "not allowed" rather than +/// "the server broke" -- a client that sees a 500 has every reason to retry forever. The same rule +/// is enforced inside the store path as the backstop. +let reservedBindingsIn (ops: List) : List = + Builtin.scmReservedBindings ( + ops |> Stdlib.List.map (fun op -> (op.id, op.blobHex, op.ts)) + ) + + let storeWithOwner (owner: String) (ops: List) @@ -213,8 +229,7 @@ let storeWithOwner match Builtin.scmStoreOps owner records with | Ok n -> - // The author's commits, kept beside the hosted ops so a pull hands them on. Stored, never folded, - // like the ops themselves: a hosted op is inert here whatever commit it names. + // The author's commits, kept beside the hosted ops so a pull hands them on. PackageOps.adoptCommits commits (ops |> Stdlib.List.map (fun op -> (op.id, op.commit))) Stdlib.Result.Result.Ok n | Error e -> Stdlib.Result.Result.Error e diff --git a/scripts/build/_buildplan.py b/scripts/build/_buildplan.py index be27d8f759..c832f67e29 100644 --- a/scripts/build/_buildplan.py +++ b/scripts/build/_buildplan.py @@ -38,6 +38,7 @@ class Should: "backend_quick_build", "run_migrations", "reload_all_packages", + "check_refs", "backend_test", "circleci_validate", "shellcheck", @@ -52,6 +53,7 @@ def __init__(self): self.backend_full_build = False self.backend_test = False self.reload_all_packages = False + self.check_refs = False self.circleci_validate = False self.run_migrations = False self.shellcheck = [] @@ -183,6 +185,12 @@ def expand(should, run_tests=False): if s.run_migrations and not packages_from_seed(): s.reload_all_packages = True + # Whenever either side of the kernel/package-set interface could have moved. An `.fs` + # change can add or move a ref; a package change can take away what one resolves to. + # Cheap enough to run on both rather than reason about which refs a change touched. + if s.backend_quick_build or s.backend_full_build or s.reload_all_packages: + s.check_refs = True + # backend_test is set by execute() on any build, but run_test() no-ops without # --test, so a plan that lists it would be lying. if not run_tests: diff --git a/scripts/build/check-seed-carries-refs b/scripts/build/check-seed-carries-refs index 7b5dabb60d..c875bd92bc 100755 --- a/scripts/build/check-seed-carries-refs +++ b/scripts/build/check-seed-carries-refs @@ -33,6 +33,8 @@ def main() -> int: if "|" in line} ) if not refs: + # Only a fresh clone before its first reload, where MSBuild has touched an empty one. + # The file is tracked, so in every other case there is something to check. print("check-seed-carries-refs: skipped (package-ref-hashes.txt is empty)") return 0 diff --git a/scripts/build/compile b/scripts/build/compile index 02c89be790..2ae19750ff 100755 --- a/scripts/build/compile +++ b/scripts/build/compile @@ -211,6 +211,17 @@ def run_migrations(): return run_backend(start, f"scripts/build/run-migrations {configuration}") +def check_refs(): + """Does the kernel this build produced agree with the package set in front of it? + + The ref closures are lazy, so an unresolvable ref is otherwise found whenever some code + path happens to reach it, which can be a different day and an unrelated command. The + case this exists for is checking out somebody's git branch without their package work. + """ + start = time.time() + return run_backend(start, "scripts/run-local-exec refs check") + + def reload_all_packages(): start = time.time() if optimize: @@ -299,6 +310,14 @@ def execute(should, outcome): if should.reload_all_packages: step("reload_all_packages", reload_all_packages) + # After anything that could have moved either side of the kernel/package-set interface. + # Decided here rather than in the plan because execute() sets some of these itself, so the + # plan's answer is not the final one. Cheap: one pass over ~200 refs against caches the + # store already holds. + if should.check_refs or should.reload_all_packages or should.backend_quick_build \ + or should.backend_full_build: + step("check_refs", check_refs) + # Gated here rather than inside backend_test(), so a build that skipped the tests # doesn't list running them among the things it did. if should.backend_test and run_tests: diff --git a/scripts/build/prepare-package-set b/scripts/build/prepare-package-set new file mode 100755 index 0000000000..612d33db15 --- /dev/null +++ b/scripts/build/prepare-package-set @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# Get this build's package set into the store, from whichever source `package-set.txt` names. +# +# commit unset reload `packages/` from disk, which is what has always happened +# commit fetch the seed at that commit from the package server, and generate the +# kernel's ref hashes from it +# +# One script rather than a branch in three CI jobs, because the two halves have to agree about what +# "the package set" is or a build embeds one and tests another. +# +# The pinned path needs `DARK_SEED_URL` (or `--url`). It deliberately does NOT fall back to +# reloading if the server is unreachable: a build that silently used the tree instead of the pin is +# the exact failure the pin exists to prevent. + +set -euo pipefail + +[[ -f package-set.txt ]] || { echo "prepare-package-set: run from the repo root" >&2; exit 2; } + +URL_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --url) URL_ARGS=(--url "${2:?--url needs a url}"); shift 2 ;; + -h|--help) sed -n '3,15p' "$0" | sed 's/^# \?//'; exit 0 ;; + *) echo "prepare-package-set: unknown argument $1" >&2; exit 2 ;; + esac +done + +# The branch bundle, if this git branch carries one. Imported whichever way the base package set +# arrives, because checking out the git branch IS asking for that branch's package code, and the +# F# in the tree may name items only the bundle has. Idempotent: ops are content-addressed, so +# re-importing what is already here changes nothing. +import_bundle() { + if [[ -f "${DARK_PACKAGE_BUNDLE:-package-branch.json}" ]]; then + scripts/packages/bundle import + fi +} + +PIN=$(awk '$1 == "commit" { print $2 }' package-set.txt) + +# Whether the kernel and the package set agree, asked wherever the package set is established so +# it cannot be forgotten. `scripts/dev/build` asks it too, as a build action; CI never runs that +# script, so without this the one invariant the whole scheme rests on went unchecked in exactly +# the place it matters most. +check_agreement() { + scripts/run-local-exec refs check +} + +if [[ -z "$PIN" || "$PIN" == "unset" ]]; then + echo "No pin in package-set.txt; building the package set from packages/." + scripts/build/reload-packages + import_bundle + check_agreement + exit 0 +fi + +echo "Pinned to ${PIN}; fetching the seed rather than reloading packages/." +scripts/fetch-seed "${URL_ARGS[@]}" --force + +# The kernel resolves its entry points by HASH, and that file is not in the seed -- it is a +# projection of it. Without this step every kernel lookup resolves to "" and the binary fails on +# its first command, somewhere that says nothing about why. +scripts/run-local-exec refs generate + +import_bundle +check_agreement diff --git a/scripts/devcontainer/_post-start b/scripts/devcontainer/_post-start index 3f5c2f91ed..f7fb8cd63f 100755 --- a/scripts/devcontainer/_post-start +++ b/scripts/devcontainer/_post-start @@ -30,6 +30,17 @@ scripts/devcontainer/_create-app-directories scripts/devcontainer/_create-cache-directories scripts/devcontainer/_setup-hosts +# In seed mode the store IS the source, so a fresh container fetches one rather than +# producing it. Disk mode builds it from `packages/` during the build below, as always. +if [[ "${DARK_CONFIG_PACKAGES_SOURCE:-disk}" == "seed" && ! -f rundir/data.db ]]; then + log "Seed mode and no store: fetching one." + if scripts/fetch-seed >> "$LOG_FILE" 2>&1; then + log "Seed installed." + else + log "Seed fetch FAILED. Install one by hand: scripts/fetch-seed --from " + fi +fi + log "Building. Follow along: tail -F rundir/logs/build.log" if scripts/dev/build >> "$LOG_FILE" 2>&1; then log "Build succeeded. scripts/dev/status for details." diff --git a/scripts/fetch-seed b/scripts/fetch-seed index 2aa69eaa5c..0546e655e5 100755 --- a/scripts/fetch-seed +++ b/scripts/fetch-seed @@ -1,14 +1,164 @@ #!/usr/bin/env bash + +# Get a package seed, from a file or from a server, and install it as this rundir's store. +# +# scripts/fetch-seed --from a seed already on disk +# scripts/fetch-seed --url that server's main at the PIN, or its head +# scripts/fetch-seed --url --commit that server's main as of +# scripts/fetch-seed DARK_SEED_URL, or the built-in default +# +# With no --commit, the commit comes from `package-set.txt`; `commit unset` there means no pin, +# and the fetch takes the server's head. A pinned fetch also CHECKS the seed's format against the +# pin, because a layout this build has never seen cannot be read by trying harder. +# +# A seed is ops, not projections, so the first open folds it (~seconds) and every store built +# from the same seed agrees on every id. +# +# It REFUSES to replace a store that already exists unless you pass --force. The previous +# version copied over `data.db` unconditionally, which silently discarded anything authored +# locally or pulled from a peer; that is a bad thing for a command called "fetch" to do. + set -euo pipefail -SEED_URL="https://pub-d5f652ca1b90456f85840a19eafcb700.r2.dev/darklang-seeds/seeds/seed.db" RUNDIR="${DARK_CONFIG_RUNDIR:-rundir}" +DEST_SEED="${RUNDIR}/seed.db" +DEST_DB="${RUNDIR}/${DARK_CONFIG_DB_NAME:-data.db}" + +# The bucket the old hardcoded URL pointed at. Kept as the fallback so the no-argument form +# still does something, but a real deployment passes --url or sets DARK_SEED_URL. +DEFAULT_URL="${DARK_SEED_URL:-https://pub-d5f652ca1b90456f85840a19eafcb700.r2.dev/darklang-seeds/seeds/seed.db}" + +FROM="" +URL="" +COMMIT="" +FORCE=false + +# The pin, if this tree has one. Read before the arguments so an explicit --commit overrides it. +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" +PINNED_COMMIT="" +PINNED_FORMAT="" +if [[ -f "$PINFILE" ]]; then + PINNED_COMMIT=$(awk '$1 == "commit" { print $2 }' "$PINFILE") + PINNED_FORMAT=$(awk '$1 == "format" { print $2 }' "$PINFILE") + if [[ "$PINNED_COMMIT" == "unset" ]]; then PINNED_COMMIT=""; fi +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + --from) FROM="${2:?--from needs a path}"; shift 2 ;; + --url) URL="${2:?--url needs a url}"; shift 2 ;; + --commit) COMMIT="${2:?--commit needs a commit}"; shift 2 ;; + --force) FORCE=true; shift ;; + -h|--help) sed -n '3,14p' "$0" | sed 's/^# \?//'; exit 0 ;; + *) echo "fetch-seed: unknown argument $1" >&2; exit 2 ;; + esac +done + +if [[ -n "$FROM" && -n "$URL" ]]; then + echo "fetch-seed: --from and --url are alternatives, not both" >&2 + exit 2 +fi + +mkdir -p "$RUNDIR" + +if [[ -n "$FROM" ]]; then + if [[ ! -f "$FROM" ]]; then + echo "fetch-seed: no seed at $FROM" >&2 + exit 1 + fi + echo "Copying seed from $FROM" + cp "$FROM" "$DEST_SEED" +else + # `/seed/.db` is the route a server serves a pinned cut from; without a commit, its + # current main. A bare url with no path is treated as already naming the seed, which is what + # the R2 fallback is. + # An explicit --commit beats the pin; the pin beats the server's head. So a build in a pinned + # tree fetches the pinned set without anyone having to remember to ask for it. + if [[ -z "$COMMIT" && -n "$PINNED_COMMIT" ]]; then + COMMIT="$PINNED_COMMIT" + echo "Using the pin from ${PINFILE}: ${COMMIT}" + fi + + if [[ -n "$COMMIT" ]]; then + SEED_URL="${URL:-$DEFAULT_URL}/seed/${COMMIT}.db" + elif [[ -n "$URL" ]]; then + SEED_URL="${URL}/seed/latest.db" + else + SEED_URL="$DEFAULT_URL" + fi + + # A seed cut at a COMMIT never changes, so it is worth keeping once per machine rather than once + # per clone. Without this, every clone and every re-clone needs the network -- which is a real + # robustness regression against today, where `git clone && scripts/dev/start` works from the repo + # alone. With it, a machine needs the network once per pin, and a second clone, a re-clone, or + # working on a plane is a file copy. + # + # Only for a pinned fetch. `latest` is not immutable and must not be cached under a name that + # claims it is. + CACHE_DIR="${DARK_SEED_CACHE:-$HOME/.darklang/seeds}" + CACHED="" + if [[ -n "$COMMIT" ]]; then + CACHED="${CACHE_DIR}/${COMMIT}.db" + fi + + if [[ -n "$CACHED" && -r "$CACHED" ]] && cp "$CACHED" "$DEST_SEED" 2>/dev/null; then + echo "Using the cached seed for ${COMMIT} (${CACHED})" + else + echo "Fetching seed from $SEED_URL" + if ! curl -fL -o "$DEST_SEED" "$SEED_URL"; then + { + echo + echo "Could not fetch a seed from ${SEED_URL}." + if [[ -n "$COMMIT" ]]; then + echo "Nothing on this machine has that commit cached, either." + fi + echo "If you have a seed file already, install it directly and skip the network:" + echo " scripts/fetch-seed --from " + } >&2 + exit 1 + fi + + # Best-effort, always. The seed is already fetched and installed by this point, so a cache + # that cannot be written costs the NEXT fetch a download and nothing else. Failing here + # instead would turn an unwritable home directory into a failed build, which is what happened + # the first time this shipped. + if [[ -n "$CACHED" ]]; then + if mkdir -p "$CACHE_DIR" 2>/dev/null && cp "$DEST_SEED" "$CACHED" 2>/dev/null; then + echo "Cached for this machine at ${CACHED}" + else + echo "(could not cache to ${CACHE_DIR}; the seed is installed regardless)" + fi + fi + fi +fi + +SEED_SIZE=$(du -h "$DEST_SEED" | cut -f1) +echo "Seed at ${DEST_SEED} (${SEED_SIZE})" -echo "Fetching latest seed from R2..." -curl -L -o "${RUNDIR}/seed.db" "$SEED_URL" +# Refuse a seed in a layout the pin did not expect, BEFORE installing it. A store is the only copy +# of the ops in it, so the failure has to happen while the old one is still there. +if [[ -n "$PINNED_FORMAT" ]] && command -v sqlite3 > /dev/null 2>&1; then + GOT_FORMAT=$(sqlite3 "$DEST_SEED" "SELECT value FROM store_meta WHERE key = 'format';" 2>/dev/null || echo "") + if [[ -n "$GOT_FORMAT" && "$GOT_FORMAT" != "$PINNED_FORMAT" ]]; then + { + echo + echo "That seed is in format ${GOT_FORMAT}; ${PINFILE} pins format ${PINNED_FORMAT}." + echo "Not installing it. Either the server moved formats or this tree's pin is stale;" + echo "re-pin with scripts/packages/pin, on a build that reads the new format." + } >&2 + exit 1 + fi +fi -# Copy as data.db so CLI/tests can grow from it -cp "${RUNDIR}/seed.db" "${RUNDIR}/data.db" +if [[ -f "$DEST_DB" && "$FORCE" != "true" ]]; then + { + echo + echo "Not installing it: ${DEST_DB} already exists." + echo "Replacing it would discard anything authored locally or pulled from a peer." + echo "Pass --force if that is what you want." + } >&2 + exit 0 +fi -SIZE=$(du -h "${RUNDIR}/data.db" | cut -f1) -echo "Seed downloaded and installed as data.db ($SIZE)" +cp "$DEST_SEED" "$DEST_DB" +echo "Installed as ${DEST_DB}. The first open folds it." diff --git a/scripts/packages/bundle b/scripts/packages/bundle new file mode 100755 index 0000000000..e854403603 --- /dev/null +++ b/scripts/packages/bundle @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# The package work on your dark branch, as a file that travels with your git branch. +# +# scripts/packages/bundle export write your dark branch to package-branch.json +# scripts/packages/bundle import recreate that branch from the file +# scripts/packages/bundle show what is in the file +# scripts/packages/bundle show --source ...and what it actually changes +# +# Why a file in the repo. An F# change can reference package code, and then the two have to +# travel, merge and be abandoned together. Git already does that for the F#; this lets it do the +# same for the package half, so one PR carries both and reviewing or dropping it is one act. +# +# EXPORT is explicit, like `git add`: producing it writes into your tree and that should be +# something you asked for. Forgetting is caught rather than silent -- `refs check` fails the build +# for anyone whose F# names package code they do not have. +# +# IMPORT is automatic (`scripts/build/prepare-package-set` runs it), because checking out the git +# branch IS asking for that branch's package code. Importing is idempotent: ops are +# content-addressed, so re-importing what you already have changes nothing. + +set -euo pipefail + +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" +[[ -f "$PINFILE" ]] || { echo "bundle: no $PINFILE (run from the repo root?)" >&2; exit 2; } + +BUNDLE="${DARK_PACKAGE_BUNDLE:-package-branch.json}" +CLI="${CLI:-./scripts/run-cli}" + +usage() { sed -n '3,19p' "$0" | sed 's/^# \?//'; } + +case "${1:-}" in + export) + branch=$("$CLI" branch 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | tail -1) + if [[ "$branch" == *"on main"* || -z "$branch" ]]; then + # Nothing to carry. Remove a stale bundle rather than leave one describing work that is + # now on main: a bundle outliving its branch is how somebody imports a branch that was + # merged weeks ago. + if [[ -f "$BUNDLE" ]]; then + rm -f "$BUNDLE" + echo "On dark main; removed $BUNDLE." + else + echo "On dark main; nothing to bundle." + fi + exit 0 + fi + + name=$(sed -E 's/^on branch "([^"]+)".*/\1/' <<<"$branch") + "$CLI" branch export "$name" "$BUNDLE" > /dev/null + echo "Wrote $BUNDLE from dark branch \"$name\" ($(wc -c < "$BUNDLE") bytes)." + echo "Commit it alongside the F# that needs it." + + # A bundle carries the BRANCH's ops. If your main has moved past the pin -- because you + # pulled -- your branch may rest on commits nobody else has, and the bundle will not be + # enough for them: they hold the pinned set plus this file and nothing else. CI says so + # too, as a failed `refs check`, but by then it is somebody else's red build. + pin=$(awk '$1 == "commit" { print $2 }' "$PINFILE") + if [[ -n "$pin" && "$pin" != "unset" ]]; then + # `--branch main` explicitly: `dark log` shows a BRANCH's ops when you are standing on + # one, and we are, so the bare form answers with an op kind rather than a commit hash. + head=$("$CLI" --branch main log 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' \ + | awk 'NR>3 { print $1; exit }') + if [[ -n "$head" && "${pin:0:${#head}}" != "$head" ]]; then + echo + echo "Note: your main is at ${head}, but ${PINFILE} pins ${pin:0:8}." + echo "If this branch rests on commits the pin does not have, this bundle will not be" + echo "enough for anyone else. Bump the pin, or wait for it to catch up." + fi + fi + ;; + + import) + if [[ ! -f "$BUNDLE" ]]; then + echo "No $BUNDLE; nothing to import." + exit 0 + fi + "$CLI" branch import "$BUNDLE" + ;; + + show) + [[ -f "$BUNDLE" ]] || { echo "No $BUNDLE." >&2; exit 1; } + python3 -c " +import json +b=json.load(open('$BUNDLE')) +print('branch:', b.get('name'), '(' + str(b.get('branchId','?'))[:8] + ')') +print('ops: ', len(b.get('ops', []))) +print('commits:', len(b.get('commits', []))) +" + if [[ "${2:-}" == "--source" ]]; then + # Rendered by importing into a DISPOSABLE store, never yours. Reviewing a stranger's + # branch should not mean taking their ops into the store you work in, and a PR reviewer + # is exactly the person with the least reason to trust the bundle in front of them. + scratch=$(mktemp -d) + # shellcheck disable=SC2064 # expand now: $scratch must not be re-evaluated at trap time + trap "rm -rf '$scratch'" EXIT + scripts/testing/_copy-store "$scratch/data.db" > /dev/null + name=$(python3 -c " +import json +print(json.load(open('$BUNDLE')).get('name','')) +") + echo + env DARK_CONFIG_RUNDIR="$scratch" HOME="$scratch" \ + ./backend/Build/out/Cli/Debug/net10.0/Cli branch import "$BUNDLE" > /dev/null 2>&1 + env DARK_CONFIG_RUNDIR="$scratch" HOME="$scratch" \ + ./backend/Build/out/Cli/Debug/net10.0/Cli branch diff "$name" --source 2>&1 \ + | sed 's/\x1b\[[0-9;]*m//g' + fi + ;; + + -h|--help|"") usage ;; + *) echo "bundle: unknown argument $1" >&2; usage >&2; exit 2 ;; +esac diff --git a/scripts/packages/pin b/scripts/packages/pin new file mode 100755 index 0000000000..c13660cffe --- /dev/null +++ b/scripts/packages/pin @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# Pin the package set to a commit on a package server, and record it in `package-set.txt`. +# +# scripts/packages/pin head pin to what that server has right now +# scripts/packages/pin pin to a named commit (a short prefix is fine) +# scripts/packages/pin --show what is pinned now +# scripts/packages/pin --unset drop the pin, back to building from `packages/` +# +# The server comes from `--url` or `DARK_SEED_URL`; see `package-set.txt` for why it is not in +# the repo. +# +# Every pin is CHECKED before it is written, through `/seed/meta?commit=`: the server resolves the +# commit and answers what a cut at it would carry. Two counts rather than twelve megabytes, so +# checking a pin is cheap enough to do every time. A commit that does not resolve is refused here +# rather than at the next CI run. + +set -euo pipefail + +# `DARK_PACKAGE_SET` points this at another copy, which is how the gates exercise a pin without +# writing to the tree's own. +PINFILE="${DARK_PACKAGE_SET:-package-set.txt}" + +[[ -f "$PINFILE" ]] || { echo "pin: no $PINFILE (run from the repo root?)" >&2; exit 2; } +URL="${DARK_SEED_URL:-}" +WANT="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --url) URL="${2:?--url needs a url}"; shift 2 ;; + --show) sed -n 's/^\(commit\|format\) */\1 /p' "$PINFILE"; exit 0 ;; + --unset) WANT="unset"; shift ;; + -h|--help) sed -n '3,17p' "$0" | sed 's/^# \?//'; exit 0 ;; + -*) echo "pin: unknown argument $1" >&2; exit 2 ;; + *) WANT="$1"; shift ;; + esac +done + +# Rewrite one `key value` line in place, leaving the comments that explain it alone. +set_field() { + local key="$1" value="$2" + local tmp + tmp=$(mktemp) + awk -v k="$key" -v v="$value" ' + $1 == k { printf "%s %s\n", k, v; next } + { print } + ' "$PINFILE" > "$tmp" + mv "$tmp" "$PINFILE" +} + +if [[ "$WANT" == "unset" ]]; then + set_field commit unset + echo "Pin dropped. The package set is built from packages/ again." + exit 0 +fi + +[[ -n "$WANT" ]] || { echo "pin: say which commit, or \`head\`, or \`--show\`" >&2; exit 2; } + +if [[ -z "$URL" ]]; then + echo "pin: no package server. Pass --url, or set DARK_SEED_URL." >&2 + exit 2 +fi + +query="$URL/seed/meta" +[[ "$WANT" == "head" ]] || query="$URL/seed/meta?commit=$WANT" + +body=$(mktemp) +code=$(curl -s -o "$body" -w '%{http_code}' --max-time 30 "$query" || echo 000) + +if [[ "$code" != "200" ]]; then + echo "pin: $URL would not pin $WANT (HTTP $code):" >&2 + sed 's/^/ /' "$body" >&2 + exit 1 +fi + +field() { sed -n "s/.*\"$1\":\"\\?\\([^,\"}]*\\).*/\\1/p" "$body"; } +COMMIT=$(field head) +FORMAT=$(field format) +OPS=$(field ops) + +[[ -n "$COMMIT" ]] || { echo "pin: $URL named no commit: $(cat "$body")" >&2; exit 1; } + +# The format is pinned alongside the commit, not just reported: a seed in a layout this kernel +# cannot read is a fetch that must fail before it folds anything, and the only way to know that +# before fetching is to have written down what was expected. +set_field commit "$COMMIT" +set_field format "$FORMAT" + +# The ref hashes move WITH the pin and only with it. That is what keeps them stable between +# bumps, so ordinary package work never touches the file and two branches that both touch +# packages never conflict in it. The diff this produces is the point: one reviewable list of +# every kernel identity that moved. +# +# Best-effort. The pin is already written, and a store that cannot regenerate (no source tree, +# say) should not lose it. +if scripts/run-local-exec refs generate > /dev/null 2>&1; then + echo "Regenerated package-ref-hashes.txt for the new pin." +else + echo "(could not regenerate package-ref-hashes.txt here; run \`refs generate\` where the store is)" +fi + +echo "Pinned to $COMMIT (format $FORMAT, $OPS ops)." +echo "Commit package-set.txt: the pin is the record of which package set this kernel was tested against." diff --git a/scripts/run-backend-tests b/scripts/run-backend-tests index 8d620282e9..76485df032 100755 --- a/scripts/run-backend-tests +++ b/scripts/run-backend-tests @@ -116,17 +116,36 @@ cd backend && \ cd .. if [[ "${DARK_CONFIG_PACKAGES_SOURCE:-disk}" == "seed" ]]; then - # Seed mode: copy seed as test DB, Tests exe auto-grows on startup + # Seed mode: the test store is a COPY of a built store; the Tests exe folds anything + # unapplied on startup. + # + # Locally that copy is YOUR store, so a package you just authored is what the suite runs + # against. Testing the seed instead would test whatever the last fetch happened to hold, + # which is the one store guaranteed not to contain your change. + # + # CI has no store of its own to prefer, and wants the pinned one anyway: reproducible is + # the whole point of the pin. SEED_PATH="${DARK_CONFIG_RUNDIR}/seed.db" - if [[ -f "$SEED_PATH" ]]; then - echo "Copying seed.db as test database" - cp "$SEED_PATH" "$DB_PATH" - else - echo "ERROR: No seed.db found at $SEED_PATH" - echo "Run: ./scripts/run-cli export-seed /home/dark/app/rundir/seed.db" - echo "Or: ./scripts/fetch-seed" + LIVE_PATH="${DARK_CONFIG_RUNDIR}/data.db" + + SOURCE="" + if [[ ! -v CI && -f "$LIVE_PATH" ]]; then + SOURCE="$LIVE_PATH" + elif [[ -f "$SEED_PATH" ]]; then + SOURCE="$SEED_PATH" + fi + + if [[ -z "$SOURCE" ]]; then + echo "ERROR: no store to copy. Looked for $LIVE_PATH and $SEED_PATH" + echo "Run: ./scripts/fetch-seed --from " + echo "Or: ./scripts/run-cli export-seed ${DARK_CONFIG_RUNDIR}/seed.db" exit 1 fi + + # `.backup`, not `cp`: the store is WAL, so a plain copy silently omits everything still + # in `data.db-wal` -- which is exactly the work you just did. + echo "Copying $(basename "$SOURCE") as test database" + sqlite3 -cmd ".timeout 10000" "$SOURCE" ".backup '$DB_PATH'" else # Disk mode: reload packages from .dark files. # Migrations already ran inline above diff --git a/scripts/testing/_gates-store b/scripts/testing/_gates-store index 803424ec42..1a39512e4b 100644 --- a/scripts/testing/_gates-store +++ b/scripts/testing/_gates-store @@ -264,3 +264,101 @@ gate_reload_is_reproducible() { fi echo "reload-is-reproducible: two reloads of one tree agree on all $n op ids." } + +# --- store-upgrade: a format bump, carried in place --------------------------------------------- +# There is only ONE op-log format so far, so the migrator has nothing real to migrate between. +# `DARK_FORMAT_VERSION` is the synthetic bump: the binary writes the version named and reads every +# version up to it, so the layout is identical and a migration across it is exactly the +# format-only case -- a blob rewrite with no identity moving. Which is the case worth having +# mechanised and tested BEFORE the first real bump, rather than discovering during one. +# +# Gate, not an F#/.dark test: the format version is read at process start, so "the same store seen +# by a build that writes a different version" needs a different PROCESS, twice. +gate_store_upgrade() { + set -uo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + ROOT=rundir/store-upgrade-test + pass=0; fail=0 + ok() { pass=$((pass+1)); printf ' ok %s\n' "$1"; } + bad() { fail=$((fail+1)); printf ' FAIL %s\n got: %s\n' "$1" "$(head -c 300 <<<"$2" | tr '\n' ' ')"; } + want() { if grep -qE "$2" <<<"$3"; then ok "$1"; else bad "$1" "$3"; fi; } + + rm -rf "$ROOT"; mkdir -p "$ROOT/home" + scripts/testing/_copy-store "$ROOT/data.db" + + # v1: this build's own version. v2: the same build pretending to write one format on. + v1() { DARK_CONFIG_RUNDIR="$PWD/$ROOT" HOME="$PWD/$ROOT/home" "$PWD/$CLI" "$@" 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; } + v2() { DARK_FORMAT_VERSION=2 DARK_CONFIG_RUNDIR="$PWD/$ROOT" HOME="$PWD/$ROOT/home" "$PWD/$CLI" "$@" 2>&1 | sed 's/\x1b\[[0-9;]*m//g'; } + ops() { sqlite3 "$ROOT/data.db" 'SELECT count(*) FROM package_ops;' 2>/dev/null || echo -1; } + fmt() { sqlite3 "$ROOT/data.db" "SELECT value FROM store_meta WHERE key = 'format';" 2>/dev/null || echo ""; } + + echo "== before" + v1 store > /dev/null + want "the store stamps itself format 1 at open" "^1$" "$(fmt)" + want "and it works" "^4$" "$(v1 eval '(Stdlib.List.length [1L, 2L, 3L, 4L]) |> Stdlib.Int.toString' | tail -1)" + + # A name authored BEFORE the upgrade, to check the rewrite carried meaning and not just bytes. + v1 fn /SU.T.before '() : Int64 = 1717L' > /dev/null + v1 commit "store-upgrade gate" -y > /dev/null + want "a fn authored at v1 resolves" "^1717$" "$(v1 eval 'SU.T.before ()' | tail -1)" + after_author_ops=$(ops) + + echo "== a build that writes the same format has nothing to do" + want "and says so rather than rewriting" "already format 1" "$(v1 store upgrade -y)" + + echo "== an id that does not re-derive stops it dead" + # The line between the two migrations, forged. A store whose ids were minted by a different + # HASHING needs the whole log re-minted, not its blobs rewritten, and the only way to tell is to + # recompute each op's id and compare. So: an op filed under an id its own content does not + # produce. The upgrade must refuse and touch nothing, because half-doing this is unrecoverable. + sqlite3 "$ROOT/data.db" "INSERT INTO package_ops (id, op_blob, applied, effective, commit_hash, origin_ts) + SELECT '00000000-0000-4000-8000-00000000ffff', op_blob, applied, effective, commit_hash, origin_ts + FROM package_ops LIMIT 1;" + forged_ops=$(ops) + out=$(v2 store upgrade -y) + want "it refuses" "re-derives a different id" "$out" + want "and names it an identity-changing migration" "identity-changing migration" "$out" + want "and says the store is untouched" "store is untouched" "$out" + want "the stamp did not move" "^1$" "$(fmt)" + want "and nothing was rewritten" "^$forged_ops$" "$(ops)" + sqlite3 "$ROOT/data.db" "DELETE FROM package_ops WHERE id = '00000000-0000-4000-8000-00000000ffff';" + + echo "== the upgrade" + out=$(v2 store upgrade -y) + want "it reports the move" "Upgraded from format 1 to 2" "$out" + want "it rewrote the whole log" "$after_author_ops op\(s\) rewritten" "$out" + want "it says where the old store is" "pre-v2" "$out" + want "the stamp moved" "^2$" "$(fmt)" + want "and \`store\` names the way back" "store rollback 2" "$(v2 store)" + want "no op was lost" "^$after_author_ops$" "$(ops)" + if [[ -f "$ROOT/data.db.pre-v2" ]]; then ok "the pre-upgrade copy is on disk"; else bad "the pre-upgrade copy is on disk" "$(ls "$ROOT")"; fi + + echo "== and the store still means what it meant" + want "the fn authored at v1 still resolves" "^1717$" "$(v2 eval 'SU.T.before ()' | tail -1)" + want "so does the package set" "^4$" "$(v2 eval '(Stdlib.List.length [1L, 2L, 3L, 4L]) |> Stdlib.Int.toString' | tail -1)" + want "authoring still works after it" "^2727$" "$(v2 fn /SU.T.after '() : Int64 = 2727L' > /dev/null; v2 eval 'SU.T.after ()' | tail -1)" + want "a second upgrade is a no-op" "already format 2" "$(v2 store upgrade -y)" + + echo "== a build that reads an older format than the store" + # The one thing that must not happen quietly. It opens (so a rollback is reachable) and says + # which way out there is. + # It dies on the first package lookup -- resolving the name of the command you typed is one -- + # so what it must do is SAY so, and name a file a person can move without a working binary. + want "it says the store is ahead of it" "format 2 and this build reads 1" "$(v1 store)" + want "and names the file to move back" "pre-v2 -- move it back over" "$(v1 store)" + want "and does not downgrade the stamp behind your back" "^2$" "$(fmt)" + + echo "== rollback" + want "it asks first" "" "$(v2 store rollback 2 < /dev/null)" + want "and rolling back needs the version" "takes the format version" "$(v2 store rollback banana -y)" + want "rollback restores" "Restored from" "$(v2 store rollback 2 -y)" + want "the stamp is back" "^1$" "$(fmt)" + want "this build reads it again" "^1717$" "$(v1 eval 'SU.T.before ()' | tail -1)" + want "and the op authored after the upgrade is gone with it" "not found|couldn't be found" "$(v1 eval 'SU.T.after ()' | tail -1)" + want "rolling back twice is refused, not repeated" "no pre-v9 copy" "$(v2 store rollback 9 -y)" + + echo + echo "passed: $pass failed: $fail" + [[ "$fail" -eq 0 ]] +} diff --git a/scripts/testing/_gates-sync b/scripts/testing/_gates-sync index 07a75b44fa..68e3d4889b 100644 --- a/scripts/testing/_gates-sync +++ b/scripts/testing/_gates-sync @@ -97,6 +97,191 @@ gate_setup() { # # Gate, not an F#/.dark test: a real relay on a real port, answering real requests. An HTTP # status and a leaked error body are properties of a request, not of a function call. +# --- the server folds what it is pushed -------------------------------------------------------- +# +# Crosses a PROCESS boundary and a machine boundary: a real server on a port, a real client with +# its own store, a real push over HTTP. Neither half can be faked in-process, because the whole +# question is whether the SERVER's projection moved. +# +# It did not, until 2026-09-12. A pushed op was stored `effective = 0` and never folded, so the +# server could not see what it hosted: the package browser said "Nothing here" for code every +# client had, and no seed could be cut from the hosted set. +# +# The half that must NOT come back with it: a pushed op is folded but is NOT this store's draft. +# `effective` was carrying both meanings and only the first was given up, so `op_owners` carries +# the second. A discard that counted a peer's push as local work would delete their data. +gate_server_folds() { + set -euo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + WORK=rundir/server-folds-test + PORT=${PORT:-9097} + SECRET='server-folds-secret' + + rm -rf "$WORK" + mkdir -p "$WORK/server/logs" "$WORK/client/logs" + scripts/testing/_copy-store "$WORK/server/data.db" + scripts/testing/_copy-store "$WORK/client/data.db" + + cli() { env DARK_CONFIG_RUNDIR="$PWD/$WORK/client/" HOME="$PWD/$WORK/client" "$PWD/$CLI" "$@"; } + + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" relay_grants "$PWD/$CLI" "$PORT" + cli permissions allow package-write > /dev/null 2>&1 + + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + > "$WORK/server/serve.log" 2>&1 & + SRV=$! + trap 'kill $SRV 2>/dev/null' EXIT + + up=false + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && { up=true; break; } + sleep 1 + done + [[ "$up" == "true" ]] || { + echo "the server never came up on $PORT (see $WORK/server/serve.log)" >&2 + exit 2 + } + + failures=0 + fail() { echo " FAIL $1" >&2; failures=$((failures + 1)); } + + name="Tests.ServerFolds.pushed" + cli fn "$name" '() : String = "pushedtoaserver"' > /dev/null 2>&1 + cli commit "server-folds gate" -y > /dev/null 2>&1 + + before=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'pushed' AND modules LIKE '%ServerFolds%';") + [[ "$before" == "0" ]] || fail "the server already knew the name before the push ($before rows)" + + cli connect "http://localhost:$PORT" --secret "$SECRET" > /dev/null 2>&1 + cli push "http://localhost:$PORT" > /dev/null 2>&1 + + after=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'pushed' AND modules LIKE '%ServerFolds%';") + if [[ "$after" == "1" ]]; then + echo " ok a pushed op reaches the server's own projection" + else + fail "the server stored the push but did not fold it ($after location rows)" + fi + + if curl -s --max-time 5 "http://localhost:$PORT/search?q=ServerFolds" | grep -q "ServerFolds"; then + echo " ok and the package browser can see it" + else + fail "the browser still cannot see a package the server hosts" + fi + + # Folded, but not mine. Hosted ops carry an `op_owners` row, and every draft clause excludes + # them; without that a discard on the server deletes whatever its clients pushed. + hosted=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_ops p + WHERE p.id IN (SELECT op_id FROM op_owners) + AND p.effective = 1;") + [[ "$hosted" -gt 0 ]] || fail "no hosted op is effective, so nothing was folded at all" + + draftable=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_ops + WHERE effective = 1 AND commit_hash IS NULL + AND id NOT IN (SELECT op_id FROM op_branches) + AND id IN (SELECT op_id FROM op_owners);") + if [[ "$draftable" == "0" ]]; then + echo " ok and a hosted op is not the server's own draft, so a discard leaves it alone" + else + fail "$draftable hosted op(s) look like this store's draft; a discard would delete them" + fi + + # Folded is not the same as RUN. A `val` body is guest code; folding it writes a projection row + # and executes nothing, and on a server that distinction is the difference between holding + # somebody's code and running it. Nothing else in the suite can ask this: it needs a real push + # from another store, and then a RESTART, because the startup grow is what would evaluate it. + cli val 'Tests.ServerFolds.hosted' '7L' > /dev/null 2>&1 + cli commit "server-folds hosted val" -y > /dev/null 2>&1 + cli push "http://localhost:$PORT" > /dev/null 2>&1 + + val_rows=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'hosted' AND modules LIKE '%ServerFolds%';") + [[ "$val_rows" == "1" ]] || fail "the pushed val did not fold into the server's projection" + + unevaluated() { + sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM package_values pv + JOIN locations l ON l.item_hash = pv.hash + WHERE l.name = 'hosted' AND l.modules LIKE '%ServerFolds%' AND pv.rt_dval IS NULL;" \ + 2>/dev/null || echo -1 + } + + if [[ "$(unevaluated)" == "1" ]]; then + echo " ok a pushed val is folded but not evaluated by the push" + else + fail "the push evaluated a pushed val's body" + fi + + kill $SRV 2>/dev/null || true + wait $SRV 2>/dev/null || true + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + >> "$WORK/server/serve.log" 2>&1 & + SRV=$! + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && break + sleep 1 + done + + if [[ "$(unevaluated)" == "1" ]]; then + echo " ok and a restart does not evaluate it either" + else + fail "the startup grow evaluated a pushed val's body" + fi + + # What a shared server's MAIN accepts from a stranger. Your own namespace takes a plain push; + # the reviewed one does not. This is `reservedOwners` reborn at the one edge where it belongs -- + # the server -- rather than on every local write, where it used to be and was wrong. + cli fn '/Newcomer.MyApp.mine' '() : String = "mine"' > /dev/null 2>&1 + cli commit "my own namespace" -y > /dev/null 2>&1 + out=$(cli push "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"Pushed"* ]]; then + echo " ok a push to your own namespace is accepted" + else + fail "a push to your own namespace was refused: $(head -c 200 <<<"$out")" + fi + + # The whole log goes up, including the `Darklang.*` baseline every client is seeded with, so + # this only works because ops the server already HAS are skipped. Checking the raw batch + # refuses the first push anybody ever makes. + cli fn '/Darklang.Stdlib.List.sneakyGate' '() : Int64 = 1L' > /dev/null 2>&1 + cli commit "reach into the shared namespace" -y > /dev/null 2>&1 + out=$(cli push "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"403"* && "$out" == *"sneakyGate"* ]]; then + echo " ok a push that binds a reserved name is refused, 403, and names it" + else + fail "a reserved-namespace push was not refused as expected: $(head -c 200 <<<"$out")" + fi + + landed=$(sqlite3 "$WORK/server/data.db" \ + "SELECT count(*) FROM locations WHERE name = 'sneakyGate';") + [[ "$landed" == "0" ]] || fail "the refused push landed anyway ($landed rows)" + + # A branch carrying the same work is fine: a branch is isolated, nobody runs it, and review is + # what moves it to main. + cli branch gate-shared-fix > /dev/null 2>&1 + cli --branch gate-shared-fix fn '/Darklang.Stdlib.List.viaGateBranch' '() : Int64 = 2L' > /dev/null 2>&1 + out=$(cli branch push gate-shared-fix "http://localhost:$PORT" 2>&1) + + if [[ "$out" == *"pushed branch"* ]]; then + echo " ok and the same work on a BRANCH is accepted" + else + fail "a branch push carrying reserved work was refused: $(head -c 200 <<<"$out")" + fi + + [[ "$failures" == "0" ]] || exit 1 + echo "server-folds: a push reaches the server's projection, stays out of its draft, is never run, and cannot rebind the reviewed namespace." +} + gate_relay_routes() { set -euo pipefail @@ -113,9 +298,9 @@ gate_relay_routes() { # the first space. SECRET='relay-test-Bearer secret' - # What a relay needs, under the effect/permission system: the port it binds, the env var its + # What a server needs, under the effect/permission system: the port it binds, the env var its # write secret arrives in, and the store it exists to hold. Granted per instance, and a gate is - # its own instance. Nothing here widens what the relay DOES: it still never folds what it hosts. + # its own instance. DARK_CONFIG_RUNDIR="$PWD/$WORK" relay_grants "$PWD/$CLI" "$PORT" DARK_CONFIG_RUNDIR="$PWD/$WORK" DARK_MATTER_WRITE_SECRET="$SECRET" \ @@ -687,3 +872,188 @@ gate_sync_multi_instance() { echo "passed: $pass failed: $fail" [[ "$fail" -eq 0 ]] } + +# --- seeds: the store as a file, cut at a commit ------------------------------------------------ +# Crosses a process AND a machine boundary: a real server on a port cuts a seed from its own store, +# a client fetches it over HTTP and becomes a store from it. Nothing in-process can answer the +# question, which is whether what CI fetches is fixed by the COMMIT it names rather than by when it +# asked. That is the property a pin rests on, so it is tested by moving the store underneath and +# asking again. +gate_seed_serving() { + set -euo pipefail + + CLI=${CLI:-backend/Build/out/Cli/Debug/net10.0/Cli} + WORK=rundir/seed-serving-test + PORT=${PORT:-9094} + SECRET='seed-serving-secret' + + rm -rf "$WORK" + mkdir -p "$WORK/server/logs" "$WORK/client/logs" "$WORK/fresh" "$WORK/fresh2" + scripts/testing/_copy-store "$WORK/server/data.db" + scripts/testing/_copy-store "$WORK/client/data.db" + + cli() { env DARK_CONFIG_RUNDIR="$PWD/$WORK/client/" HOME="$PWD/$WORK/client" "$PWD/$CLI" "$@"; } + + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" relay_grants "$PWD/$CLI" "$PORT" + # Cutting a seed WRITES a file and opens SQLite on it, so the server needs those two beyond what + # `relay_grants` gives a read-only relay. Scoped to its own rundir: the cache lives beside the store. + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" "$PWD/$CLI" permissions allow file read+write "$PWD/$WORK/server" > /dev/null 2>&1 + DARK_CONFIG_RUNDIR="$PWD/$WORK/server" "$PWD/$CLI" permissions allow native > /dev/null 2>&1 + cli permissions allow package-write > /dev/null 2>&1 + + env DARK_CONFIG_RUNDIR="$PWD/$WORK/server/" HOME="$PWD/$WORK/server" \ + DARK_MATTER_WRITE_SECRET="$SECRET" \ + "$PWD/$CLI" serve Darklang.Matter.router --port "$PORT" \ + > "$WORK/server/serve.log" 2>&1 & + SRV=$! + trap 'kill $SRV 2>/dev/null' EXIT + + up=false + for _ in $(seq 1 40); do + [[ "$(curl -s --max-time 2 "http://localhost:$PORT/ping" 2>/dev/null)" == "pong" ]] && { up=true; break; } + sleep 1 + done + [[ "$up" == "true" ]] || { + echo "the server never came up on $PORT (see $WORK/server/serve.log)" >&2 + exit 2 + } + + U="http://localhost:$PORT" + failures=0 + fail() { echo " FAIL $1" >&2; failures=$((failures + 1)); } + + # `/seed/meta` exists so a pin can be CHECKED without downloading the seed. If it needed the + # seed to answer, every "has anything moved?" would cost twelve megabytes. + meta=$(curl -s --max-time 15 "$U/seed/meta") + field() { printf '%s' "$meta" | sed -n "s/.*\"$1\":\"\\?\\([^,\"}]*\\).*/\\1/p"; } + HEAD1=$(field head) + [[ -n "$HEAD1" ]] || fail "/seed/meta named no head commit: $meta" + [[ "$(field format)" == "1" ]] || fail "/seed/meta reported format $(field format), not 1" + [[ "$(field ops)" -gt 0 ]] || fail "/seed/meta reported no ops: $meta" + echo " ok /seed/meta answers without cutting a seed ($(field ops) ops at ${HEAD1:0:8})" + + fetch() { # fetch + local dir="$1" commit="$2" args=() + [[ "$commit" == "latest" ]] || args=(--commit "$commit") + env DARK_CONFIG_RUNDIR="$PWD/$dir/" scripts/fetch-seed --url "$U" "${args[@]}" --force \ + > "$WORK/fetch.log" 2>&1 \ + || { echo " FAIL the fetch at $commit failed; the server said:" >&2 + curl -s --max-time 60 "$U/seed/$commit.db" | head -c 400 >&2; echo >&2; return 1; } + } + ops_in() { sqlite3 "$1" "SELECT count(*) FROM package_ops;" 2>/dev/null || echo -1; } + + fetch "$WORK/fresh" "$HEAD1" + SEED1="$WORK/fresh/seed.db" + [[ -f "$SEED1" ]] || { echo "the fetch produced no seed (see above)" >&2; exit 2; } + + # A seed is ONE file. Served in WAL mode, the `.db` alone is not the whole database, so whether a + # fetch got everything would depend on when a checkpoint happened to run. + if [[ -f "$SEED1-wal" ]]; then + fail "the served seed came with a -wal, so the file is not the whole database" + else + echo " ok the served seed is a single settled file" + fi + + cut_at=$(sqlite3 "$SEED1" "SELECT value FROM store_meta WHERE key = 'cut_at';" 2>/dev/null || echo "") + if [[ "$cut_at" == "$HEAD1" ]]; then + echo " ok and it says which commit it is a cut of" + else + fail "the seed says it was cut at '$cut_at', but it was asked for $HEAD1" + fi + + N1=$(ops_in "$SEED1") + + # The seed has to be a usable STORE, not just a well-formed file: a fresh install folds it and + # then resolves names out of it. This is the whole point of serving one. + env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ + "$PWD/$CLI" permissions allow package-read > /dev/null 2>&1 || true + got=$(env DARK_CONFIG_RUNDIR="$PWD/$WORK/fresh/" HOME="$PWD/$WORK/fresh" \ + "$PWD/$CLI" eval '(Stdlib.List.length [1L, 2L, 3L]) |> Stdlib.Int.toString' 2>&1 | tail -1 || true) + if [[ "$got" == *3* ]]; then + echo " ok a store grown from the fetched seed resolves names and evaluates" + else + fail "a store grown from the fetched seed could not evaluate: $got" + fi + + # Now move the server's store underneath the pin, by pushing new work to it. + cli fn 'Tests.SeedServing.after' '() : String = "afterthepin"' > /dev/null 2>&1 + cli commit "seed-serving gate" -y > /dev/null 2>&1 + cli connect "$U" --secret "$SECRET" > /dev/null 2>&1 + cli push "$U" > /dev/null 2>&1 + + HEAD2=$(curl -s --max-time 15 "$U/seed/meta" | sed -n 's/.*"head":"\([^"]*\)".*/\1/p' || true) + if [[ -n "$HEAD2" && "$HEAD2" != "$HEAD1" ]]; then + echo " ok the push moved the server's head (${HEAD1:0:8} -> ${HEAD2:0:8})" + else + fail "the push did not move the server's head, so the rest of this gate proves nothing" + exit 1 + fi + + # The cache is deleted first, so the cut is RECOMPUTED against a store that has moved. Served + # from cache the next assertion would pass for the wrong reason. + rm -rf "$WORK/server/seeds" + + fetch "$WORK/fresh2" "$HEAD1" + SEED2="$WORK/fresh2/seed.db" + N2=$(ops_in "$SEED2") + if [[ "$N2" == "$N1" ]]; then + echo " ok re-cut at the same commit after the push: same $N1 ops" + else + fail "the same commit cut $N1 ops before the push and $N2 after; a pin is not reproducible" + fi + + has_commit=$(sqlite3 "$SEED2" "SELECT count(*) FROM commits WHERE hash = '$HEAD2';" 2>/dev/null || echo -1) + if [[ "$has_commit" == "0" ]]; then + echo " ok and it does not carry work committed after it" + else + fail "a seed cut at $HEAD1 carries the later commit $HEAD2" + fi + + fetch "$WORK/fresh2" latest + N3=$(ops_in "$SEED2") + if [[ "$N3" -gt "$N1" ]]; then + echo " ok /seed/latest.db moves with the store ($N1 -> $N3 ops)" + else + fail "/seed/latest.db still cuts $N3 ops after a push that added some" + fi + + # The PIN, end to end, against a scratch copy of `package-set.txt` so the tree's own is untouched. + # This is the closest thing to a test of 8.C there can be from here: the real path runs against a + # deployed server, and nothing deploys from this clone. + cp package-set.txt "$WORK/package-set.txt" + export DARK_PACKAGE_SET="$PWD/$WORK/package-set.txt" + + if scripts/packages/pin head --url "$U" > "$WORK/pin.log" 2>&1; then + pinned=$(awk '$1 == "commit" { print $2 }' "$DARK_PACKAGE_SET") + if [[ "$pinned" == "$HEAD2" ]]; then + echo " ok \`pin head\` records the server's head, checked before writing it" + else + fail "pin wrote '$pinned', but the server's head is $HEAD2" + fi + else + fail "pin head failed: $(cat "$WORK/pin.log")" + fi + + # A commit the server does not have is refused AT PIN TIME. Without this the pin is only + # discovered to be wrong at the next CI run, by which time nobody is looking. + if scripts/packages/pin 0000000000000000 --url "$U" > "$WORK/pin.log" 2>&1; then + fail "pin accepted a commit the server does not have" + else + echo " ok and refuses a commit the server does not have" + fi + + # With a pin in place, a fetch that names no commit takes the PIN rather than the head. That is + # the whole behaviour CI depends on. + scripts/packages/pin "$HEAD1" --url "$U" > "$WORK/pin.log" 2>&1 || fail "could not pin back to $HEAD1" + rm -f "$WORK/fresh2/seed.db" + fetch "$WORK/fresh2" latest + if [[ "$(ops_in "$SEED2")" == "$N1" ]]; then + echo " ok and a fetch with no commit follows the pin, not the head" + else + fail "a pinned fetch got $(ops_in "$SEED2") ops; the pin at ${HEAD1:0:8} has $N1" + fi + unset DARK_PACKAGE_SET + + [[ "$failures" == "0" ]] || exit 1 + echo "seed-serving: a seed is fixed by the commit it names, follows the pin, and grows into a working store." +} diff --git a/scripts/testing/_gates-ui b/scripts/testing/_gates-ui index a36cf7806f..ea9230b8ac 100644 --- a/scripts/testing/_gates-ui +++ b/scripts/testing/_gates-ui @@ -136,6 +136,18 @@ gate_lsp_branches() { sqlite3 "$WORK/data.db" 'DELETE FROM config_v0;' 2>/dev/null || true export DARK_CONFIG_RUNDIR="$PWD/$WORK" + + # The policy is per INSTANCE, and this gate is its own instance, so it has to grant what the + # server needs rather than inherit whatever the dev instance happens to allow. It used to + # inherit, which is why it passed on some machines and not others: the LSP writes a debug log, + # a denied effect raises, and a raise inside the server's loop takes the loop down -- so the + # gate's only symptom was every request going unanswered. + # + # `LspServer.logFilePath` is a hardcoded absolute container path, which is its own bug and not + # this gate's to fix; granting it by name is the honest way to depend on it until it moves. + "$PWD/$CLI" permissions allow file write "/home/dark/app/rundir/logs/lsp-server.log" >/dev/null 2>&1 + "$PWD/$CLI" permissions allow package-write >/dev/null 2>&1 + "$PWD/$CLI" branch create lsp-gate >/dev/null 2>&1 "$PWD/$CLI" --branch lsp-gate fn LspGate.T.f '() : Int64 = 5L' >/dev/null 2>&1 diff --git a/scripts/testing/gates b/scripts/testing/gates index 12e20e1c50..b9cab35878 100755 --- a/scripts/testing/gates +++ b/scripts/testing/gates @@ -23,6 +23,8 @@ set -uo pipefail GATES=( setup relay-routes + server-folds + seed-serving sync-hostile-relay sync-multi-instance lsp-branches @@ -30,20 +32,26 @@ GATES=( workbench-views first-day backups-restore + store-upgrade reload-is-reproducible gates-are-clean ) -# What CI runs, in one config line: the three gates it has always run, the ones that need only -# the debug build already in backend/Build. first-day wants a published artifact and the sync -# gates add minutes; those run locally, by name or via `all`. -CI_GATES=(relay-routes workbench-scm workbench-views) +# What CI runs, in one config line: the ones that need only the debug build already in +# backend/Build. first-day wants a published artifact and the sync gates add minutes; those run +# locally, by name or via `all`. +# +# `store-upgrade` is here rather than local-only because it is the migrator's only test, and a +# migration that has quietly stopped working is discovered at the worst possible moment. +CI_GATES=(relay-routes server-folds seed-serving store-upgrade workbench-scm workbench-views) describe() { # shellcheck disable=SC2016 # the backticks are prose, not expansion case "$1" in setup) echo '`dark sync setup`, end to end, own rundir' ;; relay-routes) echo "the relay's HTTP surface, given bad input" ;; + server-folds) echo "a push folds into the server's projection, and is never run there" ;; + seed-serving) echo 'a seed cut at a commit, fetched over HTTP, grown into a store' ;; sync-hostile-relay) echo 'the sync/branch CLIENTS, given a relay that lies' ;; sync-multi-instance) echo 'four instances, branches, review queues, agents' ;; lsp-branches) echo 'the `dark/*` branch surface, as an editor drives it' ;; @@ -51,6 +59,7 @@ describe() { workbench-views) echo 'every workbench view, one session each' ;; first-day) echo 'a published binary, empty home, whole SCM workflow' ;; backups-restore) echo 'backup, wipe, restore; the destructive verbs ask first' ;; + store-upgrade) echo 'a synthetic format bump, carried in place, and rolled back' ;; reload-is-reproducible) echo 'two reloads of one tree agree on every op id' ;; gates-are-clean) echo 'no gate touches the shared dev store (slow: re-runs the rest)' ;; *) echo '?' ;; diff --git a/scripts/testing/test-build-planning.py b/scripts/testing/test-build-planning.py index 5253e69c14..56519c3e3e 100755 --- a/scripts/testing/test-build-planning.py +++ b/scripts/testing/test-build-planning.py @@ -115,8 +115,20 @@ def test_fsproj_takes_a_full_build(self): self.assertTrue(self.actions("backend/src/LibDB/LibDB.fsproj").backend_full_build) def test_dark_package_reloads_packages(self): - self.assertTrue(self.actions("packages/darklang/stdlib/list.dark") - .reload_all_packages) + # Pinned to disk mode rather than inheriting the shell's. Where packages come from + # decides this answer, so a test that does not say which it means passes or fails on + # an env var the reader cannot see. + with Env(DARK_CONFIG_PACKAGES_SOURCE="disk"): + self.assertTrue(self.actions("packages/darklang/stdlib/list.dark") + .reload_all_packages) + + def test_dark_package_routes_nowhere_from_seed(self): + # The point of seed mode: the store is the source, so editing the text changes nothing + # a build should act on. + with Env(DARK_CONFIG_PACKAGES_SOURCE="seed"): + should = self.actions("packages/darklang/stdlib/list.dark") + self.assertFalse(should.reload_all_packages) + self.assertEqual(should.unrouted, ["packages/darklang/stdlib/list.dark"]) def test_migration_runs_migrations(self): self.assertTrue(self.actions("backend/migrations/001-init.sql").run_migrations) @@ -136,14 +148,27 @@ def expand(self, path, **kw): return _buildplan.expand(should, **kw).chosen() def test_fsharp_change_reaches_the_package_reload(self): - with Env(CI=None): + # Disk mode, said out loud: the cascade from migrations to the reload exists only + # because the store is built from text. + with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="disk"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), - ["backend_quick_build", "run_migrations", "reload_all_packages"]) - with Env(CI="true"): + ["backend_quick_build", "run_migrations", "reload_all_packages", "check_refs"]) + with Env(CI="true", DARK_CONFIG_PACKAGES_SOURCE="disk"): + self.assertEqual( + self.expand("backend/src/LibDB/Queries.fs"), + ["backend_full_build", "run_migrations", "reload_all_packages", "check_refs"]) + + def test_fsharp_change_stops_at_migrations_from_seed(self): + # Half the cost of an F# change today is a package reload that usually did not need to + # happen. In seed mode it does not happen at all. + # + # `check_refs` still does, and has to: an F# change can add or move a kernel ref, and in + # seed mode there is no reload to notice that the store cannot answer for it. + with Env(CI=None, DARK_CONFIG_PACKAGES_SOURCE="seed"): self.assertEqual( self.expand("backend/src/LibDB/Queries.fs"), - ["backend_full_build", "run_migrations", "reload_all_packages"]) + ["backend_quick_build", "run_migrations", "check_refs"]) def test_full_build_replaces_the_quick_one(self): actions = self.expand("backend/src/LibDB/LibDB.fsproj")