Package bootstrapping

.circleci/config.yml

@@ -228,8 +228,8 @@ jobs:
           name: Run migrations to create database
           command: scripts/run-local-exec migrations run
       - run:
-          name: Load packages into database
-          command: scripts/build/reload-packages
+          name: Get the package set (the pin, or packages/)
+          command: scripts/build/prepare-package-set
       - run:
           name: Export the seed the publish embeds
           command: scripts/run-local-exec export-seed rundir/seed.db
@@ -315,8 +315,8 @@ jobs:
           name: Run migrations to create database
           command: scripts/run-local-exec migrations run
       - run:
-          name: Load packages into database
-          command: scripts/build/reload-packages
+          name: Get the package set (the pin, or packages/)
+          command: scripts/build/prepare-package-set
 
       - assert-clean-worktree
       # Just the host build, on main as much as anywhere else. This is a
@@ -379,27 +379,48 @@ jobs:
       - run:
           name: Run migrations to create database
           command: scripts/run-local-exec migrations run
+      # Keyed on the PIN, not on the week: a pinned package set is immutable, so this hits on every
+      # release build until someone re-pins. Costs nothing when there is no pin -- `package-set.txt`
+      # then says `commit unset`, the key is stable, and the cached rundir is simply not used
+      # because the step below reloads instead.
+      - restore_cache:
+          keys:
+            - v1-seed-{{ checksum "package-set.txt" }}
       - run:
-          name: Load packages into database
-          command: scripts/build/reload-packages
+          name: Get the package set (the pin, or packages/)
+          command: scripts/build/prepare-package-set
       - run:
           name: Export the seed
           command: |
-            sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true
-            scripts/run-local-exec export-seed rundir/seed.db
+            # A no-op on the pinned path, where the fetched file already IS the seed and is settled
+            # out of WAL. On the reload path the store is live and this is the cut.
+            if [[ -f rundir/seed.db ]] && [[ "$(awk '$1 == "commit" { print $2 }' package-set.txt)" != "unset" ]]; then
+              echo "using the fetched seed at the pin"
+            else
+              sqlite3 rundir/data.db "PRAGMA wal_checkpoint(TRUNCATE);" || true
+              scripts/run-local-exec export-seed rundir/seed.db
+            fi
+      - save_cache:
+          paths:
+            - rundir/seed.db
+          key: v1-seed-{{ checksum "package-set.txt" }}
       - persist_to_workspace:
           root: "."
           paths:
             - rundir/seed.db
             # Hand over the hash file too, not just the seed.
             #
-            # `reload-packages` above populates it, but it is gitignored, so the
-            # build jobs' own checkout has only the empty one MSBuild creates.
-            # They then embed THAT, and every ref in the shipped binary resolves
-            # to "" -> `FnNotFound` on every command. It goes unnoticed inside a
-            # source tree, because there the file on disk wins over the embedded
-            # copy and the first run regenerates it. A user running the artifact
-            # anywhere else has only the embedded copy.
+            # It is tracked now, so a checkout already has it; this carries the
+            # one the step above just PRODUCED, which is the one that matches the
+            # seed beside it. On the pinned path those can differ, and when they
+            # do the difference is the finding: the pin's package set is not the
+            # one this kernel was built against, and `assert-clean-worktree` in
+            # the build jobs is what says so.
+            #
+            # Embedding a mismatched one is not a loud failure. Every ref in the
+            # shipped binary resolves to "" -> `FnNotFound` on every command, and
+            # only OUTSIDE a source tree, because inside one the file on disk
+            # wins over the embedded copy and the first run regenerates it.
             - backend/src/LibExecution/package-ref-hashes.txt
 
   # All four linux artifacts from one x64 runner. The cross-compiles work

.gitignore

@@ -24,10 +24,6 @@ backend/src/Wasm/publish/
 backend/src/Wasm/wwwroot/packages.snapshot
 darklang-repl.zip
 
-# Generated by reload-packages. Untracked on purpose: two branches that both touch packages
-# produce conflicting hashes, and a rebase on GitHub cannot regenerate it -- you would have to pull,
-# reload and commit by hand every time.
-backend/src/LibExecution/package-ref-hashes.txt
 
 # Native SQLite archives for AOT — built by scripts/build/build-sqlite.sh
 # from the amalgamation, cached locally + in CI.

AGENTS.md

@@ -62,6 +62,24 @@ alter the serialized package format, and there's no cheap way to ask whether thi
 did. Narrowing it is the biggest remaining win in the loop, and it's entangled with
 `package-ref-hashes.txt`, so coordinate before starting.
 
+## Where the package set comes from
+
+`package-set.txt` at the root says which of two, and it ships `commit unset`, which
+means the first:
+
+    commit unset     built from `packages/` by reloading it, as always
+    commit <hash>    fetched as a seed from a package server, at that commit
+
+`scripts/build/prepare-package-set` is the one place that answers that question, and
+CI's package-reloading jobs go through it. `scripts/packages/pin` writes the pin. The
+pinned path has never run against a deployed server, so treat it as
+written-and-unverified until it has.
+
+**`docs/package-workflow.md` is the how-to**: adding a builtin and calling it from
+Dark, referencing a new package type or fn from F#, what your coworker does to build
+your branch, publishing, the pin, format changes, and what will bite. `dark docs
+packages` is the short version from inside the CLI.
+
 The container builds once when it starts. Rebuild-on-save is available but off by
 default, because a five-file change under a watcher pays for five rebuilds, four of them
 on half-finished states that produce real-looking failures:
@@ -321,11 +339,42 @@ op log directly.
 
 ## Gotchas
 
+**The test lock.** `run-backend-tests` refuses if another run holds `rundir/test.lock`. Wait for
+it. Do not clear it with a broad `pkill -f "out/Tests"`: that pattern matches every sibling clone
+on this machine and will kill somebody else's suite. Scope it to the clone if you must
+(`pkill -f "boot-migrate/backend/Build/out/Tests"`).
+
+**`Stdlib.Sqlite` parameters are `@p0`, `@p1`, not `?`.** With `?` nothing matches and nothing
+errors, so a cache silently never fills.
+
+**A new CLI command joins the registry sweep the day it is registered**, and the sweep runs every
+command with a bogus argument. An expensive command therefore taxes the whole suite; `grep` cost
+nine minutes until it learned to refuse an unscoped search.
+
+**Dark syntax traps.** No `let private`. No `rec` keyword. The list separator is `,`. A comment
+inside a list literal breaks the parser. Parenthesise a piped qualified call:
+`(Mod.f x) |> ...`.
+
+**Measure the artifact people actually run.** Debug, `publish -c Release`, R2R and AOT differ by
+about 25x on startup. Three separate wrong conclusions in one week came from measuring the wrong
+one.
+
 **PackageRefs stale hash.** `backend/src/LibExecution/package-ref-hashes.txt` isn't in git.
 Empty is tolerated; non-empty with a missing key crashes at startup with "PackageRefs: X
 hash not found". After adding a ref:
 `> backend/src/LibExecution/package-ref-hashes.txt && ./scripts/build/reload-packages`
 
+It IS tracked, and it is a projection of the store, which is the awkward combination it has to
+be: committing it is what makes a kernel entry point changing identity visible in review, and
+`assert-clean-worktree` is what enforces it. So a PR that moves one of the 206 hashes has to
+carry the regenerated file.
+
+**Resolving a conflict in it: regenerate, never hand-merge.** Two branches that both touch
+packages will conflict here, and the lines are content hashes, so picking sides is meaningless.
+`git checkout --theirs` it, then `./scripts/build/reload-packages` (or, on a store that came
+from a seed and has no `packages/` to reload, `scripts/run-local-exec refs generate`) and commit
+what that produces.
+
 **Name resolution in test files.** `backend/testfiles/` is parsed with owner "Tests", so
 `Darklang.*` names need full qualification or the `Stdlib.` shortcut. `Stdlib.Json.ParseError.toString`
 and `Darklang.SCM.Branch.mainBranchId` resolve; `SCM.Branch.mainBranchId` doesn't. Impl:

backend/migrations/schema/07-names.sql

@@ -112,6 +112,28 @@ CREATE TABLE IF NOT EXISTS package_dependencies (
 );
 CREATE INDEX IF NOT EXISTS idx_package_dependencies_depends_on
   ON package_dependencies(depends_on_hash);
+
+-- Which BUILTINS an item's body calls. A separate table from `package_dependencies` because a
+-- builtin edge is a different kind of thing: a builtin is not content-addressed, it is a (name,
+-- version) in whatever kernel you are running, so there is no hash to join on and none of the
+-- location columns apply.
+--
+-- This is what lets a store say what KERNEL it needs. Without it you can ask which package items
+-- reference each other and not which builtins they call, so half of the kernel/package-set
+-- interface is invisible and a builtin can be deleted out from under code that calls it. The
+-- checks that used to answer this grepped `.dark` text off disk, which stops being possible the
+-- day packages come from a seed rather than a tree.
+--
+-- Derived, like `package_dependencies`: rebuilt by the fold, dropped by `Seed.export`.
+CREATE TABLE IF NOT EXISTS package_builtin_deps (
+  item_hash TEXT NOT NULL,
+  builtin_name TEXT NOT NULL,
+  builtin_version INTEGER NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_package_builtin_deps_unique
+  ON package_builtin_deps(item_hash, builtin_name, builtin_version);
+CREATE INDEX IF NOT EXISTS idx_package_builtin_deps_name
+  ON package_builtin_deps(builtin_name);
 CREATE INDEX IF NOT EXISTS idx_package_dependencies_item
   ON package_dependencies(item_hash);
 -- Partial index for the propagation query: "who depends on this

backend/src/Builtins/Builtins.Matter/Libs/PM/PackageOps.fs

@@ -500,8 +500,38 @@ let fns (pm : PT.PackageManager) : List<BuiltInFn> =
       deprecated = NotDeprecated }
 
 
-    // RELAY store: bulk-insert ops + record ownership (owner) in one transaction, NO fold
-    // (a relay serves blobs, not projections). The perf path for a relay recording pushes.
+    // SERVER store: bulk-insert ops + record ownership in one transaction, THEN fold.
+    //
+    // It folded nothing until 2026-09-12, on the grounds that a server serves blobs rather than
+    // projections. That also meant it could not see what it hosted: `/m` showed "Nothing here"
+    // for packages every client had, a seed could not be cut from the hosted set, and pushing new
+    // code to a server could never change what it ran. All three are wanted, so it folds.
+    // Asked BEFORE storing, so a refusal can be a refusal rather than a server error. The same
+    // rule is enforced inside `storeOpsWithOwner` as the backstop -- this exists so the answer can
+    // carry a status code and a list of names, not so the rule lives in two places.
+    { name = fn "scmReservedBindings" 0
+      typeParams = []
+      parameters =
+        [ Param.make
+            "records"
+            (TList(TTuple(TString, TString, [ TString ])))
+            "(id, blobHex, originTs) triples" ]
+      returnType = TList TString
+      description =
+        "The reserved names these ops would bind into this store's main, and that it will not accept. Empty means the push is fine."
+      fn =
+        (function
+        | _, _, _, [| DList(_, records) |] ->
+          uply {
+            let! names = LibDB.Inserts.reservedBindingsIn (opRecords records)
+            return Dval.list KTString (names |> List.map Dval.string)
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      callEffects = set [ Effect.PackageRead ]
+      deprecated = NotDeprecated }
+
     { name = fn "scmStoreOps" 0
       typeParams = []
       parameters =
@@ -524,6 +554,9 @@ let fns (pm : PT.PackageManager) : List<BuiltInFn> =
           uply {
             try
               let! n = LibDB.Inserts.storeOpsWithOwner owner (opRecords records)
+              // Fold what just arrived, so the projection a seed and `/m` read is current. Cheap:
+              // ~116us an op, and a push is tens of ops.
+              let! _ = LibDB.Seed.applyUnappliedOps ()
               return resultOk (Dval.int (bigint n))
             with ex ->
               return resultError (Dval.string ex.Message)

backend/src/Builtins/Builtins.Matter/Libs/PM/Seed.fs

@@ -6,26 +6,34 @@ open LibExecution.Effects
 
 module Dval = LibExecution.Dval
 module Builtin = LibExecution.Builtin
+module C2DT = LibExecution.CommonToDarkTypes
+module D = LibExecution.DvalDecoder
 
 open Builtin.Shortcuts
 
 
 let fns : List<BuiltInFn> =
   [ { name = fn "pmSeedExport" 0
       typeParams = []
-      parameters = [ Param.make "outputPath" TString "" ]
+      parameters =
+        [ Param.make "outputPath" TString ""
+          Param.make
+            "upToCommit"
+            (TypeReference.option TString)
+            "cut the seed at this commit and its ancestors, so the same commit yields the same ops however far the store has moved since; `None` takes everything committed" ]
       returnType = TypeReference.result TUnit TString
       description = "Export a minimal seed.db from the current database"
       fn =
         let resultOk = Dval.resultOk KTUnit KTString
         let resultError = Dval.resultError KTUnit KTString
         (function
-        | state, vm, _, [| DString outputPath |] ->
+        | state, vm, _, [| DString outputPath; upToCommit |] ->
           uply {
             try
               let outputPath = LibExecution.Host.normalizeFilePath outputPath
               LibExecution.PermissionCheck.requireFileWrite state vm outputPath
-              do! LibDB.Seed.export outputPath
+              let upToCommit = C2DT.Option.fromDT D.string upToCommit
+              do! LibDB.Seed.exportAt outputPath upToCommit
               return resultOk DUnit
             with ex ->
               return resultError (DString ex.Message)

backend/src/Builtins/Builtins.Matter/Libs/PM/Store.fs

@@ -12,6 +12,7 @@ open LibExecution.Effects
 open LibExecution.Builtin.Shortcuts
 
 module Dval = LibExecution.Dval
+module VT = LibExecution.ValueType
 
 
 /// Refuse a store-level operation unless every non-root frame is bundled Darklang
@@ -101,6 +102,82 @@ let fns () : List<BuiltInFn> =
       callEffects = set [ Effect.PackageWrite ]
       deprecated = NotDeprecated }
 
+    // Move the store to the op-log format this build writes, and put back the copy that made.
+    //
+    // First-party only, for the same reason as backup/restore: this rewrites every op blob in the
+    // log and drops every projection. A guest holding package-write must not reach it through a
+    // wrapper.
+    //
+    // A no-op in practice until the first real format bump -- `from` and `to` are equal, and it
+    // says so rather than doing anything. The mechanism exists now so the bump is not also the
+    // first time the migration runs.
+    { name = fn "pmStoreUpgrade" 0
+      typeParams = []
+      parameters = [ Param.make "unit" TUnit "" ]
+      returnType =
+        TypeReference.result
+          (TTuple(TInt64, TInt64, [ TInt64; TInt64; TString ]))
+          TString
+      description =
+        "Rewrites this store's op log into the format this build writes. Ok is (from, to, rewritten, unreadable, backupPath)."
+      fn =
+        let okKT =
+          KTTuple(
+            VT.known KTInt64,
+            VT.known KTInt64,
+            [ VT.known KTInt64; VT.known KTInt64; VT.known KTString ]
+          )
+        (function
+        | state, vm, _, [| DUnit |] ->
+          uply {
+            requireBundledCaller state vm "pmStoreUpgrade"
+            let! result = LibDB.StoreUpgrade.upgrade ()
+            match result with
+            | Ok r ->
+              return
+                Dval.resultOk
+                  okKT
+                  KTString
+                  (DTuple(
+                    DInt64(int64 r.from),
+                    DInt64(int64 r.to_),
+                    [ DInt64(int64 r.rewritten)
+                      DInt64(int64 r.unreadable)
+                      DString r.backup ]
+                  ))
+            | Error e -> return Dval.resultError okKT KTString (DString e)
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      // `Native` alongside the writes: this opens SQLite directly to run the rewrite in one
+      // transaction, which a path rule alone cannot confine.
+      callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ]
+      deprecated = NotDeprecated }
+
+    { name = fn "pmStoreRollback" 0
+      typeParams = []
+      parameters =
+        [ Param.make "target" TInt64 "the format version that was upgraded TO" ]
+      returnType = TypeReference.result TString TString
+      description =
+        "Restores the copy `pmStoreUpgrade` took on its way to <param target>. Ok is the path restored from."
+      fn =
+        (function
+        | state, vm, _, [| DInt64 target |] ->
+          uply {
+            requireBundledCaller state vm "pmStoreRollback"
+            let! result = LibDB.StoreUpgrade.rollback (uint32 target)
+            match result with
+            | Ok path -> return Dval.resultOk KTString KTString (DString path)
+            | Error e -> return Dval.resultError KTString KTString (DString e)
+          }
+        | _ -> incorrectArgs ())
+      sqlSpec = NotQueryable
+      previewable = Impure
+      callEffects = set [ Effect.PackageRead; Effect.PackageWrite; Effect.Native ]
+      deprecated = NotDeprecated }
+
     // Whether a write secret is stored for a relay, WITHOUT handing it over.
     //
     // `dark sync setup` needs to know if one is already there, so that pressing Enter keeps it rather