From 75f2a1217ab6c9a315ce7846588afac8dae5f6ec Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Sat, 10 Oct 2026 14:08:04 +0200 Subject: [PATCH] feat(packages): build Datascripts shipped inside a Package --- docs/commands.md | 25 +++-- src/cmd/packages/build.rs | 196 +++++++++++++++++++++++++++++------ src/cmd/packages/git.rs | 2 +- src/cmd/packages/identity.rs | 38 ++++++- src/cmd/packages/mod.rs | 29 ++++-- src/cmd/packages/review.rs | 40 ++++++- 6 files changed, 271 insertions(+), 59 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index 44eb8c6..dd5d51d 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -239,8 +239,8 @@ around saying so. **Everything that can refuse the install happens before anything is copied.** The name must be one the build accepts (`[a-zA-Z][a-zA-Z0-9_-]*` — the build maps `my-package` onto the module -`pkg_my_package` and panics on anything else). The shape must be one the build accepts: client-only -and Rust-only are both valid, and when `src/` exists, `src/mod.rs` is required. The name must +`pkg_my_package` and panics on anything else). A Package must carry `src/`, `client/`, `data/`, `scripts/` or `datascripts/`. +When `src/` exists, `src/mod.rs` is required. The name must collide with neither the enabled nor the disabled inventory, compared on the Rust identifier rather than the folder name — `my-package` and `my_package` fold onto the same module. @@ -248,9 +248,8 @@ Then it prints a **Trust Review** and asks. The review is a deterministic, read- candidate folder using a port of the build's own marker scan, so a commented-out or quoted marker registers nothing here either. It reports tables, reducers, hooks, tick passes, character-owned sweeps, addons, client overrides and Runtime Script sources. Runtime Scripts are named because they -run on the realm once the Package is built. A Package cannot ship a Datascript of its own — the -authoring toolchain lives in `datascripts/`, outside any Package folder — so that row is an explicit -"none detected" rather than silence. The review states plainly that everything else in the Package's +run on the Realm once the Package is built. It also names Package-local Datascripts, which +`packages build` runs as trusted code on the author's machine. The review states plainly that everything else in the Package's Rust is trusted code and that it is an inventory, not a security guarantee. On confirmation the folder is **copied, never symlinked**: a linked Package would compile from a @@ -460,7 +459,8 @@ then up to eight steps, in this order: database. 2. `bun install --frozen-lockfile` installs exactly what `datascripts/bun.lock` records. Frozen, so a build never silently resolves a newer dependency than the next author will get. -3. `tsc --noEmit` is the typecheck gate. Nothing is emitted; the answer is the exit code. +3. `tsc --noEmit` typechecks Core's Datascript project and each enabled Package's local Datascripts. + The CLI extends Core's compiler configuration in a temporary file. No JavaScript is emitted. Steps 4 to 8 run only when an enabled Package carries a Datascript or a Runtime Script. A checkout with neither builds exactly as it did before those steps existed: @@ -469,8 +469,9 @@ with neither builds exactly as it did before those steps existed: fails fast with the exact `lyracore-importer --spell-snapshot` command to build one, once, rather than letting every Datascript fail with the same "cannot read" error in turn. Skipped when no Package has a Datascript: a Runtime Script reads no base data. -5. Every enabled Package's Datascripts run, one `bun run` subprocess per file, in name order. The - first script to throw stops the build; later scripts and later Packages never run. +5. Each enabled Package runs `datascripts/src//*.ts`, then + `packages//datascripts/*.ts`. Each directory runs in file-name order, with one `bun run` + subprocess per file. The first failure stops the build. 6. Every enabled Package with a `scripts/` folder compiles its Runtime Scripts into one Script Artifact, one `bun run` subprocess per Package, in folder-name order. The builder is handed the Module's Event Binding catalogue, read from `lyracore-delta-check --print-events`, so a mistyped @@ -524,6 +525,10 @@ the schema's authority. `src/reference.ts` is the standing schema check. It names real columns, so it is the file that fails when the schema moves under it. Keep it referencing real columns. +A Package can ship `packages//datascripts/welcome.ts`. It imports the authoring library as +`../../../datascripts/lib/index.ts`. The older `datascripts/src//` location still works. +The Build Identity covers both source directories, so editing either requires a rebuild. + ### Bun is author-side only `packages build` is the only command that needs Bun, and authoring Datascripts is the only reason @@ -553,8 +558,8 @@ packages/fire_nova/ fire_nova.script.json the Script Artifact. Committed in the Official Package Collection ``` -The sources live **inside** the Package, unlike a Datascript. A Datascript sits outside because only -artifacts belong in a Package folder; a Runtime Script is the Package's own content. +Runtime Script sources live in the Package's `scripts/` directory. Author-time Datascripts may +live beside them in `datascripts/`. Every file opens with its **Script Directives**, `//` in TypeScript and `--` in Lua: diff --git a/src/cmd/packages/build.rs b/src/cmd/packages/build.rs index 2f9b019..7c4ba5e 100644 --- a/src/cmd/packages/build.rs +++ b/src/cmd/packages/build.rs @@ -21,8 +21,8 @@ //! `datascripts/generated/base-snapshot.json` must already exist, or the build fails fast with //! the exact `lyracore-importer --spell-snapshot` command to build one, rather than letting every //! Datascript fail with the same confusing "cannot read" one at a time. -//! 5. Every enabled Package with a `datascripts/src//` folder runs each `.ts` file there, -//! in name order, as its own `bun run` SUBPROCESS — never imported. The library hashes +//! 5. Every enabled Package runs the `.ts` files in `datascripts/src//`, then in its own +//! `datascripts/` directory, in name order, each as a `bun run` subprocess. The library hashes //! `Bun.main`, the running process's own entry script, into the artifact's `source_hash`; //! importing the script into one host process instead would hash the host, not the script. The //! first script to fail stops the build: later scripts, and later Packages, never run. @@ -73,6 +73,7 @@ //! Bun is needed HERE and nowhere else. An Operator applying a prebuilt Package Delta runs no part //! of this, which is why `doctor`'s Bun check is a warning rather than a launch blocker. +use std::io::Write as _; use std::path::{Path, PathBuf}; use crate::cmd::doctor::{self, BunVersionCheck}; @@ -141,38 +142,78 @@ fn verify_bun(runner: &dyn ProcessRunner) -> Result<()> { // ---- Datascript emission and validation ---- -/// Enabled Packages carrying a Datascript, in the order step 5 runs them: the folder-name sort, -/// same rule `packages/`'s own directory listing already uses to mean "enabled". -/// -/// A Package's Datascripts live under `datascripts/src//`, named after the Package folder -/// under `packages/` — not the other way around, so a Datascript for a disabled or removed Package -/// simply has no Package to match and never runs. +/// Enabled Packages with entry scripts in either supported Datascript location. fn packages_with_datascripts(project: &ProjectLayout) -> Result> { let packages_dir = project.packages_dir(); if !packages_dir.is_dir() { return Ok(Vec::new()); } - let mut names: Vec = std::fs::read_dir(&packages_dir)? - .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().is_dir()) - .filter_map(|entry| entry.file_name().into_string().ok()) - .filter(|name| project.datascripts_src_dir().join(name).is_dir()) - .collect(); + let mut names = Vec::new(); + for entry in std::fs::read_dir(packages_dir)? { + let entry = entry?; + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if entry.path().is_dir() && !datascripts_of(project, &name)?.is_empty() { + names.push(name); + } + } names.sort(); Ok(names) } -/// One Package's Datascripts, in the deterministic order step 5 runs them: file-name sort. +/// Legacy scripts run first, then Package-local scripts. Each directory sorts by file name. fn datascripts_of(project: &ProjectLayout, package: &str) -> Result> { - let dir = project.datascripts_src_dir().join(package); - let mut scripts: Vec = std::fs::read_dir(&dir)? - .filter_map(|entry| entry.ok().map(|e| e.path())) - .filter(|path| path.extension().is_some_and(|ext| ext == "ts")) - .collect(); - scripts.sort(); + let mut scripts = Vec::new(); + for dir in [ + project.datascripts_src_dir().join(package), + project.packages_dir().join(package).join("datascripts"), + ] { + if !dir.is_dir() { + continue; + } + let mut entries = Vec::new(); + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_file() && path.extension().is_some_and(|ext| ext == "ts") { + entries.push(path); + } + } + entries.sort(); + scripts.extend(entries); + } Ok(scripts) } +/// Extend Core's compiler options and includes with the enabled Package-local entry scripts. +/// The temporary file stays in `datascripts/` so TypeScript finds its installed type packages. +fn typecheck_config( + project: &ProjectLayout, + packages: &[String], +) -> Result> { + let mut files = Vec::new(); + for package in packages { + files.extend( + datascripts_of(project, package)? + .into_iter() + .filter(|path| path.starts_with(project.packages_dir())), + ); + } + if files.is_empty() { + return Ok(None); + } + let mut config = tempfile::Builder::new() + .prefix(".packages-tsconfig-") + .suffix(".json") + .tempfile_in(project.datascripts_dir())?; + let value = serde_json::json!({ + "extends": project.datascripts_dir().join("tsconfig.json"), + "files": files, + }); + config.write_all(value.to_string().as_bytes())?; + Ok(Some(config)) +} + /// The Base Snapshot must exist before any Datascript runs — `lib/index.ts` throws a "cannot read" /// error naming the same command if it does not, one script at a time. Failing here, once, before /// the first `bun run`, gives the author the fix instead of a wall of identical errors. @@ -361,21 +402,26 @@ pub fn run(project: &ProjectLayout, runner: &dyn ProcessRunner) -> Result<()> { )) })?; + let datascript_packages = packages_with_datascripts(project)?; + let config = typecheck_config(project, &datascript_packages)?; + let mut typecheck = typecheck_command(project); + if let Some(config) = &config { + typecheck = typecheck + .arg("--project") + .arg(config.path().to_string_lossy().to_string()); + } println!("typechecking the Datascripts against the regenerated typings"); - runner - .run_streaming(&typecheck_command(project)) - .map_err(|e| { - Error::Process(format!( - "the Datascripts do not typecheck against the current Module schema. This is the \ + runner.run_streaming(&typecheck).map_err(|e| { + Error::Process(format!( + "the Datascripts do not typecheck against the current Module schema. This is the \ gate doing its job — the errors above name the file, line and column to fix. A \ column the Module renamed, retyped or removed shows up there.\n ({e})" - )) - })?; + )) + })?; println!(); println!("Datascripts typecheck against the current Module schema."); - let datascript_packages = packages_with_datascripts(project)?; let script_packages = script::packages_with_scripts(project)?; for path in script::remove_artifacts_without_sources(project)? { println!( @@ -461,6 +507,12 @@ mod tests { std::fs::write(src_dir.join(format!("{script}.ts")), "// a Datascript\n").unwrap(); } + fn with_local_datascript(project: &ProjectLayout, package: &str, script: &str) { + let dir = project.packages_dir().join(package).join("datascripts"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("{script}.ts")), "// a Datascript\n").unwrap(); + } + fn with_base_snapshot(project: &ProjectLayout) { std::fs::create_dir_all(project.datascript_types_dir()).unwrap(); std::fs::write(project.base_snapshot_file(), "{}\n").unwrap(); @@ -709,6 +761,92 @@ mod tests { // ---- Datascript emission and validation ---- + #[test] + fn enabled_packages_run_legacy_and_local_datascripts_in_order() { + let tmp = TempDir::new().unwrap(); + let project = checkout(&tmp); + with_datascript(&project, "greeter", "legacy"); + with_local_datascript(&project, "greeter", "welcome"); + with_local_datascript(&project, "disabled", "unused"); + std::fs::create_dir_all(project.packages_disabled_dir()).unwrap(); + std::fs::rename( + project.packages_dir().join("disabled"), + project.packages_disabled_dir().join("disabled"), + ) + .unwrap(); + with_base_snapshot(&project); + with_generated_artifact(&project, "greeter"); + let stack = FakeStack::new(); + + run(&project, &stack.runner()).unwrap(); + + let scripts: Vec = stack + .rendered() + .into_iter() + .filter(|call| call.starts_with("bun run")) + .collect(); + assert_eq!(scripts.len(), 2, "{scripts:?}"); + assert!( + scripts[0].ends_with("datascripts/src/greeter/legacy.ts"), + "{scripts:?}" + ); + assert!( + scripts[1].ends_with("packages/greeter/datascripts/welcome.ts"), + "{scripts:?}" + ); + assert!(project + .packages_dir() + .join("greeter/data/.generated/spell.identity") + .is_file()); + } + + #[test] + fn local_datascripts_require_a_base_snapshot_before_emission() { + let tmp = TempDir::new().unwrap(); + let project = checkout(&tmp); + with_local_datascript(&project, "greeter", "welcome"); + let stack = FakeStack::new(); + + let error = run(&project, &stack.runner()).unwrap_err(); + + assert!(error.to_string().contains("no Base Snapshot"), "{error}"); + assert!(stack + .rendered() + .iter() + .all(|call| !call.starts_with("bun run"))); + } + + #[test] + fn a_local_datascript_type_error_stops_before_emission() { + let tmp = TempDir::new().unwrap(); + let project = checkout(&tmp); + with_local_datascript(&project, "greeter", "welcome"); + with_base_snapshot(&project); + let stack = FakeStack::new().fail_on("tsc", "welcome.ts does not typecheck"); + + let error = run(&project, &stack.runner()).unwrap_err(); + + assert!(error.to_string().contains("do not typecheck"), "{error}"); + let calls = stack.rendered(); + assert!( + calls.iter().any(|call| call.contains("--project")), + "{calls:?}" + ); + assert!( + calls.iter().all(|call| !call.starts_with("bun run")), + "{calls:?}" + ); + assert!(std::fs::read_dir(project.datascripts_dir()) + .unwrap() + .all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".packages-tsconfig-") + })); + } + #[test] fn a_build_typegens_installs_typechecks_emits_then_validates_in_that_order() { let tmp = TempDir::new().unwrap(); diff --git a/src/cmd/packages/git.rs b/src/cmd/packages/git.rs index 4c609ed..e4b2a94 100644 --- a/src/cmd/packages/git.rs +++ b/src/cmd/packages/git.rs @@ -945,7 +945,7 @@ mod tests { let error = super::super::add(&project, &stack.runner(), &Answer("yes"), URL, true).unwrap_err(); - assert!(error.to_string().contains("neither src/"), "{error}"); + assert!(error.to_string().contains("not a Package"), "{error}"); assert!(!project.packages_dir().join("greeter").exists(), "{error}"); } diff --git a/src/cmd/packages/identity.rs b/src/cmd/packages/identity.rs index 609e08a..1d40dcc 100644 --- a/src/cmd/packages/identity.rs +++ b/src/cmd/packages/identity.rs @@ -83,7 +83,7 @@ impl Input { /// What changed, and where to look — the sentence a staleness refusal names this input with. pub fn description(self) -> &'static str { match self { - Input::Source => "its Datascript source under datascripts/src//", + Input::Source => "its Datascript source under packages//datascripts/ or datascripts/src//", Input::Generated => { "the generated Module schema/typings in datascripts/generated/ (a schema change)" } @@ -269,9 +269,8 @@ fn hash_file(path: &Path) -> Result { } /// The enabled Package folder an artifact was discovered inside — the top component of its path -/// relative to `packages/`. This is a Package's FOLDER name, which is what selects its Datascript -/// source under `datascripts/src//`; it need not equal the artifact's own declared -/// `package` identity. +/// relative to `packages/`. Its folder name selects both Datascript source locations; it need not +/// equal the artifact's declared `package` identity. pub fn package_dir(project: &ProjectLayout, artifact_path: &Path) -> Result { let root = project.packages_dir(); let relative = artifact_path.strip_prefix(&root).map_err(|_| { @@ -309,7 +308,18 @@ pub fn compute( })?; let datascripts = project.datascripts_dir(); - let source_hash = hash_tree(&project.datascripts_src_dir().join(name), None)?; + let legacy_hash = hash_tree(&project.datascripts_src_dir().join(name), None)?; + let local = package_dir.join("datascripts"); + let source_hash = if local.is_dir() { + let mut hasher = Sha256::new(); + hasher.update(legacy_hash.as_bytes()); + hasher.update([0u8]); + hasher.update(hash_tree(&local, None)?.as_bytes()); + format!("sha256-datascript-sources-v1:{:x}", hasher.finalize()) + } else { + // Preserve existing Build Identities when the Package has no local Datascript directory. + legacy_hash + }; let generated_hash = hash_tree( &project.datascript_types_dir(), Some(ProjectLayout::BASE_SNAPSHOT_FILE), @@ -713,6 +723,24 @@ mod tests { assert_eq!(changed, vec![Input::Source], "{changed:?}"); } + #[test] + fn changed_package_local_datascript_sources_are_named() { + let tmp = TempDir::new().unwrap(); + let project = checkout(&tmp); + let dir = package_dir_of(&project).join("datascripts"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("welcome.ts"), "// original\n").unwrap(); + let (recorded, _) = compute(&project, &package_dir_of(&project), HASH_A).unwrap(); + + std::fs::write(dir.join("welcome.ts"), "// changed\n").unwrap(); + let (current, available) = compute(&project, &package_dir_of(&project), HASH_A).unwrap(); + + assert_eq!( + recorded.changed_against(¤t, available), + vec![Input::Source] + ); + } + #[test] fn a_changed_generated_typing_is_named() { let tmp = TempDir::new().unwrap(); diff --git a/src/cmd/packages/mod.rs b/src/cmd/packages/mod.rs index 55a430b..1f80cd8 100644 --- a/src/cmd/packages/mod.rs +++ b/src/cmd/packages/mod.rs @@ -275,19 +275,16 @@ fn check_not_tracked( ))) } -/// The shapes `module/build.rs` accepts. -/// -/// A `client/` with no `src/` is a legal client-only Package. A `src/` without `src/mod.rs` is not: -/// the build panics rather than silently skipping it, so this refuses it here where the operator -/// still has the folder in front of them. Neither directory means the folder is not a Package at -/// all — the shape the build only warns about, because by then it is too late to ask. +/// A Package carries Rust, client files, generated data or authoring scripts. +/// If it carries Rust, `src/mod.rs` is its required root. pub fn validate_shape(source: &Path) -> Result<()> { let has_src = source.join("src").is_dir(); - let has_client = source.join("client").is_dir(); - if !has_src && !has_client { + let has_other_half = ["client", "data", "scripts", "datascripts"] + .iter() + .any(|name| source.join(name).is_dir()); + if !has_src && !has_other_half { return Err(Error::Usage(format!( - "{} is not a Package: it has neither src/ (Rust compiled into the module) nor client/ \ - (addons and client overrides). One of the two is required.", + "{} is not a Package: it needs src/, client/, data/, scripts/ or datascripts/.", source.display() ))); } @@ -1168,7 +1165,17 @@ pub(super) mod tests { let empty = tmp.path().join("empty"); std::fs::create_dir_all(&empty).unwrap(); let error = validate_shape(&empty).unwrap_err(); - assert!(error.to_string().contains("neither src/"), "{error}"); + assert!(error.to_string().contains("not a Package"), "{error}"); + } + + #[test] + fn authored_or_generated_data_without_rust_is_a_package() { + let tmp = TempDir::new().unwrap(); + for half in ["data", "scripts", "datascripts"] { + let package = tmp.path().join(half); + std::fs::create_dir_all(package.join(half)).unwrap(); + validate_shape(&package).unwrap(); + } } // ---- the install ---- diff --git a/src/cmd/packages/review.rs b/src/cmd/packages/review.rs index 09f2709..c0851ee 100644 --- a/src/cmd/packages/review.rs +++ b/src/cmd/packages/review.rs @@ -71,6 +71,8 @@ pub struct TrustReview { pub client_overrides: usize, /// Runtime Scripts this Package would run, by artifact name or pre-build source file. pub runtime_scripts: Vec, + /// Package-local Datascripts run by the author during `packages build`. + pub datascripts: Vec, /// Physical generated Package Delta artifacts. pub package_delta_count: usize, /// Generated Package Deltas grouped by Import Family. @@ -149,6 +151,17 @@ impl TrustReview { .collect(); source_scripts.sort(); + review.datascripts = entries + .iter() + .filter(|entry| { + entry.kind == EntryKind::File + && entry.relative.parent() == Some(Path::new("datascripts")) + && entry.relative.extension().is_some_and(|ext| ext == "ts") + }) + .map(|entry| entry.relative.to_string_lossy().into_owned()) + .collect(); + review.datascripts.sort(); + let generated = artifact::summarize_package_artifacts(package_dir)?; review.runtime_scripts = if generated.has_script_artifact { generated.runtime_scripts @@ -200,9 +213,7 @@ impl TrustReview { self.character_owned.sort(); } - /// Does this Package register anything at all? A folder with neither Rust nor client content - /// is rejected before the review runs, so a review with nothing in it means Rust that - /// registers nothing — which is still trusted code. + /// Whether the inventory contains no registered behavior or authoring scripts. pub fn registers_nothing(&self) -> bool { self.tables.is_empty() && self.reducers.is_empty() @@ -212,6 +223,7 @@ impl TrustReview { && self.addons.is_empty() && self.client_overrides == 0 && self.runtime_scripts.is_empty() + && self.datascripts.is_empty() && self.package_delta_count == 0 } @@ -250,6 +262,14 @@ impl TrustReview { self.runtime_scripts.len(), &self.runtime_scripts, )); + out.push_str(&row( + "datascripts", + self.datascripts.len(), + &self.datascripts, + )); + if !self.datascripts.is_empty() { + out.push_str(" Datascripts are trusted code run on your machine by packages build.\n"); + } out.push_str(&format!( " trusted Rust {} file(s), {} line(s)\n", self.rust_files, self.rust_lines @@ -286,6 +306,7 @@ impl TrustReview { ("addons", self.addons.len()), ("client overrides", self.client_overrides), ("runtime scripts", self.runtime_scripts.len()), + ("datascripts", self.datascripts.len()), ] { if count > 0 { parts.push(format!("{count} {label}")); @@ -801,6 +822,19 @@ mod tests { ); } + #[test] + fn package_local_datascripts_are_named_before_installation() { + let tmp = package(&[ + ("datascripts/welcome.ts", "// a Datascript\n"), + ("datascripts/README.md", "not a script\n"), + ]); + let review = TrustReview::scan(tmp.path()).unwrap(); + assert_eq!(review.datascripts, ["datascripts/welcome.ts"]); + assert!(!review.registers_nothing()); + assert!(review.kinds_summary().contains("1 datascripts")); + assert!(review.render(tmp.path()).contains("run on your machine")); + } + #[test] fn a_prebuilt_script_artifact_is_counted_without_script_sources() { let tmp = package(&[(