diff --git a/Cargo.lock b/Cargo.lock index a22e5bcdf2..6d86fab5c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3011,7 +3011,10 @@ name = "pkg" version = "0.1.0" dependencies = [ "flate2", + "tar", + "toyos", "toyos-manifest", + "toyos-sha2", "toyos-sha2-hw", "toyos-tmpdir", ] diff --git a/build/hostedclang.rs b/build/hostedclang.rs index 13969397e8..539dd8aafb 100644 --- a/build/hostedclang.rs +++ b/build/hostedclang.rs @@ -22,8 +22,13 @@ //! `hosted-clang//` in the store holds `bin/clang`, `bin/ld.lld` and //! clang's resource headers in `lib/clang//include`, where clang looks //! beside itself; once its [`SOURCE`] file exists it is read-only. +//! +//! **What a ToyOS machine installs is a package of it** ([`HostedClang::package`]): +//! `clang-v-toyos-x86_64.tar.gz` beside the `SHA256SUMS` that verifies +//! it, which `/system/bin/pkg` installs as `/apps/clang`. use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; @@ -92,11 +97,17 @@ const BINARIES: [(&str, &str); 2] = [("clang", "clang"), ("lld", "ld.lld")]; /// A ToyOS-hosted clang and LLD, held in use for as long as this lives. pub struct HostedClang { pub dir: PathBuf, + /// The C sysroot it was built against, which is the one it compiles + /// against. + pub c: CSysroot, + /// The licence of the LLVM it was built from: clang's, LLD's, libc++'s + /// and the resource headers'. + licence: PathBuf, _using: Guard, } -/// `cargo run -- --hosted-clang`: make this tree's, and name each binary with -/// its size. +/// `cargo run -- --hosted-clang`: make this tree's, name each binary with its +/// size, and package it in `target/hosted-clang`. pub fn dispatch(root: &Path) { let hosted = ensure(root); for (_, kept) in BINARIES { @@ -104,6 +115,111 @@ pub fn dispatch(root: &Path) { let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); println!("{} ({size} bytes)", binary.display()); } + let packaged = hosted.package(&root.join("target/hosted-clang")); + println!("{}, verified by {}", packaged.archive.display(), packaged.sums.display()); +} + +/// The package in its directory: the archive, and `SHA256SUMS` beside it. +pub struct Packaged { + pub archive: PathBuf, + pub sums: PathBuf, +} + +/// The package's name: the directory it is under `/apps`, and the program in +/// its `bin` that a launch starts. +const PACKAGE: &str = "clang"; + +/// The links of the package's `bin`: the driver's other names, which it reads +/// its language from. +const LINKS: [(&str, &str); 2] = [("cc", "clang"), ("clang++", "clang")]; + +impl HostedClang { + /// Write the package and its `SHA256SUMS` into `into`, and answer where. + /// **A function of the product and the sysroot**: every entry in one + /// order, with no owner or time, so one toolchain packs to one digest. + /// + /// Beside the binaries it holds clang's resource headers where clang looks + /// for them, `lib/clang`, and the sysroot's `include` and `lib` at + /// `sysroot`, which the configuration file clang reads beside itself names + /// (`.cfg`, `clang/docs/UsersManual.rst`, "Configuration files"), + /// so a compile there needs no `--sysroot`. + pub fn package(&self, into: &Path) -> Packaged { + pack(&self.dir, &self.c.dir, &self.licence, &said_version(&self.c.clang).0, into) + } +} + +/// [`HostedClang::package`] of the product at `product` and the C sysroot at +/// `sysroot`, LLVM `version` under the licence at `licence`. +fn pack(product: &Path, sysroot: &Path, licence: &Path, version: &str, into: &Path) -> Packaged { + let name = format!("{PACKAGE}-v{version}-toyos-{}.tar.gz", TARGET.name()); + fs::create_dir_all(into).unwrap_or_else(|e| panic!("create {}: {e}", into.display())); + let archive = into.join(&name); + let file = fs::File::create(&archive).unwrap_or_else(|e| panic!("create {}: {e}", archive.display())); + let gzip = flate2::write::GzEncoder::new(std::io::BufWriter::new(file), flate2::Compression::default()); + let mut tar = tar::Builder::new(gzip); + let mut append = |path: &str, mode: u32, data: &[u8]| { + let mut header = tar::Header::new_ustar(); + header.set_size(data.len() as u64); + header.set_mode(mode); + header.set_mtime(0); + tar.append_data(&mut header, format!("{PACKAGE}/{path}"), data).unwrap_or_else(|e| panic!("pack {path}: {e}")); + }; + append("LICENSE.TXT", 0o644, &read(licence)); + for (_, kept) in BINARIES { + append(&format!("bin/{kept}"), 0o755, &read(&product.join("bin").join(kept))); + } + append(&format!("bin/{}.cfg", TARGET.userland()), 0o644, b"--sysroot=/../sysroot\n"); + for (rel, from) in files(&product.join("lib/clang")) { + append(&format!("lib/clang/{rel}"), 0o644, &read(&from)); + } + for part in ["include", "lib"] { + for (rel, from) in files(&sysroot.join(part)) { + append(&format!("sysroot/{part}/{rel}"), 0o644, &read(&from)); + } + } + for (link, target) in LINKS { + let mut header = tar::Header::new_ustar(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_mtime(0); + tar.append_link(&mut header, format!("{PACKAGE}/bin/{link}"), target) + .unwrap_or_else(|e| panic!("pack bin/{link}: {e}")); + } + let packed = tar.into_inner().and_then(|gzip| gzip.finish()).and_then(|mut file| file.flush()); + packed.unwrap_or_else(|e| panic!("{}: {e}", archive.display())); + + let digest: String = toyos_sha2_hw::sha256_digest(read(&archive)).iter().map(|b| format!("{b:02x}")).collect(); + let sums = into.join("SHA256SUMS"); + fs::write(&sums, format!("{digest} {name}\n")).unwrap_or_else(|e| panic!("write {}: {e}", sums.display())); + Packaged { archive, sums } +} + +fn read(path: &Path) -> Vec { + fs::read(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +/// Every file under `dir`, by its path relative to `dir`, in sorted order. A +/// link in a product is a defect of its maker, refused here rather than packed +/// as the file it points at. +fn files(dir: &Path) -> Vec<(String, PathBuf)> { + let mut out = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(at) = pending.pop() { + for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); + let kind = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())).file_type(); + assert!(!kind.is_symlink(), "{} is a link, which no product of the store holds", path.display()); + if kind.is_dir() { + pending.push(path); + } else { + let rel = path.strip_prefix(dir).expect("under the walked directory"); + out.push((rel.to_string_lossy().into_owned(), path)); + } + } + } + out.sort(); + out } /// This tree's, made if nobody on this host has made it. @@ -124,7 +240,7 @@ fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) let dir = Keyed::HostedClang.store(store).join(&key); let make = || place(fork, llvm, sysroot, ninja, &key, &dir); let using = keystore::made(store, Keyed::HostedClang, &key, || defect(&dir), make); - HostedClang { dir, _using: using } + HostedClang { dir, c: CSysroot::of(sysroot, TARGET), licence: llvm.join("src/llvm/LICENSE.TXT"), _using: using } } /// The key the store product at `dir` is filed under. @@ -213,7 +329,7 @@ fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, native.extend(crate::llvm::NO_HOST_LIBRARIES.iter().map(|name| format!("-D{name}=OFF"))); let mut definitions = options(); definitions.extend(crate::llvm::stamp_options(fork)); - definitions.push(("LLVM_VERSION_SUFFIX".to_string(), version_suffix(&c.clang))); + definitions.push(("LLVM_VERSION_SUFFIX".to_string(), said_version(&c.clang).1)); definitions.extend([ ("CMAKE_TOOLCHAIN_FILE".to_string(), path(&c.cmake_toolchain())), ("CMAKE_C_COMPILER".to_string(), path(&c.clang)), @@ -233,10 +349,11 @@ fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, run(make, "build", ninja); } -/// What the host's clang, the sysroot's, says it is beyond LLVM's version: -/// bootstrap's `LLVM_VERSION_SUFFIX`. Every object a clang compiles carries -/// what it says in its `.comment`, so this one says it too. -fn version_suffix(clang: &Path) -> String { +/// What the host's clang, the sysroot's, says its version is: LLVM's release, +/// which the package is named by, and what it says beyond it, bootstrap's +/// `LLVM_VERSION_SUFFIX`. Every object a clang compiles carries the whole in +/// its `.comment`, so the ToyOS-hosted one is built to say it too. +fn said_version(clang: &Path) -> (String, String) { let mut command = Command::new(clang); command.arg("--version"); crate::llvm::clear(&mut command); @@ -244,14 +361,15 @@ fn version_suffix(clang: &Path) -> String { assert!(out.status.success(), "{command:?} failed: {}", String::from_utf8_lossy(&out.stderr)); let said = String::from_utf8_lossy(&out.stdout); let first = said.lines().next().unwrap_or_default(); - suffix(first).unwrap_or_else(|| panic!("{} says no version: {said}", clang.display())) + split_version(first).unwrap_or_else(|| panic!("{} says no version: {said}", clang.display())) } -/// The suffix of the version `line`, clang's first line of `--version`, names: -/// what follows its first `-`, and none without one. -fn suffix(line: &str) -> Option { +/// The version `line`, clang's first line of `--version`, names, split at its +/// first `-`: the suffix is empty without one. +fn split_version(line: &str) -> Option<(String, String)> { let version = line.strip_prefix("clang version ")?.split_whitespace().next()?; - Some(version.find('-').map_or(String::new(), |at| version[at..].to_string())) + let at = version.find('-').unwrap_or(version.len()); + Some((version[..at].to_string(), version[at..].to_string())) } /// Run `command` seeing nothing of this process's environment but what the @@ -299,11 +417,73 @@ mod tests { /// **The suffix is what follows the version's first `-`**, none without /// one, and a line that names no clang version names none. #[test] - fn the_suffix_is_what_follows_the_version() { + fn the_version_splits_at_its_first_dash() { let rev = "(https://github.com/ToyOSOrg/llvm-project.git ceaf0fbb8440)"; - assert_eq!(suffix(&format!("clang version 22.1.8-rust-dev {rev}")).as_deref(), Some("-rust-dev")); - assert_eq!(suffix(&format!("clang version 22.1.8 {rev}")).as_deref(), Some("")); - assert_eq!(suffix("Apple clang version 17.0.0 (clang-1700.0.13.5)"), None); + let split = split_version; + assert_eq!(split(&format!("clang version 22.1.8-rust-dev {rev}")), Some(("22.1.8".into(), "-rust-dev".into()))); + assert_eq!(split(&format!("clang version 22.1.8 {rev}")), Some(("22.1.8".into(), String::new()))); + assert_eq!(split("Apple clang version 17.0.0 (clang-1700.0.13.5)"), None); + } + + /// **The package is the product and the sysroot under one `clang`, + /// POSIX ustar throughout**, with the driver's other names as links and + /// the configuration that names the sysroot, and one input packs to one + /// digest. + #[test] + fn the_package_is_the_product_and_the_sysroot_under_one_directory() { + let scratch = TempDir::new("hosted-clang-package"); + let write = |rel: &str| { + let path = scratch.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, rel).unwrap(); + }; + for rel in ["product/bin/clang", "product/bin/ld.lld", "product/lib/clang/22/include/stddef.h", "product/SOURCE"] { + write(rel); + } + for rel in ["c/include/stdio.h", "c/include/c++/v1/vector", "c/lib/libtoyos_c.a", "c/toolchain.cmake", "LICENSE.TXT"] { + write(rel); + } + let (product, c) = (scratch.join("product"), scratch.join("c")); + let licence = scratch.join("LICENSE.TXT"); + let first = pack(&product, &c, &licence, "22.1.8", &scratch.join("one")); + assert_eq!(first.archive.file_name().unwrap(), "clang-v22.1.8-toyos-x86_64.tar.gz"); + + let gz = fs::File::open(&first.archive).unwrap(); + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(gz)); + let mut got = Vec::new(); + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + assert_eq!(&entry.header().as_bytes()[257..265], b"ustar\x0000", "a header that is not POSIX ustar"); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + let held = match entry.link_name().unwrap() { + Some(target) => format!("-> {}", target.display()), + None => { + let mut text = String::new(); + std::io::Read::read_to_string(&mut entry, &mut text).unwrap(); + text + } + }; + got.push((path, held)); + } + let want = [ + ("clang/LICENSE.TXT", "LICENSE.TXT"), + ("clang/bin/clang", "product/bin/clang"), + ("clang/bin/ld.lld", "product/bin/ld.lld"), + ("clang/bin/x86_64-unknown-toyos.cfg", "--sysroot=/../sysroot\n"), + ("clang/lib/clang/22/include/stddef.h", "product/lib/clang/22/include/stddef.h"), + ("clang/sysroot/include/c++/v1/vector", "c/include/c++/v1/vector"), + ("clang/sysroot/include/stdio.h", "c/include/stdio.h"), + ("clang/sysroot/lib/libtoyos_c.a", "c/lib/libtoyos_c.a"), + ("clang/bin/cc", "-> clang"), + ("clang/bin/clang++", "-> clang"), + ]; + let want: Vec<(String, String)> = want.iter().map(|(p, h)| (p.to_string(), h.to_string())).collect(); + assert_eq!(got, want); + + let digest = |sums: &Path| fs::read_to_string(sums).unwrap(); + let again = pack(&product, &c, &licence, "22.1.8", &scratch.join("two")); + assert_eq!(digest(&first.sums), digest(&again.sums), "one input packed to two digests"); + assert!(digest(&first.sums).ends_with(" clang-v22.1.8-toyos-x86_64.tar.gz\n")); } /// **A product missing a binary or clang's headers is not whole**, nor one diff --git a/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md b/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md index e49d25894e..70b9c72aaa 100644 --- a/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md +++ b/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md @@ -84,7 +84,10 @@ The storage track's users and mount-protocol stages do not block this one. binaries a launch starts and nothing else, so a writable `/apps` cannot name a device or a right. The stage-then-commit above amends it: `pkg` writes `/apps//` in place today, its `manifest.toml` last - (`system/pkg/src/main.rs`). + (`system/pkg/src/main.rs`). It streams an archive of any size that fits + the room `/apps`'s volume has, carries links that resolve inside the + package, and launches `/` or `/bin/`; + `hosted_clang_hello` installs the toolchain's `clang` package with it. 2. The HTTPS fetch: TLS client under `pkg`, the GitHub redirect, the signed repository's files and the archives its targets names. This is the internet-client track's last diff --git a/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md b/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md index dfc8adadaa..e190d72944 100644 --- a/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md +++ b/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md @@ -18,6 +18,16 @@ IOMMU's superpages, the kernel's own mappings. One paging design serves both, on x86-64 and on the ARM64 the tree is kept portable for (`issues/toyos-runs-on-arm64.md`). +**A large program spawned from a large parent is refused.** A program a file +server holds is spawned from a copy in one memory object, and `SYS_SHM_CREATE` +allocates that object physically contiguous (`kernel/src/object/shm.rs`, +`alloc_contiguous`): about 148 MB for the hosted clang. A small std parent +spawns `ld.lld` that way, but clang's driver, starting `cc1` and `ld.lld` from +`/apps/clang`, is refused `ResourceExhausted` (`ENOMEM` in libc), so +`hosted_clang_hello` compiles and cannot link. That the contiguous allocation +is the one refused is inferred: the errno is measured, the allocation is +not. Removing contiguous user memory is the exit's; this row is its check. + **The constraint that makes this a track, measured on the T14** (run 29 readback, `PMM: 100/16038MB used` at 11 s with four user processes and three kernel threads live; the same record's rows: stack 16 pages held, init-tls 5, diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index eb7dab5e8f..b06f59d84e 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -64,73 +64,84 @@ An owner is an issue under `issues/`, or this track. | 2 | C reaches every file a process holds: descriptors (#975, #978) and served files through `toyos::files` (#988) | done | this track | — | | 3 | `pkg` installs a toolchain: a streamed archive bounded by its volume's room, and links. #970 installs the clang package (83.8 MB archive, 279 MB installed) on a 1 GiB DATA in QEMU | in progress | `a-package-is-a-directory-under-apps-and-the-installer-is-a-program` | — | | 4 | Room for a toolchain on DATA, durable. DATA serves `/apps`, `/config`, `/home` and `/state`; the full toolchain is about 1 GB | in progress | `storage-is-layers-and-a-role-is-a-filesystem` | — | -| 5 | **M2 exit**: a `pkg`-installed clang compiles, links and runs `hello.c` (#970's guest row), as `hosted_clang_hello` already does with a clang staged on ROOT | in progress | this track, `toyos-has-no-c-compiler-inside-it` | 2–4 | +| 5 | User memory without physical contiguity. clang's driver, spawning `cc1` from `/apps`, is refused `ENOMEM` (measured), where a small parent spawns the same images. The likely refusal is the image's memory object, which `SYS_SHM_CREATE` allocates physically contiguous (`kernel/src/object/shm.rs`), about 148 MB for clang; that it is this allocation and not the spawn's own is inferred. Memory stages 1–3 remove contiguous user memory | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count` | engineering: item 13's design, stages 1–3 | +| 6 | **M2 exit**: a `pkg`-installed clang compiles, links and runs `hello.c` (#970's guest row, which compiles and stops at the link on 5), as `hosted_clang_hello` did with a clang staged on ROOT | in progress | this track, `toyos-has-no-c-compiler-inside-it` | 2–5 | **M3: an LLVM-backed rustc.** | # | Item | Status | Owner | Blocked on | |---|---|---|---|---| -| 6 | Threads mature enough for LLVM | not started | `std-and-libc-drop-the-answer-thread-join-gives` | — | -| 7 | Concurrent reads and writes of one file: `pread` and `pwrite`. Without them LLVM reads a file slice with `lseek` and `read`, which moves an offset other threads share | not started | `libc-has-no-pread-or-pwrite` | — | -| 8 | LLVM, clang and lld built by bootstrap for a ToyOS host, and the libc names they call | not started | `bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host`, `rustc-llvm-cannot-build-for-a-toyos-host`, `libc-lacks-names-llvm-for-a-toyos-host-calls`, `a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine` | — | -| 9 | A std binary links the C++ runtime | not started | `a-rust-std-binary-cannot-link-the-cxx-runtime` | 8 | -| 10 | A package ships its own libraries (`librustc_driver`); moot for M2's static PIEs | not started | `a-package-cannot-ship-its-own-libraries` | 3 | -| 11 | **M3 exit**: the hosted rustc, which runs LLVM on several threads, builds and runs a program in the guest, linking with the guest's `ld.lld` | not started | `nothing-builds-the-toyos-hosted-rustc`, `the-hosted-rustc-names-a-linker-toyos-does-not-have`, `the-hosted-rustcs-stage2-carries-seventeen-proc-macro-libraries-no-image-needs` | 5–10 | +| 7 | Threads mature enough for LLVM | not started | `std-and-libc-drop-the-answer-thread-join-gives` | — | +| 8 | Concurrent reads and writes of one file: `pread` and `pwrite`. Without them LLVM reads a file slice with `lseek` and `read`, which moves an offset other threads share | not started | `libc-has-no-pread-or-pwrite` | — | +| 9 | LLVM, clang and lld built by bootstrap for a ToyOS host, and the libc names they call | not started | `bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host`, `rustc-llvm-cannot-build-for-a-toyos-host`, `libc-lacks-names-llvm-for-a-toyos-host-calls`, `a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine` | — | +| 10 | A std binary links the C++ runtime | not started | `a-rust-std-binary-cannot-link-the-cxx-runtime` | 9 | +| 11 | A package ships its own libraries (`librustc_driver`); moot for M2's static PIEs | not started | `a-package-cannot-ship-its-own-libraries` | 3 | +| 12 | **M3 exit**: the hosted rustc, which runs LLVM on several threads, builds and runs a program in the guest, linking with the guest's `ld.lld` | not started | `nothing-builds-the-toyos-hosted-rustc`, `the-hosted-rustc-names-a-linker-toyos-does-not-have`, `the-hosted-rustcs-stage2-carries-seventeen-proc-macro-libraries-no-image-needs` | 6–11 | **M4: cargo, rustc and clang build ToyOS in the guest, byte-identical.** | # | Item | Status | Owner | Blocked on | |---|---|---|---|---| -| 12 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count`, `one-mapping-inventory-nobody-can-read`, `anonymous-mmap-is-not-demand-paged`, `nothing-charges-kernel-memory-to-a-process` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | -| 13 | Record locks the file server serves, and std's and cargo's locks on them | direction set | `std-file-lock-answers-ok-and-locks-nothing`, `libc-refuses-what-toyos-cannot-yet-answer` | engineering: the lock design, part of 12's | -| 14 | An unlinked open file lives until its last close or unmap | direction set | `an-unlinked-open-file-answers-stale` | 12 for the mapping | -| 15 | An exclusive create on `/tmp` (`O_CREAT\|O_EXCL`, `create_new`) | not started | `create-new-on-a-kernel-path-is-not-exclusive` | — | -| 16 | libc tells the truth: `sigaction` refuses what it does not keep, `st_nlink` is 1, and the user and group ids are the process's own | not started | `libc-sigaction-answers-success-and-keeps-nothing`, `libc-answers-st-nlink-0`, `libc-answers-uid-0-for-every-process` | — | -| 17 | libc gains `select`, `setvbuf` and `socketpair`. A descriptor is a small table number since #978, so `FD_SETSIZE` holds | not started | this track | — | -| 18 | The `libc` crate gains a ToyOS module, checked by `ctest` against `sdk/libc`'s headers | not started | this track | 17 | -| 19 | Children: `SIGCHLD`, sessions, a quit notice and Ctrl+C, the process track's stages 4–7 | not started | `a-childs-end-is-an-event-and-a-parent-takes-its-children-down` | — | -| 20 | Sockets shared by their holders and inherited by a child, which curl, libssh2 and every inheriting server need | direction set | `a-stream-cannot-move-between-processes`, `libc-close-of-a-socket-is-unproven-on-a-connected-socket` | engineering: the sharing and recovery design, out of the tree, under review | -| 21 | cargo's eight C libraries cross-built: curl, libgit2, libssh2, OpenSSL, SQLite, nghttp2, zlib, blake3; five stopped on headers, types and functions libc lacked at `15625e0cb` | not started | this track | 17, 18 | -| 22 | cargo's TLS: OpenSSL through its Perl `Configure`, or curl's rustls backend on `ToyOSOrg/ring`. A build from vendored sources needs neither, and neither needs the `toyos-crypto` track | not started, choice open | this track, `the-internet-clients-work-unchanged` | 21; HTTPS on `the-lan-is-not-yet-production-grade` | -| 23 | cargo at run time: its SQLite and its locks, sockets and children | not started | this track, `sqlite-and-postgresql-run-on-toyos` | 13–16, 19–21 | -| 24 | The tools a build runs: Perl and Python started by spawn, brush as `sh`, uutils, make, CMake with libuv on spawn, awk, and a script started by its interpreter line | not started | `ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have`, `nothing-launches-a-script-by-its-interpreter-line` | 19 for the C tools | -| 25 | git in the guest | not started | this track | 21 | -| 26 | The toolchain as `pkg` packages, none of it in the image. #970 makes the clang package; how the rest splits, and how a self-host test's guest reaches them, is open | in progress | this track | 3, 10 | -| 27 | Storage durable and fast enough for an LLVM build tree, unmeasured | not started | `storage-is-layers-and-a-role-is-a-filesystem`, `an-fsync-of-a-served-file-syncs-its-whole-volume` | 4 | -| 28 | A build reproduces: an image's GUIDs from the commit, and one checkout building one kernel | not started | `an-images-guids-are-drawn-from-the-hosts-random-source`, `one-checkout-of-one-tree-builds-two-kernels` | — | -| 29 | **M4 exit**: the guest builds a userland program and then the kernel, and the hashes equal the host build's of the same commit | not started | this track | 11–28 | +| 13 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count`, `one-mapping-inventory-nobody-can-read`, `anonymous-mmap-is-not-demand-paged`, `nothing-charges-kernel-memory-to-a-process` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | +| 14 | Record locks the file server serves, and std's and cargo's locks on them | direction set | `std-file-lock-answers-ok-and-locks-nothing`, `libc-refuses-what-toyos-cannot-yet-answer` | engineering: the lock design, part of 13's | +| 15 | An unlinked open file lives until its last close or unmap | direction set | `an-unlinked-open-file-answers-stale` | 13 for the mapping | +| 16 | An exclusive create on `/tmp` (`O_CREAT\|O_EXCL`, `create_new`) | not started | `create-new-on-a-kernel-path-is-not-exclusive` | — | +| 17 | libc tells the truth: `sigaction` refuses what it does not keep, `st_nlink` is 1, and the user and group ids are the process's own | not started | `libc-sigaction-answers-success-and-keeps-nothing`, `libc-answers-st-nlink-0`, `libc-answers-uid-0-for-every-process` | — | +| 18 | libc gains `select`, `setvbuf` and `socketpair`. A descriptor is a small table number since #978, so `FD_SETSIZE` holds | not started | this track | — | +| 19 | The `libc` crate gains a ToyOS module, checked by `ctest` against `sdk/libc`'s headers | not started | this track | 18 | +| 20 | Children: `SIGCHLD`, sessions, a quit notice and Ctrl+C, the process track's stages 4–7 | not started | `a-childs-end-is-an-event-and-a-parent-takes-its-children-down` | — | +| 21 | Sockets shared by their holders and inherited by a child, which curl, libssh2 and every inheriting server need | direction set | `a-stream-cannot-move-between-processes`, `libc-close-of-a-socket-is-unproven-on-a-connected-socket` | engineering: the sharing and recovery design, out of the tree, under review | +| 22 | cargo's eight C libraries cross-built: curl, libgit2, libssh2, OpenSSL, SQLite, nghttp2, zlib, blake3; five stopped on headers, types and functions libc lacked at `15625e0cb` | not started | this track | 18, 19 | +| 23 | cargo's TLS: OpenSSL through its Perl `Configure`, or curl's rustls backend on `ToyOSOrg/ring`. A build from vendored sources needs neither, and neither needs the `toyos-crypto` track | not started, choice open | this track, `the-internet-clients-work-unchanged` | 22; HTTPS on `the-lan-is-not-yet-production-grade` | +| 24 | cargo at run time: its SQLite and its locks, sockets and children | not started | this track, `sqlite-and-postgresql-run-on-toyos` | 14–17, 20–22 | +| 25 | The tools a build runs: Perl and Python started by spawn, brush as `sh`, uutils, make, CMake with libuv on spawn, awk, and a script started by its interpreter line | not started | `ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have`, `nothing-launches-a-script-by-its-interpreter-line` | 20 for the C tools | +| 26 | git in the guest | not started | this track | 22 | +| 27 | The toolchain as `pkg` packages, none of it in the image. #970 makes the clang package; how the rest splits, and how a self-host test's guest reaches them, is open | in progress | this track | 3, 11 | +| 28 | Storage durable and fast enough for an LLVM build tree, unmeasured | not started | `storage-is-layers-and-a-role-is-a-filesystem`, `an-fsync-of-a-served-file-syncs-its-whole-volume` | 4 | +| 29 | A build reproduces: an image's GUIDs from the commit, and one checkout building one kernel | not started | `an-images-guids-are-drawn-from-the-hosts-random-source`, `one-checkout-of-one-tree-builds-two-kernels` | — | +| 30 | **M4 exit**: the guest builds a userland program and then the kernel, and the hashes equal the host build's of the same commit | not started | this track | 12–29 | What the items rest on. The compiles were made against libc at `15625e0cb` (2026-10-01), before #975, #978 and #988 changed it: -- **3, 26, measured:** the old `pkg` inflated a whole archive in memory under +- **3, 27, measured:** the old `pkg` inflated a whole archive in memory under `MAX_INFLATED`, 256 MiB, and refused every link (`system/pkg/src/main.rs`, `archive.rs`), which #970 replaces. Measured on the Linux-host release `toolchain-linux-x86_64-48dd24f826263d6c`, stripped: `librustc_driver` 184.7 MB, clang 122.8 MB, lld 78.6 MB, and 181.1 MB of `x86_64-unknown-toyos` libraries. -- **10, read:** rustc's launcher names `librustc_driver` as needed +- **11, read:** rustc's launcher names `librustc_driver` as needed (`llvm-readobj --needed-libs`). -- **17, read:** curl 8.21.0, which `curl-sys` 0.4.90 builds, does not compile +- **18, read:** curl 8.21.0, which `curl-sys` 0.4.90 builds, does not compile without `select` (`lib/curlx/wait.c`), and waits through it, since `curl-sys` does not define `HAVE_POLL`. curl's multi handle wakes through a `socketpair`, and its TLS key log sets its buffering through `setvbuf`. -- **18, read:** `libc` 0.2.189 has no ToyOS module (`rg -i toyos` over `src`) +- **19, read:** `libc` 0.2.189 has no ToyOS module (`rg -i toyos` over `src`) and is empty for an OS it does not know; `ToyOSOrg/libc` does not exist. cargo's `curl-sys`, `libssh2-sys` and `libgit2-sys` import its C types unconditionally. -- **21, compiled at `15625e0cb`:** nghttp2, zlib and blake3 build with nothing +- **22, compiled at `15625e0cb`:** nghttp2, zlib and blake3 build with nothing undefined; curl, libgit2, libssh2, OpenSSL and SQLite stop on headers, types or functions libc lacked. -- **21 and 23, read:** SQLite's unix VFS takes `fcntl` record locks keyed on +- **22 and 24, read:** SQLite's unix VFS takes `fcntl` record locks keyed on `st_dev` and `st_ino`; libgit2 opens every repository through `realpath` and reads packs through a file `mmap`; curl and libssh2 wait on non-blocking sockets. `openssl-src` 300.6.1 refuses a target it does not know (`src/lib.rs`), and `openssl-sys` is a `cfg(unix)` dependency of `curl-sys` and `libssh2-sys`, which ToyOS is not. That cargo bundles SQLite is read from its dependencies, not run. -- **24, read:** Perl 5.44.0's sources never call `posix_spawn`. Which Python, +- **25, read:** Perl 5.44.0's sources never call `posix_spawn`. Which Python, which make and which awk is open. +Owed by the `clang` package (item 3), and needed by no exit: +- **Licences.** The package carries LLVM's `LICENSE.TXT` and none of the + licences of what `libtoyos_c.a` links besides `sdk/libc`, though + `build/hostedclang.rs` packs `sysroot/lib`; a package published beyond a + test's ROOT owes them. +- **A command by its name.** A shell finds no installed package's command by + name: its `PATH` is `/system/bin` (`system/shell/src/main.rs`), so + `clang hello.c` is typed `/apps/clang/bin/clang hello.c`. How an installed + command is reached by its name is open. + Beside the path, and needed by none of its exits: - **The boot handoff** (owner, 2026-10-21) removes the kernel's ROOT mount: the loader hands the supervisor and the system file server over as signed @@ -149,5 +160,5 @@ Beside the path, and needed by none of its exits: - Every AArch64 stage lands one step behind x86-64, on `toyos-runs-on-arm64`. **No image ships a linker** (owner, 2026-10-04: "Just get rid of toyos ld"). -Until item 5 lands, `ld.lld` links inside ToyOS only where `hosted_clang_hello` +Until item 6 lands, `ld.lld` links inside ToyOS only where `hosted_clang_hello` stages it. diff --git a/lib/bcachefs/src/alloc_bitmap.rs b/lib/bcachefs/src/alloc_bitmap.rs index af76e98260..04e216f827 100644 --- a/lib/bcachefs/src/alloc_bitmap.rs +++ b/lib/bcachefs/src/alloc_bitmap.rs @@ -141,7 +141,7 @@ impl BitmapAllocator { /// What an allocation may take now: every free block, but for those /// [`NODE_RESERVE`] keeps from all but an operation that draws on it. - fn spare(&self) -> u64 { + pub(crate) fn spare(&self) -> u64 { match !self.shadow || self.op.as_ref().is_some_and(|op| op.reserve == Reserve::Draw) { true => self.free_blocks, false => self.free_blocks.saturating_sub(NODE_RESERVE), diff --git a/lib/bcachefs/src/fs.rs b/lib/bcachefs/src/fs.rs index 28355c73d8..418cee4118 100644 --- a/lib/bcachefs/src/fs.rs +++ b/lib/bcachefs/src/fs.rs @@ -607,6 +607,12 @@ impl Mounted { fs.alloc.count_free(&fs.io)?; Ok(fs) } + + /// The bytes a change may still take: every free block but those + /// [`crate::alloc_bitmap::NODE_RESERVE`] keeps for a delete. + pub fn free_bytes(&self) -> u64 { + self.alloc.spare() * BLOCK_SIZE as u64 + } } impl Mounted { diff --git a/lib/manifest/src/package.rs b/lib/manifest/src/package.rs index b616fc62b2..039fc092c7 100644 --- a/lib/manifest/src/package.rs +++ b/lib/manifest/src/package.rs @@ -36,8 +36,8 @@ pub struct Package { /// The archive this directory was unpacked from, as the release's own /// `SHA256SUMS` spelled it. pub digest: String, - /// The whole guest path a launch starts, always a file directly inside - /// this package's own directory. + /// The whole guest path a launch starts, always a file inside this + /// package's own directory. pub program: String, } @@ -181,13 +181,14 @@ pub fn package_of(path: &str) -> Option<&str> { Some(name) } -/// Whether `program` names a file directly inside package `name`'s directory. +/// Whether `program` names a file inside package `name`'s directory, by a path +/// with no empty, `.` or `..` component. pub fn program_is_inside(name: &str, program: &str) -> bool { if name_is_bad(name) { return false; } let Some(file) = program.strip_prefix(&format!("{DIR}/{name}/")) else { return false }; - !file.is_empty() && !file.contains('/') && file != "." && file != ".." + file.split('/').all(|part| !part.is_empty() && part != "." && part != "..") } fn name_is_bad(name: &str) -> bool { @@ -249,13 +250,17 @@ mod tests { "/system/bin/supervisor", "/apps/other/other", "/apps/gbae/../../system/bin/supervisor", - "/apps/gbae/sub/x", + "/apps/gbae/bin/../../other/other", + "/apps/gbae/bin//gbae", + "/apps/gbae/./gbae", + "/apps/gbae/bin/", "/apps/gbae/", ] { let bad = Package { program: program.into(), ..sample() }; assert!(bad.render().is_err(), "{program} was accepted"); } assert!(sample().render().is_ok()); + assert!(Package { program: "/apps/gbae/bin/gbae".into(), ..sample() }.render().is_ok()); } #[test] diff --git a/sdk/toyos/src/fs.rs b/sdk/toyos/src/fs.rs index dc0f7f86e3..deb14ef00d 100644 --- a/sdk/toyos/src/fs.rs +++ b/sdk/toyos/src/fs.rs @@ -92,6 +92,8 @@ pub const SEEK: u32 = 21; /// carries its other end. It spends a served slot of this connection's share /// from here, and names no path. pub const EXPORT: u32 = 22; +/// The free space of the volume behind the capability ([`Room`]). +pub const ROOM: u32 = 23; pub const SEEK_START: u64 = 0; pub const SEEK_CURRENT: u64 = 1; @@ -453,6 +455,16 @@ pub struct Stat { pub node: u64, } +/// How much more the volume behind a capability holds: `free` bytes, taken in +/// allocation units of `unit` bytes, so a file of `n` bytes takes `n` rounded +/// up to `unit`. A volume's own structures take room too, so it is a bound and +/// never a promise. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Room { + pub free: u64, + pub unit: u64, +} + /// A write answered. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Written { @@ -743,6 +755,18 @@ impl Dir { }) } + /// The volume's [`Room`]. Like [`Self::sync`], it names no path. + pub fn room(&mut self) -> Result { + let reply = self.path_call(ROOM, Request::new(), "", "").map_err(|refused| match refused { + Refused::Error(e) => e, + Refused::Link(_) => Error::Malformed, + })?; + match reply.value2 { + 0 => Err(Error::Malformed), + unit => Ok(Room { free: reply.value, unit }), + } + } + /// A pipe whose bytes the server writes to this file at the description's /// offset, which they move, for a program's standard output: the write end. pub fn stream(&mut self, fid: u64, generation: u64) -> Result { diff --git a/system/fileserver/src/absent.rs b/system/fileserver/src/absent.rs index 6b26b43345..da923e553d 100644 --- a/system/fileserver/src/absent.rs +++ b/system/fileserver/src/absent.rs @@ -102,6 +102,10 @@ impl Volume for Absent { Ok(Vec::new()) } + fn room(&mut self) -> Result { + Ok(toyos::fs::Room { free: 0, unit: 1 }) + } + fn describe(&self) -> String { format!("absent: {}", self.why) } diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index c49fbca2ef..e587f82842 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -769,6 +769,10 @@ impl Volume for DataVolume { Ok(unwritten) } + fn room(&mut self) -> Result { + Ok(toyos::fs::Room { free: self.fs.free_bytes(), unit: BLOCK as u64 }) + } + fn describe(&self) -> String { let c = self.cache.counts(); format!( @@ -1420,4 +1424,19 @@ mod tests { assert_eq!(names(&mut v, "home/staged"), ["a", "z"]); assert_eq!(v.lstat(&to), Err(Error::NotFound)); } + + /// What a write takes is what the room answers less, block for block, + /// and the room never names the blocks the reserve keeps for a delete. + #[test] + fn a_write_takes_its_blocks_out_of_the_room() { + let mut v = vol(); + let n = v.open("apps/x", CREATE).unwrap(); + let before = v.room().unwrap(); + assert_eq!(before.unit, BLOCK as u64); + v.write(n, 0, &vec![7u8; 3 * BLOCK + 1]).unwrap(); + assert_eq!(v.room().unwrap().free, before.free - 4 * BLOCK as u64); + v.close(n).unwrap(); + let free = free_blocks(&mut v); + assert!(v.room().unwrap().free < free * BLOCK as u64, "the reserve's blocks were counted as room"); + } } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 7b08554a74..ec48e11ce1 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -458,6 +458,15 @@ impl Volume for FatVolume { Ok(unlevel) } + fn room(&mut self) -> Result { + let unit = u64::from(self.fs.geometry().bytes_per_cluster()); + if !self.writable { + return Ok(fs::Room { free: 0, unit }); + } + let free = self.fs.free_bytes().map_err(|e| logged("free space", "", e))?; + Ok(fs::Room { free, unit }) + } + fn describe(&self) -> String { let c = self.cache.counts(); format!( diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 8a5657ccdd..b3d4a20fad 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -761,6 +761,10 @@ impl Server { self.sync(Some(node)) } SYNC => self.sync(None), + ROOM => { + let room = self.volume.room()?; + Ok(Answer::Reply(Reply { value: room.free, value2: room.unit, ..Reply::ok() })) + } STREAM => { let file = self.fid(id, r.fid)?; if !file.borrow().write { diff --git a/system/fileserver/src/rights.rs b/system/fileserver/src/rights.rs index eba3a0afa0..a8a6186823 100644 --- a/system/fileserver/src/rights.rs +++ b/system/fileserver/src/rights.rs @@ -13,7 +13,7 @@ use toyos::fs::*; pub fn changes(op: u32, flags: u64) -> Option { match op { OPEN => Some(flags & (O_WRITE | O_APPEND | O_CREATE | O_TRUNCATE | O_CREATE_NEW) != 0), - HELLO | CLOSE | READ | STAT | LSTAT | FSTAT | FSYNC | SYNC | READDIR | READLINK | DUP | SEEK | EXPORT => { + HELLO | CLOSE | READ | STAT | LSTAT | FSTAT | FSYNC | SYNC | READDIR | READLINK | DUP | SEEK | EXPORT | ROOM => { Some(false) } WRITE | TRUNCATE | MKDIR | RMDIR | UNLINK | RENAME | SYMLINK | STREAM => Some(true), @@ -27,7 +27,7 @@ mod tests { /// Every request the wire has, by what it does to the directory: the /// independent spelling `changes` is held to. - const READS: [u32; 13] = [HELLO, CLOSE, READ, STAT, LSTAT, FSTAT, FSYNC, SYNC, READDIR, READLINK, DUP, SEEK, EXPORT]; + const READS: [u32; 14] = [HELLO, CLOSE, READ, STAT, LSTAT, FSTAT, FSYNC, SYNC, READDIR, READLINK, DUP, SEEK, EXPORT, ROOM]; const WRITES: [u32; 8] = [WRITE, TRUNCATE, MKDIR, RMDIR, UNLINK, RENAME, SYMLINK, STREAM]; #[test] @@ -41,7 +41,7 @@ mod tests { // Every request number the wire has is one of the two, or `OPEN`. let mut named: Vec = READS.iter().chain(&WRITES).copied().chain([OPEN]).collect(); named.sort_unstable(); - assert_eq!(named, (HELLO..=EXPORT).collect::>()); + assert_eq!(named, (HELLO..=ROOM).collect::>()); } /// An open changes the directory by any one flag that writes, creates or @@ -60,7 +60,7 @@ mod tests { /// client is let go whatever its grant. #[test] fn a_request_the_wire_does_not_have_is_neither() { - for op in [0, EXPORT + 1, REPLY, LINK, u32::MAX] { + for op in [0, ROOM + 1, REPLY, LINK, u32::MAX] { assert_eq!(changes(op, 0), None, "request {op}"); } } diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 2fb7c53b4d..0278b56346 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -132,6 +132,9 @@ pub trait Volume { /// volume's own flush refused. fn sync(&mut self) -> Result, Error>; + /// What more can be written to the volume: none, where nothing can be. + fn room(&mut self) -> Result; + /// One line of what the volume and its cache have done, for a log. fn describe(&self) -> String; } diff --git a/system/pkg/Cargo.toml b/system/pkg/Cargo.toml index 20f281bede..5f9be099a3 100644 --- a/system/pkg/Cargo.toml +++ b/system/pkg/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pkg" -description = "The installer: an archive checked against its release's SHA256SUMS and unpacked as one directory under /apps; the decoders are a library tested on the host." +description = "The installer: an archive checked against its release's SHA256SUMS and streamed into one directory under /apps within the room its volume has; the decoders are a library tested on the host." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" @@ -12,10 +12,18 @@ license = "MIT OR Apache-2.0" doctest = false [dependencies] +# The room `/apps`'s volume has, which only its file server answers. +toyos = { path = "../../sdk/toyos" } toyos-manifest = { path = "../../lib/manifest" } +toyos-sha2 = { path = "../../lib/sha2" } toyos-sha2-hw = { path = "../../lib/sha2-hw" } flate2 = { version = "1", default-features = false, features = ["rust_backend"] } [dev-dependencies] +# Archives made by a writer that is not this crate's reader. +tar = { version = "0.4.46", default-features = false } # Test scratch that is gone on every way out; `--ci host` reds on any left behind. toyos-tmpdir = { path = "../../lib/tmpdir" } + +[package.metadata.toyos.host] +exempt.manages = "ToyOS's installed packages: the directories under /apps the supervisor launches from" diff --git a/system/pkg/src/archive.rs b/system/pkg/src/archive.rs index aeb799cccf..b7cd77b47c 100644 --- a/system/pkg/src/archive.rs +++ b/system/pkg/src/archive.rs @@ -1,41 +1,48 @@ -//! The ustar half of a `.tar.gz`, decoded as far as unpacking a release -//! archive needs and no further. +//! The ustar half of a `.tar.gz`, read as a stream: one header at a time, and +//! the entry's data after it, so no archive is ever held whole. //! //! **Every field is the archive's claim about itself**, and this is the one //! place a downloaded file decides where bytes are written. A name that leaves -//! the package directory, a header whose own checksum does not match, a size -//! that runs off the end and every type that is not a plain file or a -//! directory are refused by name rather than skipped. +//! the package directory, a header whose own checksum does not match, an +//! archive that ends inside an entry and every type that is not a plain file, +//! a directory or a symlink are refused by name rather than skipped. Where a +//! symlink may point is the package's business (`crate::Planner`). //! -//! GNU long names, pax headers, links and device nodes are refused rather than -//! implemented: an archive that needs one is one this installer does not +//! GNU long names, pax headers, hard links and device nodes are refused rather +//! than implemented: an archive that needs one is one this installer does not //! install, said out loud. +use std::io::Read; + const BLOCK: usize = 512; const TYPE_FLAG: usize = 156; const CHKSUM: usize = 148; const CHKSUM_LEN: usize = 8; +const LINK_NAME: usize = 157; +const MAGIC: usize = 257; /// What a tar entry may be here. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum Kind { File, Dir, + /// A symlink, holding its target as the archive wrote it. + Symlink(String), } -/// One entry, borrowing the decompressed archive. -#[derive(Debug)] -pub struct Entry<'a> { +/// One entry's header. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Header { /// Slash-separated, relative, and already checked to stay inside itself. pub path: String, pub kind: Kind, - /// Empty for a directory. - pub data: &'a [u8], + /// The data that follows the header: 0 but for a file. + pub size: u64, /// The low nine permission bits; the rest of the mode is ignored. pub mode: u32, } -impl Entry<'_> { +impl Header { /// Whether the archive marked this entry executable for its owner. pub fn executable(&self) -> bool { self.mode & 0o100 != 0 @@ -48,49 +55,100 @@ impl Entry<'_> { } } -/// Decode every entry of an uncompressed tar. -pub fn entries(tar: &[u8]) -> Result>, String> { - let mut out = Vec::new(); - let mut at = 0; - while at + BLOCK <= tar.len() { - let header = &tar[at..at + BLOCK]; +/// An uncompressed tar, an entry at a time. +pub struct Reader { + inner: R, + /// The current entry's data not yet read. + left: u64, + /// The zeros that pad it to a block. + pad: u64, + /// The path of the current entry, for a refusal that names it. + current: String, +} + +impl Reader { + pub fn new(inner: R) -> Self { + Self { inner, left: 0, pad: 0, current: String::new() } + } + + /// The next entry, past whatever of the last one was not read, or `None` at + /// the block of zeros that ends the archive. + pub fn next_entry(&mut self) -> Result, String> { + self.skip(self.left + self.pad)?; + self.left = 0; + self.pad = 0; + let mut header = [0u8; BLOCK]; + self.inner.read_exact(&mut header).map_err(|e| match e.kind() { + std::io::ErrorKind::UnexpectedEof => String::from("tar: the archive ends inside a header"), + _ => format!("tar: {e}"), + })?; if header.iter().all(|&b| b == 0) { - return Ok(out); + return Ok(None); } - checksum_matches(header)?; - let magic = &header[257..262]; - if magic != b"ustar" { - return Err(format!("tar: block at {at} is not ustar")); + let header = decode(&header)?; + self.current = header.path.clone(); + self.left = header.size; + self.pad = header.size.next_multiple_of(BLOCK as u64) - header.size; + Ok(Some(header)) + } + + /// Up to `buf.len()` bytes of the current entry's data: 0 once it is all + /// read, and a refusal where the archive ends before it does. + pub fn read_data(&mut self, buf: &mut [u8]) -> Result { + let want = buf.len().min(usize::try_from(self.left).unwrap_or(usize::MAX)); + if want == 0 { + return Ok(0); } - let size = octal(header, 124, 12, "size")?; - let mode = octal(header, 100, 8, "mode")? as u32; - let kind = match header[TYPE_FLAG] { - b'0' | 0 => Kind::File, - b'5' => Kind::Dir, - other => { - return Err(format!( - "tar: entry {:?} is type {:?}, and only a file or a directory is unpacked", - field(header, 0, 100), - other as char - )) - } - }; - let path = path_of(header)?; - at += BLOCK; - let data = match kind { - Kind::Dir => &tar[at..at], - Kind::File => { - let end = at - .checked_add(size as usize) - .filter(|end| *end <= tar.len()) - .ok_or_else(|| format!("tar: {path} claims {size} bytes the archive lacks"))?; - &tar[at..end] + let n = self.inner.read(&mut buf[..want]).map_err(|e| format!("tar: {}: {e}", self.current))?; + if n == 0 { + return Err(format!("tar: the archive ends inside {}", self.current)); + } + self.left -= n as u64; + Ok(n) + } + + /// Discard up to `bytes` of the stream. One that ends first is refused by + /// the header read after it. + fn skip(&mut self, bytes: u64) -> Result<(), String> { + std::io::copy(&mut (&mut self.inner).take(bytes), &mut std::io::sink()) + .map(drop) + .map_err(|e| format!("tar: {}: {e}", self.current)) + } +} + +fn decode(header: &[u8; BLOCK]) -> Result { + checksum_matches(header)?; + // POSIX ustar is `ustar\0` then `00`; GNU tar's own format is `ustar \0`, + // and keeps other fields where ustar has the name's prefix. + let posix = match &header[MAGIC..MAGIC + 8] { + b"ustar\x0000" => true, + b"ustar \0" => false, + _ => return Err(format!("tar: {:?} is not a ustar header", field(header, 0, 100))), + }; + let mode = octal(header, 100, 8, "mode")? as u32 & 0o777; + let size = octal(header, 124, 12, "size")?; + let path = path_of(header, posix)?; + let kind = match header[TYPE_FLAG] { + b'0' | 0 => Kind::File, + b'5' => Kind::Dir, + b'2' => { + let target = field(header, LINK_NAME, 100); + if target.is_empty() { + return Err(format!("tar: the symlink {path} holds no target")); } - }; - at += (size as usize).div_ceil(BLOCK) * BLOCK; - out.push(Entry { path, kind, data, mode }); + Kind::Symlink(target) + } + other => { + return Err(format!( + "tar: entry {path:?} is type {:?}, and only a file, a directory or a symlink is unpacked", + other as char + )) + } + }; + if kind != Kind::File && size != 0 { + return Err(format!("tar: {path} claims {size} bytes of data, which only a file carries")); } - Err(String::from("tar: the archive ends inside a header")) + Ok(Header { path, kind, size, mode }) } /// The header's own checksum, which is what makes a truncated or misaligned @@ -113,9 +171,9 @@ fn checksum_matches(header: &[u8]) -> Result<(), String> { /// `prefix/name`, refused unless it stays inside the archive's own tree: a /// leading `/` or a `..` component is how an archive writes outside the /// directory it was unpacked into. -fn path_of(header: &[u8]) -> Result { +fn path_of(header: &[u8], posix: bool) -> Result { let name = field(header, 0, 100); - let prefix = field(header, 345, 155); + let prefix = if posix { field(header, 345, 155) } else { String::new() }; let joined = if prefix.is_empty() { name.clone() } else { format!("{prefix}/{name}") }; let path = joined.trim_end_matches('/'); if path.is_empty() || path.starts_with('/') { @@ -146,29 +204,35 @@ fn octal(header: &[u8], at: usize, len: usize, what: &str) -> Result Vec { + /// One POSIX ustar header as a writer would build it, so the refusals below + /// are tested against archives that are otherwise valid. + pub(crate) fn header(name: &str, kind: u8, size: u64, mode: u32, link: &str) -> Vec { let mut h = vec![0u8; BLOCK]; h[..name.len()].copy_from_slice(name.as_bytes()); h[100..108].copy_from_slice(format!("{mode:07o}\0").as_bytes()); h[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); h[TYPE_FLAG] = kind; - h[257..263].copy_from_slice(b"ustar\0"); - h[263..265].copy_from_slice(b"00"); + h[LINK_NAME..LINK_NAME + link.len()].copy_from_slice(link.as_bytes()); + h[MAGIC..MAGIC + 8].copy_from_slice(b"ustar\x0000"); + seal(&mut h); + h + } + + /// Write the header's checksum over what it holds now. + pub(crate) fn seal(h: &mut [u8]) { h[CHKSUM..CHKSUM + CHKSUM_LEN].copy_from_slice(b" "); let sum: u64 = h.iter().map(|&b| u64::from(b)).sum(); h[CHKSUM..CHKSUM + CHKSUM_LEN].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - h } - fn archive(parts: &[(&str, u8, &[u8], u32)]) -> Vec { + /// `(name, type, data, mode, link target)` each, then the end. + pub(crate) fn archive(parts: &[(&str, u8, &[u8], u32, &str)]) -> Vec { let mut out = Vec::new(); - for (name, kind, data, mode) in parts { - out.extend(header(name, *kind, data.len() as u64, *mode)); + for (name, kind, data, mode, link) in parts { + out.extend(header(name, *kind, data.len() as u64, *mode, link)); out.extend(*data); out.resize(out.len().div_ceil(BLOCK) * BLOCK, 0); } @@ -176,54 +240,116 @@ mod tests { out } + /// Every entry with its data, read the way the installer reads them. + fn all(tar: &[u8]) -> Result)>, String> { + let mut reader = Reader::new(tar); + let mut out = Vec::new(); + while let Some(h) = reader.next_entry()? { + let mut data = Vec::new(); + let mut buf = [0u8; 3]; + loop { + match reader.read_data(&mut buf)? { + 0 => break, + n => data.extend_from_slice(&buf[..n]), + } + } + out.push((h, data)); + } + Ok(out) + } + #[test] - fn a_directory_and_two_files_come_back_whole() { + fn a_directory_two_files_and_a_link_come_back_whole() { let tar = archive(&[ - ("gbae/", b'5', b"", 0o755), - ("gbae/gbae", b'0', b"ELF...", 0o755), - ("gbae/LICENSE", b'0', b"MIT", 0o644), + ("gbae/", b'5', b"", 0o755, ""), + ("gbae/gbae", b'0', b"ELF...", 0o755, ""), + ("gbae/LICENSE", b'0', b"MIT", 0o644, ""), + ("gbae/run", b'2', b"", 0o777, "gbae"), ]); - let got = entries(&tar).expect("entries"); - assert_eq!(got.len(), 3); - assert_eq!(got[0].path, "gbae"); - assert_eq!(got[0].kind, Kind::Dir); - assert_eq!(got[1].path, "gbae/gbae"); - assert_eq!(got[1].data, b"ELF..."); - assert!(got[1].executable()); - assert_eq!(got[2].data, b"MIT"); - assert!(!got[2].executable()); - assert!(got.iter().all(|e| e.top() == "gbae")); + let got = all(&tar).expect("entries"); + assert_eq!(got.len(), 4); + assert_eq!((got[0].0.path.as_str(), &got[0].0.kind), ("gbae", &Kind::Dir)); + assert_eq!((got[1].0.path.as_str(), got[1].1.as_slice()), ("gbae/gbae", &b"ELF..."[..])); + assert!(got[1].0.executable()); + assert_eq!(got[2].1, b"MIT"); + assert!(!got[2].0.executable()); + assert_eq!(got[3].0.kind, Kind::Symlink("gbae".into())); + assert!(got.iter().all(|(h, _)| h.top() == "gbae")); + } + + /// An entry whose data the reader never asked for is stepped over, padding + /// and all, so the next header is where the archive put it. + #[test] + fn data_left_unread_is_skipped_to_the_next_header() { + let tar = archive(&[("a/x", b'0', &[1u8; 700], 0o644, ""), ("a/y", b'0', b"y", 0o644, "")]); + let mut reader = Reader::new(tar.as_slice()); + assert_eq!(reader.next_entry().unwrap().unwrap().path, "a/x"); + assert_eq!(reader.next_entry().unwrap().unwrap().path, "a/y"); + let mut buf = [0u8; 8]; + assert_eq!(reader.read_data(&mut buf).unwrap(), 1); + assert_eq!(reader.next_entry().unwrap(), None); } /// The refusal the installer's write path rests on. #[test] fn a_name_that_leaves_the_archive_is_refused() { - for name in ["/etc/passwd", "gbae/../../system/bin/supervisor", "../x", "./x", ""] { - let tar = archive(&[(name, b'0', b"x", 0o644)]); - assert!(entries(&tar).is_err(), "{name:?} was accepted"); + for name in ["/etc/passwd", "gbae/../../system/bin/supervisor", "../x", "./x", "a//b", ""] { + let tar = archive(&[(name, b'0', b"x", 0o644, "")]); + assert!(all(&tar).is_err(), "{name:?} was accepted"); } } /// A type silently skipped is a package missing a file the archive carried. #[test] - fn every_type_but_a_file_and_a_directory_is_refused() { - for kind in [b'1', b'2', b'3', b'4', b'6', b'L', b'x', b'g'] { - let tar = archive(&[("gbae/x", kind, b"", 0o644)]); - assert!(entries(&tar).is_err(), "type {} was accepted", kind as char); + fn every_type_but_a_file_a_directory_and_a_symlink_is_refused() { + for kind in *b"13467LKxg" { + let tar = archive(&[("gbae/x", kind, b"", 0o644, "y")]); + let why = all(&tar).expect_err(&format!("type {} was accepted", kind as char)); + assert!(why.contains("only a file, a directory or a symlink"), "{why}"); + } + assert!(all(&archive(&[("gbae/x", b'2', b"", 0o777, "")])).is_err(), "a link to nothing"); + let link_with_data = archive(&[("gbae/x", b'2', b"data", 0o777, "y")]); + assert!(all(&link_with_data).is_err(), "a link that carries data"); + } + + #[test] + fn a_truncated_archive_is_refused_wherever_it_ends() { + let whole = archive(&[("gbae/gbae", b'0', &[7u8; 1000], 0o755, "")]); + // Inside the header, inside the data, inside the padding, and with no + // end block at all. + for cut in [1, BLOCK - 1, BLOCK + 1, BLOCK + 999, BLOCK + 1000, 3 * BLOCK] { + let why = all(&whole[..cut]).expect_err(&format!("cut at {cut} was accepted")); + assert!(why.contains("ends inside"), "cut at {cut}: {why}"); } + assert_eq!(all(&whole).unwrap().len(), 1); + assert_eq!(all(&[0u8; BLOCK]).expect("an end block ends it").len(), 0); } #[test] - fn a_header_that_does_not_sum_and_a_size_past_the_end_are_both_refused() { - let mut tampered = archive(&[("gbae/gbae", b'0', b"ELF", 0o755)]); + fn a_header_that_does_not_sum_or_is_not_ustar_is_refused() { + let mut tampered = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); tampered[3] ^= 0x20; - assert!(entries(&tampered).is_err()); + assert!(all(&tampered).is_err()); + + let mut v7 = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); + v7[MAGIC..MAGIC + 8].fill(0); + seal(&mut v7[..BLOCK]); + assert!(all(&v7).unwrap_err().contains("not a ustar header")); + } - let mut short = archive(&[("gbae/gbae", b'0', b"ELF", 0o755)]); - short.truncate(BLOCK + 1); - assert!(entries(&short).is_err()); + /// GNU tar's format keeps times where POSIX keeps the prefix, so its + /// prefix field is never part of the name. + #[test] + fn a_gnu_header_s_prefix_field_is_not_its_name() { + let mut tar = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); + tar[MAGIC..MAGIC + 8].copy_from_slice(b"ustar \0"); + tar[345..352].copy_from_slice(b"1234567"); + seal(&mut tar[..BLOCK]); + assert_eq!(all(&tar).unwrap()[0].0.path, "gbae/gbae"); - assert!(entries(&[0u8; 7]).is_err()); - assert_eq!(entries(&[0u8; BLOCK]).expect("an end block ends it").len(), 0); + let mut posix = archive(&[("gbae", b'0', b"ELF", 0o755, "")]); + posix[345..349].copy_from_slice(b"deep"); + seal(&mut posix[..BLOCK]); + assert_eq!(all(&posix).unwrap()[0].0.path, "deep/gbae"); } } diff --git a/system/pkg/src/lib.rs b/system/pkg/src/lib.rs index b6069eadc7..dc1e3324ce 100644 --- a/system/pkg/src/lib.rs +++ b/system/pkg/src/lib.rs @@ -2,96 +2,168 @@ //! //! **A package is a directory**, so installing one is: agree with the archive //! on a single top-level name, find the program inside it that name promises, -//! and write the manifest recording where the bytes came from. +//! check that it fits the room the volume has, and write the manifest +//! recording where the bytes came from. The archive is planned as it streams +//! past ([`Planner`]), and only a plan that finished is written. pub mod archive; pub mod sums; -use archive::{Entry, Kind}; +use std::collections::BTreeMap; + +use archive::{Header, Kind}; use toyos_manifest::package::{self, Package}; /// The suffix an installable archive carries. Not a guess about compression — /// the version is read out of the name, so the name has a shape or it has none. pub const SUFFIX: &str = ".tar.gz"; +/// The most entries one archive installs. Policy on the primitive: the plan +/// holds every entry's header until the archive has been read to its end, so +/// this bounds the installer's memory. A toolchain is a few thousand. +pub const MAX_ENTRIES: usize = 1 << 16; + /// Everything the writer needs, and nothing it has to decide. -pub struct Plan<'a> { +#[derive(Debug)] +pub struct Plan { pub manifest: Package, - /// Directories to create, shallowest first. - pub dirs: Vec, - /// `(whole guest path, contents)`. **No mode travels with them**: this + /// Every entry, in the archive's order, which the second read of the same + /// archive must meet again header for header. **No mode is written**: this /// filesystem has no permission bits, so the archive's mode is read as its /// statement about which file is the program and is never written. - pub files: Vec<(String, &'a [u8])>, + pub entries: Vec
, + /// The room the files take on the volume, each rounded up to its unit. + pub need: u64, +} + +/// A plan, made one header at a time. +pub struct Planner<'a> { + file_name: &'a str, + free: u64, + unit: u64, + entries: Vec
, + /// Every path the archive names or implies, and whether it is a directory. + /// One path is one thing: a path named twice, or a path beneath a file or a + /// link, would decide by write order what is at it, or write through a + /// link to somewhere the plan never checked. + seen: BTreeMap, + need: u64, } -/// What installing `file_name` would do, or why it would not. -pub fn plan<'a>( - file_name: &str, - digest: &str, - entries: &[Entry<'a>], -) -> Result, String> { - let name = single_top(entries)?; - let version = version_of(file_name, name)?; - let program = format!("{}/{name}", Package::dir(name)); - let inside = format!("{name}/{name}"); - match entries.iter().find(|e| e.path == inside) { - Some(e) if e.kind == Kind::File && e.executable() => {} - Some(_) => { - return Err(format!("pkg: {file_name}'s {inside} is not an executable file")); +impl<'a> Planner<'a> { + /// A plan for `file_name` on a volume with `free` bytes in units of `unit`. + pub fn new(file_name: &'a str, free: u64, unit: u64) -> Self { + Self { file_name, free, unit: unit.max(1), entries: Vec::new(), seen: BTreeMap::new(), need: 0 } + } + + /// The next header, refused the moment it breaks the package's shape or + /// takes the plan past the room: what an archive claims about its size is + /// refused before any of that size is read. + pub fn add(&mut self, header: Header) -> Result<(), String> { + let name = self.file_name; + if self.entries.len() == MAX_ENTRIES { + return Err(format!("pkg: {name} holds more than {MAX_ENTRIES} entries")); } - None => { + if let Some(first) = self.entries.first() { + if header.top() != first.top() { + return Err(format!( + "pkg: the archive holds both {:?} and {:?} at its top level", + first.top(), + header.top() + )); + } + } + if header.path == format!("{}/{}", header.top(), package::FILE) { + return Err(format!("pkg: {name} carries {}, which is the installer's to write", header.path)); + } + if let Kind::Symlink(target) = &header.kind { + link_stays_inside(&header.path, target) + .map_err(|why| format!("pkg: {name}'s link {} -> {target:?} {why}", header.path))?; + } + let parts: Vec<&str> = header.path.split('/').collect(); + for depth in 1..parts.len() { + let parent = parts[..depth].join("/"); + if !*self.seen.entry(parent.clone()).or_insert(true) { + return Err(format!("pkg: {name} puts {} beneath {parent}, which is not a directory", header.path)); + } + } + let is_dir = header.kind == Kind::Dir; + match self.seen.insert(header.path.clone(), is_dir) { + Some(true) if is_dir => {} + Some(_) => return Err(format!("pkg: {name} names {} twice", header.path)), + None => {} + } + self.need = self.need.saturating_add(header.size.div_ceil(self.unit).saturating_mul(self.unit)); + if self.need > self.free { return Err(format!( - "pkg: {file_name} carries no {inside}, so there is no program to launch" - )) + "pkg: {name} needs more than the {} bytes free under {}", + self.free, + package::DIR + )); } + self.entries.push(header); + Ok(()) } - let mut dirs = vec![Package::dir(name)]; - let mut files = Vec::new(); - for entry in entries { - let dest = format!("{}/{}", package::DIR, entry.path); - match entry.kind { - Kind::Dir => dirs.push(dest), - Kind::File => files.push((dest, entry.data)), - } + /// The plan, once the archive has ended, with the archive's `digest`. + pub fn finish(self, digest: &str) -> Result { + let file_name = self.file_name; + let name = + self.entries.first().ok_or_else(|| String::from("pkg: the archive is empty"))?.top().to_string(); + let version = version_of(file_name, &name)?; + let inside = program_of(&self.entries, &name) + .map_err(|why| format!("pkg: {file_name} {why}"))?; + let manifest = Package { + name, + version, + digest: digest.to_string(), + program: format!("{}/{inside}", package::DIR), + }; + // The render is the check: it is what refuses a name, a version or a + // program that would not come back out of the file the same way. + manifest.render()?; + Ok(Plan { manifest, entries: self.entries, need: self.need }) } - dirs.sort(); - dirs.dedup(); - // One path, one kind, over both lists: the write order is directories then - // files, so a path named as both would decide by that order what is at it. - let mut seen: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); - seen.extend(dirs.iter().map(String::as_str)); - seen.sort_unstable(); - let count = seen.len(); - seen.dedup(); - if seen.len() != count { - return Err(format!("pkg: {file_name} names one path twice")); - } - - let manifest = Package { - name: name.to_string(), - version, - digest: digest.to_string(), - program, - }; - // The render is the check: it is what refuses a name, a version or a - // program that would not come back out of the file the same way. - manifest.render()?; - Ok(Plan { manifest, dirs, files }) } -/// The one directory every entry is inside, which is the package's name. An -/// archive that spreads over two top-level names installs neither: picking the -/// first would install half of it under a name it never chose. -fn single_top<'a>(entries: &'a [Entry<'a>]) -> Result<&'a str, String> { - let first = entries.first().ok_or_else(|| String::from("pkg: the archive is empty"))?.top(); - match entries.iter().map(Entry::top).find(|top| *top != first) { - Some(other) => { - Err(format!("pkg: the archive holds both {first:?} and {other:?} at its top level")) - } - None => Ok(first), +/// The program a package launches: `/`, as a package of one +/// program has it, or `/bin/`, as a toolchain does, an executable +/// file either way. Both is a package that has not said which. +fn program_of(entries: &[Header], name: &str) -> Result { + let candidates = [format!("{name}/{name}"), format!("{name}/bin/{name}")]; + let found: Vec<&Header> = + entries.iter().filter(|e| candidates.contains(&e.path)).collect(); + match found[..] { + [one] if one.kind == Kind::File && one.executable() => Ok(one.path.clone()), + [one] => Err(format!("has {} and it is not an executable file", one.path)), + [] => Err(format!("carries neither {} nor {}, so there is no program to launch", candidates[0], candidates[1])), + _ => Err(format!("carries both {} and {}, so which one launches is not said", candidates[0], candidates[1])), + } +} + +/// Whether the symlink at `path` reaching `target` lands inside the package. +/// +/// **Only `..` first and names after**: each leading `..` climbs one real +/// directory of the link's own path, which [`Planner::add`] has made sure is a +/// directory and not a link, and the climb may not pass the package's own +/// directory. A `..` after a name would climb from wherever that name, perhaps +/// itself a link, resolves to, which no reading of the archive alone can say. +fn link_stays_inside(path: &str, target: &str) -> Result<(), &'static str> { + if target.starts_with('/') { + return Err("is absolute"); + } + let parts: Vec<&str> = target.split('/').collect(); + let climbs = parts.iter().take_while(|p| **p == "..").count(); + if parts[climbs..].iter().any(|p| p.is_empty() || *p == "." || *p == "..") { + return Err("has an empty, `.` or `..` component after a name"); + } + // `top/a/link` is in two directories of the package, `top/a` and `top`; + // `top` itself is in none. + let depth = path.split('/').count().checked_sub(2).ok_or("is the package's own directory")?; + if climbs > depth { + return Err("climbs out of the package"); } + Ok(()) } /// The version, read out of the release's own file name. @@ -117,69 +189,173 @@ mod tests { const DIGEST: &str = "99fcd8a7263b5c25cd90cead1baaa7200ef272100fc2226e008a4e8205ba2916"; const ASSET: &str = "gbae-v0.2.0-toyos-x86_64.tar.gz"; + const ROOMY: u64 = 1 << 40; - fn entry(path: &str, kind: Kind, data: &'static [u8], mode: u32) -> Entry<'static> { - Entry { path: path.to_string(), kind, data, mode } + fn entry(path: &str, kind: Kind, size: u64, mode: u32) -> Header { + Header { path: path.to_string(), kind, size, mode } } - fn gbae() -> Vec> { + fn link(path: &str, target: &str) -> Header { + entry(path, Kind::Symlink(target.to_string()), 0, 0o777) + } + + fn gbae() -> Vec
{ vec![ - entry("gbae", Kind::Dir, b"", 0o755), - entry("gbae/gbae", Kind::File, b"ELF...", 0o755), - entry("gbae/README.md", Kind::File, b"# gbae", 0o644), - entry("gbae/LICENSE", Kind::File, b"MIT", 0o644), + entry("gbae", Kind::Dir, 0, 0o755), + entry("gbae/gbae", Kind::File, 6, 0o755), + entry("gbae/README.md", Kind::File, 6, 0o644), + entry("gbae/LICENSE", Kind::File, 3, 0o644), ] } + fn plan_of(file_name: &str, entries: Vec
, free: u64) -> Result { + let mut planner = Planner::new(file_name, free, 4096); + for e in entries { + planner.add(e)?; + } + planner.finish(DIGEST) + } + #[test] fn the_real_asset_plans_the_directory_the_track_describes() { - let plan = plan(ASSET, DIGEST, &gbae()).expect("plan"); + let plan = plan_of(ASSET, gbae(), ROOMY).expect("plan"); assert_eq!(plan.manifest.name, "gbae"); assert_eq!(plan.manifest.version, "0.2.0"); assert_eq!(plan.manifest.digest, DIGEST); assert_eq!(plan.manifest.program, "/apps/gbae/gbae"); - assert_eq!(plan.dirs, ["/apps/gbae"]); - let paths: Vec<&str> = plan.files.iter().map(|(p, _)| p.as_str()).collect(); - assert_eq!(paths, ["/apps/gbae/gbae", "/apps/gbae/README.md", "/apps/gbae/LICENSE"]); + let paths: Vec<&str> = plan.entries.iter().map(|e| e.path.as_str()).collect(); + assert_eq!(paths, ["gbae", "gbae/gbae", "gbae/README.md", "gbae/LICENSE"]); + assert_eq!(plan.need, 3 * 4096); } + /// A toolchain's shape: its program in `bin`, beside a link to it, and no + /// explicit directory entries at all. #[test] - fn an_archive_that_is_not_one_directory_installs_nothing() { + fn a_toolchain_launches_its_bin_and_carries_its_links() { + let entries = vec![ + entry("clang/bin/clang", Kind::File, 10_000, 0o755), + link("clang/bin/cc", "clang"), + link("clang/lib/clang/22/x", "../../../bin/clang"), + entry("clang/sysroot/lib/libtoyos_c.a", Kind::File, 1, 0o644), + ]; + let plan = plan_of("clang-v22.1.0-toyos-x86_64.tar.gz", entries, ROOMY).expect("plan"); + assert_eq!(plan.manifest.program, "/apps/clang/bin/clang"); + assert_eq!(plan.manifest.version, "22.1.0"); + assert_eq!(plan.need, 3 * 4096 + 4096); + } + + #[test] + fn an_archive_that_is_not_one_directory_or_has_no_one_program_installs_nothing() { let mut two = gbae(); - two.push(entry("other/x", Kind::File, b"x", 0o644)); - assert!(plan(ASSET, DIGEST, &two).is_err()); - assert!(plan(ASSET, DIGEST, &[]).is_err()); + two.push(entry("other/x", Kind::File, 1, 0o644)); + assert!(plan_of(ASSET, two, ROOMY).unwrap_err().contains("at its top level")); + assert!(plan_of(ASSET, Vec::new(), ROOMY).is_err()); - let no_program = vec![ - entry("gbae", Kind::Dir, b"", 0o755), - entry("gbae/README.md", Kind::File, b"# gbae", 0o644), - ]; - assert!(plan(ASSET, DIGEST, &no_program).is_err()); + let no_program = vec![entry("gbae", Kind::Dir, 0, 0o755), entry("gbae/README.md", Kind::File, 6, 0o644)]; + assert!(plan_of(ASSET, no_program, ROOMY).unwrap_err().contains("no program to launch")); // A directory where the program should be is not a program, and - // neither is a file the archive never marked executable. - let dir_instead = - vec![entry("gbae", Kind::Dir, b"", 0o755), entry("gbae/gbae", Kind::Dir, b"", 0o755)]; - assert!(plan(ASSET, DIGEST, &dir_instead).is_err()); - let not_executable = - vec![entry("gbae", Kind::Dir, b"", 0o755), entry("gbae/gbae", Kind::File, b"x", 0o644)]; - assert!(plan(ASSET, DIGEST, ¬_executable).is_err()); + // neither is a file the archive never marked executable, nor a link. + for program in [ + entry("gbae/gbae", Kind::Dir, 0, 0o755), + entry("gbae/gbae", Kind::File, 1, 0o644), + link("gbae/gbae", "README.md"), + ] { + let why = plan_of(ASSET, vec![entry("gbae", Kind::Dir, 0, 0o755), program], ROOMY).unwrap_err(); + assert!(why.contains("not an executable file"), "{why}"); + } + + let mut both = gbae(); + both.push(entry("gbae/bin/gbae", Kind::File, 1, 0o755)); + assert!(plan_of(ASSET, both, ROOMY).unwrap_err().contains("which one launches")); } #[test] - fn a_path_the_archive_names_as_both_a_file_and_a_directory_is_refused() { - let mut both = gbae(); - both.push(entry("gbae/README.md", Kind::Dir, b"", 0o755)); - let why = plan(ASSET, DIGEST, &both).err().expect("a path named twice was planned"); - assert!(why.contains("names one path twice"), "{why}"); + fn a_path_named_twice_or_beneath_a_file_or_a_link_is_refused() { + for extra in [ + entry("gbae/README.md", Kind::Dir, 0, 0o755), + entry("gbae/LICENSE", Kind::File, 5, 0o644), + link("gbae/LICENSE", "README.md"), + entry("gbae/gbae", Kind::Dir, 0, 0o755), + entry("gbae", Kind::File, 1, 0o644), + ] { + let mut twice = gbae(); + twice.push(extra.clone()); + let why = plan_of(ASSET, twice, ROOMY).expect_err(&format!("{extra:?} was planned")); + assert!(why.contains("twice"), "{why}"); + } + let mut manifest = gbae(); + manifest.push(entry("gbae/manifest.toml", Kind::File, 1, 0o644)); + assert!(plan_of(ASSET, manifest, ROOMY).unwrap_err().contains("the installer's to write")); + + // A directory given again is the same directory. + let mut again = gbae(); + again.push(entry("gbae", Kind::Dir, 0, 0o755)); + assert!(plan_of(ASSET, again, ROOMY).is_ok()); - let mut twice = gbae(); - twice.push(entry("gbae/LICENSE", Kind::File, b"other", 0o644)); - assert!(plan(ASSET, DIGEST, &twice).is_err()); + // Written beneath a link, a file lands wherever the link points. + for beneath in [ + vec![link("gbae/lib", "share"), entry("gbae/lib/x", Kind::File, 1, 0o644)], + vec![link("gbae/a/up", ".."), entry("gbae/a/up/b/x", Kind::File, 1, 0o644)], + vec![entry("gbae/LICENSE/x", Kind::File, 1, 0o644)], + ] { + let mut entries = gbae(); + entries.extend(beneath); + let why = plan_of(ASSET, entries, ROOMY).unwrap_err(); + assert!(why.contains("which is not a directory"), "{why}"); + } + } - let mut program = gbae(); - program.push(entry("gbae/gbae", Kind::Dir, b"", 0o755)); - assert!(plan(ASSET, DIGEST, &program).is_err()); + /// The links a package may carry are the ones that resolve inside it, + /// whatever the links around them resolve to. + #[test] + fn a_link_whose_target_leaves_the_package_is_refused() { + let top = plan_of(ASSET, vec![link("gbae", "elsewhere")], ROOMY).unwrap_err(); + assert!(top.contains("the package's own directory"), "{top}"); + for (path, target) in [ + ("gbae/x", "/system/bin/supervisor"), + ("gbae/x", ".."), + ("gbae/x", "../other/other"), + ("gbae/a/x", "../../.."), + // `a/..` is `gbae` read as text, and wherever `a`'s link points + // read as the filesystem does. + ("gbae/x", "a/../.."), + ("gbae/x", "a/.."), + ("gbae/x", "./gbae"), + ("gbae/x", "a//b"), + ] { + let mut entries = gbae(); + entries.push(link(path, target)); + let why = plan_of(ASSET, entries, ROOMY).expect_err(&format!("{path} -> {target} was planned")); + assert!(why.contains("link"), "{why}"); + } + for (path, target) in [("gbae/x", "gbae"), ("gbae/a/x", ".."), ("gbae/a/b/x", "../../gbae"), ("gbae/a/x", "../b/c")] + { + let mut entries = gbae(); + entries.push(link(path, target)); + assert!(plan_of(ASSET, entries, ROOMY).is_ok(), "{path} -> {target} was refused"); + } + } + + /// What does not fit is refused at the header that claims it, before a + /// byte of its data is read. + #[test] + fn an_entry_past_the_room_or_past_the_count_is_refused_at_its_header() { + let mut planner = Planner::new(ASSET, 2 * 4096, 4096); + planner.add(entry("gbae/gbae", Kind::File, 4096, 0o755)).unwrap(); + planner.add(entry("gbae/a", Kind::File, 1, 0o644)).unwrap(); + let why = planner.add(entry("gbae/b", Kind::File, 1, 0o644)).unwrap_err(); + assert!(why.contains("needs more than the 8192 bytes free"), "{why}"); + + let huge = entry("gbae/gbae", Kind::File, u64::MAX / 2, 0o755); + assert!(Planner::new(ASSET, ROOMY, 4096).add(huge).is_err()); + + let mut many = Planner::new(ASSET, ROOMY, 4096); + many.add(entry("gbae", Kind::Dir, 0, 0o755)).unwrap(); + for i in 1..MAX_ENTRIES { + many.add(entry(&format!("gbae/{i}"), Kind::Dir, 0, 0o755)).unwrap(); + } + assert!(many.add(entry("gbae/last", Kind::Dir, 0, 0o755)).unwrap_err().contains("more than")); } #[test] @@ -196,7 +372,13 @@ mod tests { /// producing a directory the supervisor cannot resolve. #[test] fn a_digest_the_manifest_cannot_carry_stops_the_install() { - assert!(plan(ASSET, "not-a-digest", &gbae()).is_err()); - assert!(plan(ASSET, &DIGEST.to_uppercase(), &gbae()).is_err()); + let planned = |digest: &str| { + let mut planner = Planner::new(ASSET, ROOMY, 4096); + gbae().into_iter().try_for_each(|e| planner.add(e)).unwrap(); + planner.finish(digest) + }; + assert!(planned(DIGEST).is_ok()); + assert!(planned("not-a-digest").is_err()); + assert!(planned(&DIGEST.to_uppercase()).is_err()); } } diff --git a/system/pkg/src/main.rs b/system/pkg/src/main.rs index e1dc2a89be..5133a1660e 100644 --- a/system/pkg/src/main.rs +++ b/system/pkg/src/main.rs @@ -6,27 +6,34 @@ //! `SHA256SUMS`, ask, unpack, and write the `manifest.toml` that records which //! binary the directory launches. //! +//! **An archive is read twice and never held.** The first read hashes it and +//! plans it within the room `/apps`'s volume has; the second writes it, and has +//! to meet the plan header for header and the digest again at its end, so an +//! archive changed between the two leaves nothing installed. +//! //! The decisions are `lib.rs`'s and are made on the host; what is left here is //! the filesystem and the question put to the user. use std::fs; -use std::io::{BufRead, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::Path; -use flate2::read::GzDecoder; -use pkg::{archive, sums}; +use flate2::bufread::MultiGzDecoder; +use pkg::archive::{self, Kind}; +use pkg::{sums, Plan, Planner}; use toyos_manifest::package::{self, Package}; /// Answers the consent prompt in advance, for a caller with no terminal. /// Spelled out rather than `-y`: installing without asking says so in full. const ASSUME_YES: &str = "--yes"; -const MAX_INFLATED: u64 = 256 * 1024 * 1024; - /// The release publishes one covering every asset, so it is found by sitting /// beside the archive rather than by being named on the command line. const SUMS: &str = "SHA256SUMS"; +/// How much of an archive one read takes, and of a file one write. +const CHUNK: usize = 1 << 20; + fn main() -> std::process::ExitCode { let args: Vec = std::env::args().skip(1).collect(); let words: Vec<&str> = args.iter().map(String::as_str).collect(); @@ -64,20 +71,18 @@ fn install(file: &Path, assume_yes: bool) -> Result<(), String> { .map_err(|e| format!("pkg: cannot read {}: {e}", beside.display()))?; let want = sums::digest_for(&sums_text, &name)?; - let bytes = fs::read(file).map_err(|e| format!("pkg: cannot read {}: {e}", file.display()))?; - let got = sums::hex(&toyos_sha2_hw::sha256_digest(&bytes)); + let mut apps = apps()?; + let room = apps.room().map_err(|e| format!("pkg: {} answers no room: {e:?}", package::DIR))?; + let (planner, got) = survey(file, Planner::new(&name, room.free, room.unit))?; if got != want { return Err(format!("pkg: {name} hashes to {got} and {SUMS} says {want}")); } println!("pkg: verified {name} against {SUMS} ({got})"); - - let tar = inflate(&name, &bytes, MAX_INFLATED)?; - let entries = archive::entries(&tar)?; - let plan = pkg::plan(&name, &got, &entries)?; + let plan = planner.finish(&got)?; let package = &plan.manifest; let dir = Package::dir(&package.name); - if fs::metadata(&dir).is_ok() { + if fs::symlink_metadata(&dir).is_ok() { return Err(format!( "pkg: {dir} is already there — remove it first, this installer does not replace" )); @@ -85,73 +90,152 @@ fn install(file: &Path, assume_yes: bool) -> Result<(), String> { if !assume_yes && !consented(package, file)? { return Err(format!("pkg: not installing {}", package.name)); } - - // The manifest is written last, so a directory carrying one is a finished - // install: the supervisor and the launcher both reach a package through it. - write_package( - &dir, - &plan.dirs, - &plan.files, - &Package::path(&package.name), - &package.render()?, - )?; + let mut durable = || apps.sync().map_err(|e| format!("pkg: {}'s volume did not write what it was given: {e:?}", package::DIR)); + write_package(file, &plan, Path::new(package::DIR), &mut durable)?; println!( - "pkg: installed {} {} at {dir}, launching {}", - package.name, package.version, package.program + "pkg: installed {} {} at {dir} ({} bytes), launching {}", + package.name, package.version, plan.need, package.program ); Ok(()) } -/// Inflate `bytes`, refusing an archive that claims more than `ceiling`. -/// -/// **The inflated size is the archive's claim and `SHA256SUMS` says nothing -/// about it**, so the reader is bounded rather than the output measured after: -/// measuring is the allocation this refuses. -fn inflate(name: &str, bytes: &[u8], ceiling: u64) -> Result, String> { - use std::io::Read; - let mut tar = Vec::new(); - GzDecoder::new(bytes) - .take(ceiling + 1) - .read_to_end(&mut tar) - .map_err(|e| format!("pkg: {name} is not gzip: {e}"))?; - if tar.len() as u64 > ceiling { - return Err(format!("pkg: {name} inflates past {ceiling} bytes and is refused unread")); +/// `/apps`'s own capability, for what std has no word for: the room on its +/// volume, and a sync of the whole volume that answers for every file in it. +fn apps() -> Result { + let names = toyos::endow::namespace() + .ok_or_else(|| String::from("pkg: this process holds no directory capabilities"))?; + let name = format!("{}{}", toyos::fs::CAPABILITY_PREFIX, package::DIR); + toyos::fs::Dir::connect(names, &name).map_err(|e| format!("pkg: cannot reach {}: {e:?}", package::DIR)) +} + +/// The first read: every header through `planner`, and the whole file's +/// digest. +fn survey<'a>(file: &Path, mut planner: Planner<'a>) -> Result<(Planner<'a>, String), String> { + let mut hashed = Hashed::open(file)?; + let mut tar = archive::Reader::new(gunzip(&mut hashed)); + while let Some(header) = tar.next_entry()? { + planner.add(header)?; } - Ok(tar) + drop(tar); + Ok((planner, hashed.finish()?)) } -/// Write a package's directory, and take it back down if any write fails: a -/// verified archive can still run out of volume half way, and a directory with -/// no `manifest.toml` is a name `install` refuses for the volume's life. +/// Unpack `plan` from `file` under `root` and write its manifest last, or take +/// the package's directory back down: a verified archive can still run out of +/// volume half way, and a directory with no `manifest.toml` is a name `install` +/// refuses for the volume's life. Its directory is made first and alone, so a +/// name some other writer took is refused and never swept. +/// +/// **`durable` is asked before the manifest is written**, and refuses if the +/// volume holds any file it could not write: a file's last close can be +/// refused, and std's `File` drops that answer. fn write_package( - dir: &str, - dirs: &[String], - files: &[(String, &[u8])], - manifest: &str, - text: &str, + file: &Path, + plan: &Plan, + root: &Path, + durable: &mut dyn FnMut() -> Result<(), String>, ) -> Result<(), String> { - match write_all(dirs, files, manifest, text) { + let dir = root.join(&plan.manifest.name); + fs::create_dir(&dir).map_err(|e| format!("pkg: cannot create {}: {e}", dir.display()))?; + match unpack(file, plan, root).and_then(|()| durable()).and_then(|()| write_manifest(&dir, &plan.manifest)) { Ok(()) => Ok(()), - Err(why) => match remove_tree(Path::new(dir)) { - Ok(()) => Err(format!("{why}; {dir} was taken back down")), + Err(why) => match remove_tree(&dir) { + Ok(()) => Err(format!("{why}; {} was taken back down", dir.display())), Err(swept) => Err(format!("{why}; {swept}")), }, } } -fn write_all( - dirs: &[String], - files: &[(String, &[u8])], - manifest: &str, - text: &str, -) -> Result<(), String> { - for dir in dirs { - fs::create_dir_all(dir).map_err(|e| format!("pkg: cannot create {dir}: {e}"))?; +/// The second read, writing what the first one planned. +fn unpack(file: &Path, plan: &Plan, root: &Path) -> Result<(), String> { + let changed = || format!("pkg: {} changed since it was verified", file.display()); + let mut hashed = Hashed::open(file)?; + let mut tar = archive::Reader::new(gunzip(&mut hashed)); + let mut buf = vec![0u8; CHUNK]; + for want in &plan.entries { + if tar.next_entry()?.as_ref() != Some(want) { + return Err(changed()); + } + let dest = root.join(&want.path); + let parent = || match dest.parent() { + Some(up) => fs::create_dir_all(up), + None => Ok(()), + }; + let made = match &want.kind { + Kind::Dir => fs::create_dir_all(&dest).map_err(|e| e.to_string()), + // The one link call std has for every OS this compiles for: ToyOS, + // and the host its tests run on. + #[allow(deprecated)] + Kind::Symlink(target) => parent().and_then(|()| fs::soft_link(target, &dest)).map_err(|e| e.to_string()), + Kind::File => parent() + .and_then(|()| fs::File::create_new(&dest)) + .map_err(|e| e.to_string()) + .and_then(|out| copy_data(&mut tar, out, &mut buf)), + }; + made.map_err(|e| format!("pkg: cannot write {}: {e}", dest.display()))?; + } + if tar.next_entry()?.is_some() { + return Err(changed()); + } + drop(tar); + match hashed.finish()? == plan.manifest.digest { + true => Ok(()), + false => Err(changed()), + } +} + +/// The current entry's data, into `out`. +fn copy_data(tar: &mut archive::Reader, mut out: fs::File, buf: &mut [u8]) -> Result<(), String> { + loop { + match tar.read_data(buf)? { + 0 => return Ok(()), + n => out.write_all(&buf[..n]).map_err(|e| e.to_string())?, + } } - for (path, data) in files { - fs::write(path, data).map_err(|e| format!("pkg: cannot write {path}: {e}"))?; +} + +/// The manifest, made durable before the install is called done: written last, +/// so a directory carrying one is a finished install, which the supervisor and +/// the launcher both reach a package through. +fn write_manifest(dir: &Path, package: &Package) -> Result<(), String> { + let path = dir.join(package::FILE); + let text = package.render()?; + fs::File::create_new(&path) + .and_then(|mut f| f.write_all(text.as_bytes()).and_then(|()| f.sync_all())) + .map_err(|e| format!("pkg: cannot write {}: {e}", path.display())) +} + +fn gunzip(r: R) -> MultiGzDecoder> { + MultiGzDecoder::new(BufReader::with_capacity(CHUNK, r)) +} + +/// A file read through a SHA-256 of every byte read. +struct Hashed { + file: fs::File, + hash: toyos_sha2::Sha256, +} + +impl Hashed { + fn open(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|e| format!("pkg: cannot read {}: {e}", path.display()))?; + Ok(Self { file, hash: toyos_sha2_hw::sha256() }) + } + + /// The whole file's digest: what was read, and the rest, read now. A tar + /// ends before its file does, in the zeros gzip and tar pad it with, and + /// `SHA256SUMS` covers every byte. + fn finish(mut self) -> Result { + std::io::copy(&mut self, &mut std::io::sink()).map_err(|e| format!("pkg: {e}"))?; + Ok(sums::hex(&self.hash.finalize())) + } +} + +impl Read for Hashed { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.file.read(buf)?; + self.hash.update(&buf[..n]); + Ok(n) } - fs::write(manifest, text).map_err(|e| format!("pkg: cannot write {manifest}: {e}")) } /// The question, asked where the user typed the command. A closed or empty @@ -195,8 +279,8 @@ fn remove(name: &str) -> Result<(), String> { Ok(()) } -/// How deep this walk goes. Policy on the primitive: any process can plant a -/// Delete a directory and everything under it. +/// Delete a directory and everything under it, a link as itself and never +/// what it points at. /// /// **Not `fs::remove_dir_all`**: that one empties a directory and leaves it /// (`issues/remove-dir-all-empties-a-directory-and-leaves-it.md`), @@ -261,21 +345,177 @@ mod tests { use super::*; use toyos_tmpdir::TempDir; + const ASSET: &str = "tool-v1.0-toyos-x86_64.tar.gz"; + + /// A gzipped tar of `(path, data)` files and `(path, target)` links, as a + /// writer outside this crate makes one. + fn gzipped(files: &[(&str, &[u8])], links: &[(&str, &str)]) -> Vec { + let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast())); + for (path, data) in files { + let mut header = tar::Header::new_ustar(); + header.set_size(data.len() as u64); + header.set_mode(if path.ends_with("tool") { 0o755 } else { 0o644 }); + tar.append_data(&mut header, path, *data).expect("a file"); + } + for (path, target) in links { + let mut header = tar::Header::new_ustar(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + tar.append_link(&mut header, path, target).expect("a link"); + } + tar.into_inner().and_then(|gz| gz.finish()).expect("an archive") + } + + /// A toolchain's shape: its program in `bin`, a link to it, and a file + /// past several reads, whose bytes `seed` decides. + fn tool(seed: u32) -> Vec { + let big: Vec = (0..3 * CHUNK as u32 + 7).map(|i| ((i + seed) * 31 % 251) as u8).collect(); + gzipped(&[("tool/bin/tool", b"ELF"), ("tool/share/big", &big)], &[("tool/bin/cc", "tool")]) + } + + /// The archive at `scratch/ASSET`, surveyed on a volume of `free` bytes. + fn surveyed(scratch: &Path, bytes: &[u8], free: u64) -> Result { + let file = scratch.join(ASSET); + fs::write(&file, bytes).unwrap(); + let (planner, digest) = survey(&file, Planner::new(ASSET, free, 4096))?; + planner.finish(&digest) + } + + #[test] + fn an_archive_installs_whole_its_files_its_links_and_its_manifest() { + let scratch = TempDir::new("pkg-install"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + assert_eq!(plan.manifest.program, "/apps/tool/bin/tool"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect("installed"); + + let big = fs::read(root.join("tool/share/big")).unwrap(); + assert_eq!(big.len(), 3 * CHUNK + 7); + assert!(big.iter().enumerate().all(|(i, &b)| b == (i as u32 * 31 % 251) as u8)); + assert_eq!(fs::read_link(root.join("tool/bin/cc")).unwrap(), Path::new("tool")); + assert_eq!(fs::read(root.join("tool/bin/cc")).unwrap(), b"ELF"); + let manifest = fs::read_to_string(root.join("tool/manifest.toml")).unwrap(); + assert_eq!(Package::parse(&manifest).unwrap(), plan.manifest); + } + + /// An archive replaced between the read that verified it and the one that + /// writes it installs nothing. + #[test] + fn an_archive_changed_after_its_survey_installs_nothing() { + let scratch = TempDir::new("pkg-changed"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + let other = gzipped(&[("tool/bin/tool", b"ELF"), ("tool/share/big", b"short")], &[]); + // The same headers over other bytes: only the digest tells. + for replaced in [other, tool(1)] { + fs::write(scratch.join(ASSET), &replaced).unwrap(); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("a changed archive installed"); + assert!(why.contains("was taken back down"), "{why}"); + assert!(!root.join("tool").exists(), "a changed archive left its directory: {why}"); + } + } + + /// The second read writes only what the first one checked: an archive + /// swapped for one that plants a link and writes through it reaches + /// nothing outside the package, though its digest is only known at its end. + #[test] + fn a_swapped_archive_writes_nothing_its_plan_did_not_check() { + let scratch = TempDir::new("pkg-swapped"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let (root, outside) = (scratch.join("apps"), scratch.join("outside")); + fs::create_dir(&root).unwrap(); + fs::create_dir(&outside).unwrap(); + let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast())); + let mut link = tar::Header::new_ustar(); + link.set_entry_type(tar::EntryType::Symlink); + link.set_size(0); + link.set_mode(0o777); + tar.append_link(&mut link, "tool/out", "../../outside").unwrap(); + let mut file = tar::Header::new_ustar(); + file.set_size(1); + file.set_mode(0o644); + tar.append_data(&mut file, "tool/out/planted", &b"x"[..]).unwrap(); + fs::write(scratch.join(ASSET), tar.into_inner().and_then(|gz| gz.finish()).unwrap()).unwrap(); + + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("a swapped archive installed"); + assert!(why.contains("changed since it was verified"), "{why}"); + assert!(!outside.join("planted").exists(), "the swapped archive wrote through its link"); + assert!(!root.join("tool").exists()); + } + + /// The digest is of every byte of the file, past the tar's end and the + /// gzip member's, which the archive's reader never asks for; a cut archive + /// is refused where it ends. #[test] - fn an_install_that_cannot_finish_takes_its_directory_back_down() { - let root = TempDir::new("pkg-unwind"); - let dir = root.join("gbae"); - let dir_s = dir.to_str().unwrap().to_string(); - let good = format!("{dir_s}/gbae"); - // `gbae` is a file, so a write to `gbae/inside` cannot succeed. - let doomed = format!("{good}/inside"); - let files: Vec<(String, &[u8])> = vec![(good.clone(), b"ELF".as_slice()), (doomed, b"x")]; - - let why = write_package(&dir_s, &[dir_s.clone()], &files, "unreached", "unreached") - .expect_err("the second write cannot succeed"); - assert!(why.contains("was taken back down"), "{why}"); - assert!(!dir.exists(), "{dir_s} survived a failed install"); - assert!(root.exists(), "the unwind went past the package's own directory"); + fn the_digest_covers_the_whole_file_and_a_cut_archive_is_refused() { + let scratch = TempDir::new("pkg-truncated"); + let file = scratch.join(ASSET); + let survey_of = |bytes: &[u8]| { + fs::write(&file, bytes).unwrap(); + survey(&file, Planner::new(ASSET, 1 << 30, 4096)).map(|(_, digest)| digest) + }; + let mut whole = tool(0); + whole.extend(vec![0u8; 3 * CHUNK]); + assert_eq!(survey_of(&whole).unwrap(), sums::hex(&toyos_sha2_hw::sha256_digest(&whole))); + assert!(survey_of(&whole[..tool(0).len() / 3]).unwrap_err().contains("tar:")); + assert!(survey_of(b"not gzip at all").is_err()); + } + + /// The room is the bound on what a survey inflates: a header claiming + /// more is refused at that header, and nothing past it is read. + #[test] + fn an_archive_bigger_than_the_room_is_refused_unread() { + let scratch = TempDir::new("pkg-room"); + let why = surveyed(&scratch, &tool(0), 64 * 1024).expect_err("3 MiB planned into 64 KiB"); + assert!(why.contains("needs more than the 65536 bytes free"), "{why}"); + + // A bomb: a 4 GiB file of zeros, a few megabytes compressed, refused + // at its header. + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); + let mut header = tar::Header::new_ustar(); + header.set_path("tool/bin/tool").unwrap(); + header.set_size(4 << 30); + header.set_mode(0o755); + header.set_cksum(); + gz.write_all(header.as_bytes()).unwrap(); + let zeros = vec![0u8; 1 << 20]; + for _ in 0..64 { + gz.write_all(&zeros).unwrap(); + } + let bomb = gz.finish().unwrap(); + let why = surveyed(&scratch, &bomb, 1 << 30).expect_err("a 4 GiB entry planned into 1 GiB"); + assert!(why.contains("needs more than"), "{why}"); + } + + /// A volume that could not write every file of the package installs + /// nothing, though every write was answered. + #[test] + fn a_volume_that_refuses_its_sync_installs_nothing() { + let scratch = TempDir::new("pkg-durable"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + let mut refused = || Err(String::from("pkg: the volume did not write it")); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut refused).expect_err("installed unwritten"); + assert!(why.contains("did not write it") && why.contains("was taken back down"), "{why}"); + assert!(!root.join("tool").exists()); + } + + /// A name some other writer holds is refused and left as it was found: + /// only a directory this install made is one its unwind may take down. + #[test] + fn a_directory_already_at_the_name_is_refused_and_never_swept() { + let scratch = TempDir::new("pkg-taken"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir_all(root.join("tool")).unwrap(); + fs::write(root.join("tool/theirs"), b"x").unwrap(); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("an existing directory"); + assert!(why.contains("cannot create"), "{why}"); + assert!(root.join("tool/theirs").exists(), "a directory the installer did not make was swept"); } /// 33 is the first depth the old bound refused; 256 is far past it. @@ -294,23 +534,16 @@ mod tests { // A second branch, so the walk is a tree and not a chain. fs::create_dir_all(root.join("wide/er")).expect("a second branch"); fs::write(root.join("wide/er/leaf"), b"y").expect("a second leaf"); + // A link to outside the tree goes as itself. + let outside = scratch.join("outside"); + fs::create_dir(&outside).unwrap(); + fs::write(outside.join("kept"), b"z").unwrap(); + #[allow(deprecated)] + fs::soft_link(&outside, root.join("wide/out")).unwrap(); remove_tree(&root).unwrap_or_else(|e| panic!("depth {depth}: {e}")); assert!(!root.exists(), "a tree {depth} deep survived its removal"); + assert!(outside.join("kept").exists(), "the removal followed a link"); } } - - #[test] - fn an_archive_that_inflates_past_the_ceiling_is_refused_unread() { - use std::io::Write; - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); - gz.write_all(&vec![0u8; 64 * 1024]).expect("deflate"); - let bomb = gz.finish().expect("a gzip stream"); - assert!(bomb.len() < 1024, "the fixture is not a bomb: {} bytes", bomb.len()); - - let why = inflate("bomb.tar.gz", &bomb, 4096).expect_err("64 KiB past a 4 KiB ceiling"); - assert!(why.contains("inflates past 4096 bytes"), "{why}"); - assert_eq!(inflate("bomb.tar.gz", &bomb, 128 * 1024).expect("under it").len(), 64 * 1024); - assert!(inflate("not.tar.gz", b"not gzip at all", 4096).is_err()); - } } diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs index ffbf72ba93..3178859ed1 100644 --- a/tests/common/hostedclang.rs +++ b/tests/common/hostedclang.rs @@ -1,20 +1,18 @@ -//! clang and LLD built for ToyOS (`build/hostedclang.rs`), run inside a guest: -//! M2 of `issues/toyos-builds-itself.md`. +//! clang and LLD built for ToyOS (`build/hostedclang.rs`), installed by `pkg` +//! and run inside a guest: M2 of `issues/toyos-builds-itself.md`. //! -//! The binaries, clang's resource headers and the C sysroot's headers and -//! `libtoyos_c.a` go on the ROOT of `tests/images/hostedclangcase.toml`, as fixtures -//! beside the image's own files: ROOT is memory the loader filled, so the -//! guest reads them with no disk and no server behind it. The job compiles -//! `hello.c` there and has the driver link it, which starts LLD as a child, -//! runs it, and compares each product with what the host's clang and LLD, of -//! the same LLVM commit, made of the same input on the same command line. +//! The package and its `SHA256SUMS` go on the ROOT of +//! `tests/images/hostedclangcase.toml` as fixtures, beside `hello.c` and what +//! the host's clang and LLD, of the same LLVM commit, made of it on the same +//! command line. The job installs the package into `/apps` on DATA with +//! `/system/bin/pkg`, compiles `hello.c` with the installed clang, has its +//! driver link it, which starts LLD as a child, runs it, and compares each +//! product with the host's. use std::fs; -use std::path::Path; use std::process::Command; use std::time::Duration; - use super::{compile, qemu}; use qemu::{BootOptions, QemuInstance}; @@ -27,51 +25,51 @@ const STAGED: &str = "share/hosted-clang"; /// What `hello.c` prints. const HELLO: &str = "hello from a C program compiled and linked inside ToyOS"; -/// The host's half: its clang and LLD on `hello.c`, the guest's on the same -/// file with the same arguments, and the guest's verdict on their bytes. +/// DATA's size: the installed toolchain is about 300 MB, which the profile's +/// 128 MiB does not hold. +const DATA_BYTES: u64 = 1 << 30; + +/// The host's half: the package, its clang and LLD on `hello.c`, and the +/// guest's verdict on the bytes its own made of the same file with the same +/// arguments. pub fn hosted_clang_hello() -> Result<(), String> { let hosted = toyos_build::hostedclang::ensure(&compile::repo_root()); - let c = compile::c_sysroot(); - let (case, sources) = (compile::repo_root().join("tests/images/hostedclangcase.toml"), compile::repo_root().join("tests/guest/c")); + let sources = compile::repo_root().join("tests/guest/c"); + let case = compile::repo_root().join("tests/images/hostedclangcase.toml"); let scratch = super::lane::dir().join(JOB); fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; let (object, linked) = (scratch.join("hello.o"), scratch.join("a.out")); for (args, to) in [(&["-c"][..], &object), (&[][..], &linked)] { - let mut cc = Command::new(&c.clang); - cc.args(c.args()).args(args).arg("hello.c").arg("-o").arg(to).current_dir(&sources); + let mut cc = Command::new(&hosted.c.clang); + cc.args(hosted.c.args()).args(args).arg("hello.c").arg("-o").arg(to).current_dir(&sources); host_run(cc)?; } + let packaged = hosted.package(&scratch.join("package")); let mut staged: Vec<(String, Vec)> = Vec::new(); - let mut stage = |rel: &str, from: &Path| { - let bytes = fs::read(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())); + let mut stage = |rel: &str, from: &std::path::Path| -> Result<(), String> { + let bytes = fs::read(from).map_err(|e| format!("read {}: {e}", from.display()))?; staged.push((format!("{STAGED}/{rel}"), bytes)); + Ok(()) }; - stage("bin/clang", &hosted.dir.join("bin/clang")); - stage("bin/ld.lld", &hosted.dir.join("bin/ld.lld")); - stage("sysroot/lib/libtoyos_c.a", &c.dir.join("lib/libtoyos_c.a")); - stage("src/hello.c", &sources.join("hello.c")); + let archive = packaged.archive.file_name().and_then(|n| n.to_str()).expect("a package file name").to_string(); + stage(&archive, &packaged.archive)?; + stage("SHA256SUMS", &packaged.sums)?; + stage("src/hello.c", &sources.join("hello.c"))?; for (name, product) in [("hello.o", &object), ("a.out", &linked)] { let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?; let sha256: String = toyos_sha2_hw::sha256_digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {sha256}", bytes.len()); - stage(&format!("host/{name}"), product); - } - // clang finds its resource headers beside itself, `../lib/clang`. - for (rel, from) in tree(&hosted.dir.join("lib/clang")) { - stage(&format!("lib/clang/{rel}"), &from); - } - // The C headers alone: `include/c++` is libc++'s, which no C compile reads. - for (rel, from) in tree(&c.dir.join("include")).into_iter().filter(|(rel, _)| !rel.starts_with("c++/")) { - stage(&format!("sysroot/include/{rel}"), &from); + stage(&format!("host/{name}"), product)?; } + eprintln!(" [{JOB}] the package: {archive}, {}", fs::read_to_string(&packaged.sums).unwrap_or_default().trim()); fs::remove_dir_all(&scratch).map_err(|e| format!("remove {}: {e}", scratch.display()))?; - let options = BootOptions { extra_root_files: staged, ..Default::default() }; + let options = BootOptions { extra_root_files: staged, data_bytes: Some(DATA_BYTES), ..Default::default() }; let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); let mut qemu = QemuInstance::boot_with_options(&case, &[], &[(JOB.to_string(), bin)], options); - let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED}"), Duration::from_secs(600)); + let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED} {archive}"), Duration::from_secs(900)); if let Some(why) = &result.error { return Err(format!("{why}\nthe job said:\n{}", result.stdout)); } @@ -83,7 +81,7 @@ pub fn hosted_clang_hello() -> Result<(), String> { return Err(format!("the job never said `{said}`:\n{}", result.stdout)); } } - for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: "))) { + for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: ")) || l.contains("pkg: ")) { eprintln!(" [{JOB}] {}", line.trim()); } Ok(()) @@ -102,21 +100,3 @@ fn host_run(mut command: Command) -> Result<(), String> { } Ok(()) } - -/// Every file under `dir`, by its path relative to `dir`. -fn tree(dir: &Path) -> Vec<(String, std::path::PathBuf)> { - let mut files = Vec::new(); - let mut pending = vec![dir.to_path_buf()]; - while let Some(at) = pending.pop() { - for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { - let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); - if path.is_dir() { - pending.push(path); - } else { - let rel = path.strip_prefix(dir).expect("under the walked directory"); - files.push((rel.to_string_lossy().into_owned(), path)); - } - } - } - files -} diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 5552b48ff6..ba2940e518 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -1175,6 +1175,9 @@ pub struct BootOptions { /// the image is memoized on their names and bytes, so two boots staging /// different fixtures do not share one. pub extra_root_files: Vec<(String, Vec)>, + /// DATA's size in place of the profile's, for a boot that writes more + /// than [`NVME_SMALL`] holds, on a profile whose disk carries the image. + pub data_bytes: Option, /// Have QEMU record every PSCI call a vCPU makes into this file, with the /// calling CPU's affinity: the firmware side's own account of what the /// kernel asked of it. @@ -1219,6 +1222,7 @@ impl Default for BootOptions { mute: false, ready_marker: DEFAULT_READY, extra_root_files: Vec::new(), + data_bytes: None, psci_trace: None, rtc_base: None, } @@ -1452,7 +1456,11 @@ impl QemuInstance { ¶ms, options.debug_wait, ); - let storage = options.profile.shape().storage; + let storage = match (options.profile.shape().storage, options.data_bytes) { + (storage, None) => storage, + (Storage::Disk { .. }, Some(data_bytes)) => Storage::Disk { data_bytes }, + (Storage::Stick { .. }, Some(_)) => panic!("[qemu] a DATA size is a profile's whose disk carries its image"), + }; match storage { Storage::Stick { .. } => fs::write(&boot_image, image).expect("Failed to write test boot image"), Storage::Disk { data_bytes } => toyos_build::image::install(&image, &boot_image, data_bytes), diff --git a/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs b/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs index 10a9aebb3e..cb79eb15d2 100644 --- a/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs +++ b/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs @@ -1,41 +1,46 @@ -//! clang built for ToyOS compiles a C program here, and its driver links one, -//! starting `ld.lld` as a child; the program runs, and what each wrote is the -//! host's clang's bytes. +//! clang built for ToyOS, installed by `pkg`, compiles a C program here, and +//! its driver links one, starting `ld.lld` as a child; the program runs, and +//! what each wrote is the host's clang's bytes. //! -//! Its one argument is the directory the harness staged on ROOT: `bin/clang` -//! and `bin/ld.lld` with clang's resource headers at `lib/clang`, the C -//! sysroot at `sysroot`, the program at `src/hello.c`, and what the host's -//! clang made of it at `host/hello.o` and `host/a.out`. +//! Its arguments are the directory the harness staged on ROOT and the package +//! file in it: the package beside its `SHA256SUMS`, the program at +//! `src/hello.c`, and what the host's clang made of it at `host/hello.o` and +//! `host/a.out`. use std::path::{Path, PathBuf}; use std::process::Command; -/// Where this job writes: `/tmp` is the guest's one writable tree. +/// Where this job writes: `/tmp` is the guest's one writable tree outside the +/// package. const OUT: &str = "/tmp/hosted_clang_hello"; +/// The installed toolchain's `bin`. +const BIN: &str = "/apps/clang/bin"; + fn main() { - let staged = PathBuf::from(std::env::args().nth(1).expect("the staged directory, as the one argument")); - let sysroot = staged.join("sysroot"); + let mut args = std::env::args().skip(1); + let staged = PathBuf::from(args.next().expect("the staged directory, as the first argument")); + let package = staged.join(args.next().expect("the package's file name, as the second argument")); + let mut install = Command::new("/system/bin/pkg"); + install.arg("install").arg(&package).arg("--yes"); + run(install, "pkg install"); + let out = Path::new(OUT); std::fs::create_dir_all(out).unwrap_or_else(|e| panic!("create {OUT}: {e}")); let (object, linked) = (out.join("hello.o"), out.join("a.out")); - // The arguments are the host's, so the two products can be compared. - let clang = |args: &[&str], to: &Path| { - let mut command = Command::new(staged.join("bin/clang")); - command - .arg("--target=x86_64-unknown-toyos") - .arg(format!("--sysroot={}", sysroot.display())) - .args(args) - .arg("hello.c") - .arg("-o") - .arg(to) - .current_dir(staged.join("src")); + // The arguments are the host's but its target and sysroot, which are the + // installed clang's own: its default target, and the sysroot its + // configuration file names. + let compile = |driver: &str, args: &[&str], to: &Path| { + let mut command = Command::new(Path::new(BIN).join(driver)); + command.args(args).arg("hello.c").arg("-o").arg(to).current_dir(staged.join("src")); command }; - run(clang(&["-c"], &object), "clang -c"); - // `clang hello.c`: the driver compiles, and links by starting `ld.lld`. - run(clang(&[], &linked), "clang"); + run(compile("clang", &["-c"], &object), "clang -c"); + // `cc hello.c`: the driver by the link the package carries for it, + // compiling, and linking by starting `ld.lld`. + run(compile("cc", &[], &linked), "cc"); let ran = Command::new(&linked).output().unwrap_or_else(|e| panic!("run {}: {e}", linked.display())); assert!(ran.status.success(), "{} ended {:?}: {}", linked.display(), ran.status, String::from_utf8_lossy(&ran.stderr)); @@ -44,7 +49,7 @@ fn main() { } let mut differ = false; - for (made, by) in [(&object, "clang -c"), (&linked, "clang")] { + for (made, by) in [(&object, "clang -c"), (&linked, "cc")] { let name = made.file_name().expect("a file name"); differ |= !same(made, &staged.join("host").join(name), by); } @@ -52,16 +57,13 @@ fn main() { println!("hosted_clang_hello: ok"); } -/// Run `command`, and refuse its failure with what it said. +/// Run `command`, say what it printed, and refuse its failure. fn run(mut command: Command, what: &str) { let done = command.output().unwrap_or_else(|e| panic!("{what} did not start: {e}")); - assert!( - done.status.success(), - "{what} ended {:?}:\n{}{}", - done.status, - String::from_utf8_lossy(&done.stdout), - String::from_utf8_lossy(&done.stderr) - ); + for line in String::from_utf8_lossy(&done.stdout).lines() { + println!("hosted_clang_hello: {what}: {line}"); + } + assert!(done.status.success(), "{what} ended {:?}:\n{}", done.status, String::from_utf8_lossy(&done.stderr)); println!("hosted_clang_hello: {what} ended 0"); } diff --git a/tests/images/hostedclangcase.toml b/tests/images/hostedclangcase.toml index 95de68ebdc..9447402b5d 100644 --- a/tests/images/hostedclangcase.toml +++ b/tests/images/hostedclangcase.toml @@ -1,11 +1,30 @@ -# The boot `hosted_clang_hello` stages ToyOS's own clang and LLD on, with the C -# sysroot they read: a ROOT of its own, since they are a quarter of a gigabyte -# and no other test reads them. +# The boot `hosted_clang_hello` installs ToyOS's own clang and LLD in, by `pkg` +# from the package staged on its ROOT: a ROOT of its own, since the package is +# a hundred megabytes and no other test reads it, and DATA, where `/apps` is. [boot] -start = ["logkeeper", "test-runner"] +start = ["logkeeper", "diskserver", "fileserver", "test-runner"] [programs.logkeeper] service = true syscap = ["logread"] [programs.test-runner] + +# The installer the job runs, as `images/system.toml` declares it. +[programs.pkg] + +# The block service, the NVMe driver for the disk DATA is on, and the file +# servers. +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +[programs.nvme] +drives = ["pci:01:08:02/1b36:0010"] +receives = ["disk"] + +[programs.fileserver] +restart = true +roles = ["data", "log", "boot"] +receives = ["block"] diff --git a/tests/toyos.rs b/tests/toyos.rs index 2998c713f9..156991dfe9 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -212,8 +212,8 @@ const RUST_SKIP: &[&str] = &[ // must not take from its members: `bar_map_again` and the // `msix_table_read_in_place` metal row run it. "msix_in_place", - // It needs ToyOS's own clang and LLD on ROOT, which only its boot stages: - // `hosted_clang_hello` runs it on tests/images/hostedclangcase.toml. + // It needs ToyOS's own clang and LLD packaged on ROOT, which only its boot + // stages: `hosted_clang_hello` runs it on tests/images/hostedclangcase.toml. "hosted_clang_hello", // A std child's stdio, beside libc's: `libc_children` runs it on // tests/images/libccase.toml. @@ -529,9 +529,11 @@ const MACHINE_TESTS: &[&str] = &[ // block service's sessions through have no host build, and the T14 boots // from a stick beside an NVMe disk that is another system's. "nvme_disk_boots", - // clang and LLD built for ToyOS, compiling and linking inside it: the - // claim is that they run on ToyOS, which no host test can make, and the - // T14 would read nothing a guest does not. On demand: [`ON_DEMAND`]. + // clang and LLD built for ToyOS, installed by `pkg` and compiling and + // linking inside it: the claim is that they install and run on ToyOS, + // which no host test can make, and the T14 would read nothing a guest does + // not. On demand: [`ON_DEMAND`]. Red at the link until item 5 of + // `issues/toyos-builds-itself.md` lands. "hosted_clang_hello", // libc's children: each program starts and waits for its own, which is // order, completion, contents and counts, and no hardware is in it.