From 3cbebc5b3c7250026551f7017e1712634d790cc0 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 21 Oct 2026 06:23:31 +0200 Subject: [PATCH] pkg streams a package into /apps within its volume's room, and packages the toolchain pkg read a whole archive into memory under a 256 MiB ceiling and refused every link, so the toolchain M2 needs (clang, LLD and the C sysroot, 279 MB installed) could not be installed. pkg now reads an archive twice and never holds it: a survey hashes the whole file and plans it header by header within the room the volume under /apps has, and the unpack meets that plan header for header and the digest again at the end, so an archive changed between the reads installs nothing and writes nothing the plan did not check. The room replaces MAX_INFLATED as the bound on what a survey inflates. Before the manifest is written the volume is synced through /apps's capability, which refuses if any file was left unwritten: a file's last close can be refused, and std's File drops that answer. The room is a new file-server request, ROOM: the free bytes of the volume behind a capability and its allocation unit, answered by bcachefs's spare blocks, FAT32's free clusters, and none for an absent or read-only volume. A package may carry symlinks that resolve inside it: relative, `..` only first and never past the package's own directory, and nothing is written beneath a link or a file. The package's directory is no link, and the manifest is the installer's alone. Hard links, pax and GNU long names stay refused. A package's program is `/` or `/bin/`, so a manifest's program may be nested inside its directory. pkg asks ToyOS's file server for room, so it is declared exempt as ToyOS-only. `--hosted-clang` packages its clang and LLD with clang's resource headers, the C sysroot and LLVM's licence as clang-v-toyos-x86_64.tar.gz, deterministic, with `cc` and `clang++` linked to clang and a `.cfg` naming the sysroot beside the binaries. hosted_clang_hello installs that package with pkg on a 1 GiB DATA and compiles and links hello.c with /apps/clang/bin. Its image starts the block service and the NVMe driver as the split of #971 has them. The guest row installs and compiles, and stops at the link: clang's driver, starting cc1 from /apps through libc's posix_spawn of a served image (toyos::files::read_image), is refused ENOMEM, while a small std parent starts ld.lld from there. The image object is allocated physically contiguous, about 148 MB for clang, and the memory track's stages 1 to 3 remove that; the track and the memory issue record it as the row's blocker. The package's licences and a command reached by its name are recorded as owed. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 3 + build/hostedclang.rs | 214 ++++++++- ...der-apps-and-the-installer-is-a-program.md | 5 +- ...b-pages-and-that-caps-the-process-count.md | 10 + issues/toyos-builds-itself.md | 77 ++-- lib/bcachefs/src/alloc_bitmap.rs | 2 +- lib/bcachefs/src/fs.rs | 6 + lib/manifest/src/package.rs | 15 +- sdk/toyos/src/fs.rs | 24 + system/fileserver/src/absent.rs | 4 + system/fileserver/src/data.rs | 19 + system/fileserver/src/fat.rs | 9 + system/fileserver/src/main.rs | 4 + system/fileserver/src/rights.rs | 8 +- system/fileserver/src/volume.rs | 3 + system/pkg/Cargo.toml | 10 +- system/pkg/src/archive.rs | 312 +++++++++---- system/pkg/src/lib.rs | 392 ++++++++++++----- system/pkg/src/main.rs | 415 ++++++++++++++---- tests/common/hostedclang.rs | 86 ++-- tests/common/qemu.rs | 10 +- .../rust-tests/src/bin/hosted_clang_hello.rs | 68 +-- tests/images/hostedclangcase.toml | 27 +- tests/toyos.rs | 12 +- 24 files changed, 1288 insertions(+), 447 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a22e5bcdf29..6d86fab5c0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3011,7 +3011,10 @@ name = "pkg" version = "0.1.0" dependencies = [ "flate2", + "tar", + "toyos", "toyos-manifest", + "toyos-sha2", "toyos-sha2-hw", "toyos-tmpdir", ] diff --git a/build/hostedclang.rs b/build/hostedclang.rs index 13969397e8f..539dd8aafb4 100644 --- a/build/hostedclang.rs +++ b/build/hostedclang.rs @@ -22,8 +22,13 @@ //! `hosted-clang//` in the store holds `bin/clang`, `bin/ld.lld` and //! clang's resource headers in `lib/clang//include`, where clang looks //! beside itself; once its [`SOURCE`] file exists it is read-only. +//! +//! **What a ToyOS machine installs is a package of it** ([`HostedClang::package`]): +//! `clang-v-toyos-x86_64.tar.gz` beside the `SHA256SUMS` that verifies +//! it, which `/system/bin/pkg` installs as `/apps/clang`. use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; @@ -92,11 +97,17 @@ const BINARIES: [(&str, &str); 2] = [("clang", "clang"), ("lld", "ld.lld")]; /// A ToyOS-hosted clang and LLD, held in use for as long as this lives. pub struct HostedClang { pub dir: PathBuf, + /// The C sysroot it was built against, which is the one it compiles + /// against. + pub c: CSysroot, + /// The licence of the LLVM it was built from: clang's, LLD's, libc++'s + /// and the resource headers'. + licence: PathBuf, _using: Guard, } -/// `cargo run -- --hosted-clang`: make this tree's, and name each binary with -/// its size. +/// `cargo run -- --hosted-clang`: make this tree's, name each binary with its +/// size, and package it in `target/hosted-clang`. pub fn dispatch(root: &Path) { let hosted = ensure(root); for (_, kept) in BINARIES { @@ -104,6 +115,111 @@ pub fn dispatch(root: &Path) { let size = fs::metadata(&binary).unwrap_or_else(|e| panic!("stat {}: {e}", binary.display())).len(); println!("{} ({size} bytes)", binary.display()); } + let packaged = hosted.package(&root.join("target/hosted-clang")); + println!("{}, verified by {}", packaged.archive.display(), packaged.sums.display()); +} + +/// The package in its directory: the archive, and `SHA256SUMS` beside it. +pub struct Packaged { + pub archive: PathBuf, + pub sums: PathBuf, +} + +/// The package's name: the directory it is under `/apps`, and the program in +/// its `bin` that a launch starts. +const PACKAGE: &str = "clang"; + +/// The links of the package's `bin`: the driver's other names, which it reads +/// its language from. +const LINKS: [(&str, &str); 2] = [("cc", "clang"), ("clang++", "clang")]; + +impl HostedClang { + /// Write the package and its `SHA256SUMS` into `into`, and answer where. + /// **A function of the product and the sysroot**: every entry in one + /// order, with no owner or time, so one toolchain packs to one digest. + /// + /// Beside the binaries it holds clang's resource headers where clang looks + /// for them, `lib/clang`, and the sysroot's `include` and `lib` at + /// `sysroot`, which the configuration file clang reads beside itself names + /// (`.cfg`, `clang/docs/UsersManual.rst`, "Configuration files"), + /// so a compile there needs no `--sysroot`. + pub fn package(&self, into: &Path) -> Packaged { + pack(&self.dir, &self.c.dir, &self.licence, &said_version(&self.c.clang).0, into) + } +} + +/// [`HostedClang::package`] of the product at `product` and the C sysroot at +/// `sysroot`, LLVM `version` under the licence at `licence`. +fn pack(product: &Path, sysroot: &Path, licence: &Path, version: &str, into: &Path) -> Packaged { + let name = format!("{PACKAGE}-v{version}-toyos-{}.tar.gz", TARGET.name()); + fs::create_dir_all(into).unwrap_or_else(|e| panic!("create {}: {e}", into.display())); + let archive = into.join(&name); + let file = fs::File::create(&archive).unwrap_or_else(|e| panic!("create {}: {e}", archive.display())); + let gzip = flate2::write::GzEncoder::new(std::io::BufWriter::new(file), flate2::Compression::default()); + let mut tar = tar::Builder::new(gzip); + let mut append = |path: &str, mode: u32, data: &[u8]| { + let mut header = tar::Header::new_ustar(); + header.set_size(data.len() as u64); + header.set_mode(mode); + header.set_mtime(0); + tar.append_data(&mut header, format!("{PACKAGE}/{path}"), data).unwrap_or_else(|e| panic!("pack {path}: {e}")); + }; + append("LICENSE.TXT", 0o644, &read(licence)); + for (_, kept) in BINARIES { + append(&format!("bin/{kept}"), 0o755, &read(&product.join("bin").join(kept))); + } + append(&format!("bin/{}.cfg", TARGET.userland()), 0o644, b"--sysroot=/../sysroot\n"); + for (rel, from) in files(&product.join("lib/clang")) { + append(&format!("lib/clang/{rel}"), 0o644, &read(&from)); + } + for part in ["include", "lib"] { + for (rel, from) in files(&sysroot.join(part)) { + append(&format!("sysroot/{part}/{rel}"), 0o644, &read(&from)); + } + } + for (link, target) in LINKS { + let mut header = tar::Header::new_ustar(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_mtime(0); + tar.append_link(&mut header, format!("{PACKAGE}/bin/{link}"), target) + .unwrap_or_else(|e| panic!("pack bin/{link}: {e}")); + } + let packed = tar.into_inner().and_then(|gzip| gzip.finish()).and_then(|mut file| file.flush()); + packed.unwrap_or_else(|e| panic!("{}: {e}", archive.display())); + + let digest: String = toyos_sha2_hw::sha256_digest(read(&archive)).iter().map(|b| format!("{b:02x}")).collect(); + let sums = into.join("SHA256SUMS"); + fs::write(&sums, format!("{digest} {name}\n")).unwrap_or_else(|e| panic!("write {}: {e}", sums.display())); + Packaged { archive, sums } +} + +fn read(path: &Path) -> Vec { + fs::read(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +/// Every file under `dir`, by its path relative to `dir`, in sorted order. A +/// link in a product is a defect of its maker, refused here rather than packed +/// as the file it points at. +fn files(dir: &Path) -> Vec<(String, PathBuf)> { + let mut out = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(at) = pending.pop() { + for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { + let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); + let kind = fs::symlink_metadata(&path).unwrap_or_else(|e| panic!("stat {}: {e}", path.display())).file_type(); + assert!(!kind.is_symlink(), "{} is a link, which no product of the store holds", path.display()); + if kind.is_dir() { + pending.push(path); + } else { + let rel = path.strip_prefix(dir).expect("under the walked directory"); + out.push((rel.to_string_lossy().into_owned(), path)); + } + } + } + out.sort(); + out } /// This tree's, made if nobody on this host has made it. @@ -124,7 +240,7 @@ fn resolve(store: &Path, fork: &Path, llvm: &Path, sysroot: &Path, ninja: &Path) let dir = Keyed::HostedClang.store(store).join(&key); let make = || place(fork, llvm, sysroot, ninja, &key, &dir); let using = keystore::made(store, Keyed::HostedClang, &key, || defect(&dir), make); - HostedClang { dir, _using: using } + HostedClang { dir, c: CSysroot::of(sysroot, TARGET), licence: llvm.join("src/llvm/LICENSE.TXT"), _using: using } } /// The key the store product at `dir` is filed under. @@ -213,7 +329,7 @@ fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, native.extend(crate::llvm::NO_HOST_LIBRARIES.iter().map(|name| format!("-D{name}=OFF"))); let mut definitions = options(); definitions.extend(crate::llvm::stamp_options(fork)); - definitions.push(("LLVM_VERSION_SUFFIX".to_string(), version_suffix(&c.clang))); + definitions.push(("LLVM_VERSION_SUFFIX".to_string(), said_version(&c.clang).1)); definitions.extend([ ("CMAKE_TOOLCHAIN_FILE".to_string(), path(&c.cmake_toolchain())), ("CMAKE_C_COMPILER".to_string(), path(&c.clang)), @@ -233,10 +349,11 @@ fn build(fork: &Path, sources: &Path, built: &Path, c: &CSysroot, ninja: &Path, run(make, "build", ninja); } -/// What the host's clang, the sysroot's, says it is beyond LLVM's version: -/// bootstrap's `LLVM_VERSION_SUFFIX`. Every object a clang compiles carries -/// what it says in its `.comment`, so this one says it too. -fn version_suffix(clang: &Path) -> String { +/// What the host's clang, the sysroot's, says its version is: LLVM's release, +/// which the package is named by, and what it says beyond it, bootstrap's +/// `LLVM_VERSION_SUFFIX`. Every object a clang compiles carries the whole in +/// its `.comment`, so the ToyOS-hosted one is built to say it too. +fn said_version(clang: &Path) -> (String, String) { let mut command = Command::new(clang); command.arg("--version"); crate::llvm::clear(&mut command); @@ -244,14 +361,15 @@ fn version_suffix(clang: &Path) -> String { assert!(out.status.success(), "{command:?} failed: {}", String::from_utf8_lossy(&out.stderr)); let said = String::from_utf8_lossy(&out.stdout); let first = said.lines().next().unwrap_or_default(); - suffix(first).unwrap_or_else(|| panic!("{} says no version: {said}", clang.display())) + split_version(first).unwrap_or_else(|| panic!("{} says no version: {said}", clang.display())) } -/// The suffix of the version `line`, clang's first line of `--version`, names: -/// what follows its first `-`, and none without one. -fn suffix(line: &str) -> Option { +/// The version `line`, clang's first line of `--version`, names, split at its +/// first `-`: the suffix is empty without one. +fn split_version(line: &str) -> Option<(String, String)> { let version = line.strip_prefix("clang version ")?.split_whitespace().next()?; - Some(version.find('-').map_or(String::new(), |at| version[at..].to_string())) + let at = version.find('-').unwrap_or(version.len()); + Some((version[..at].to_string(), version[at..].to_string())) } /// Run `command` seeing nothing of this process's environment but what the @@ -299,11 +417,73 @@ mod tests { /// **The suffix is what follows the version's first `-`**, none without /// one, and a line that names no clang version names none. #[test] - fn the_suffix_is_what_follows_the_version() { + fn the_version_splits_at_its_first_dash() { let rev = "(https://github.com/ToyOSOrg/llvm-project.git ceaf0fbb8440)"; - assert_eq!(suffix(&format!("clang version 22.1.8-rust-dev {rev}")).as_deref(), Some("-rust-dev")); - assert_eq!(suffix(&format!("clang version 22.1.8 {rev}")).as_deref(), Some("")); - assert_eq!(suffix("Apple clang version 17.0.0 (clang-1700.0.13.5)"), None); + let split = split_version; + assert_eq!(split(&format!("clang version 22.1.8-rust-dev {rev}")), Some(("22.1.8".into(), "-rust-dev".into()))); + assert_eq!(split(&format!("clang version 22.1.8 {rev}")), Some(("22.1.8".into(), String::new()))); + assert_eq!(split("Apple clang version 17.0.0 (clang-1700.0.13.5)"), None); + } + + /// **The package is the product and the sysroot under one `clang`, + /// POSIX ustar throughout**, with the driver's other names as links and + /// the configuration that names the sysroot, and one input packs to one + /// digest. + #[test] + fn the_package_is_the_product_and_the_sysroot_under_one_directory() { + let scratch = TempDir::new("hosted-clang-package"); + let write = |rel: &str| { + let path = scratch.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, rel).unwrap(); + }; + for rel in ["product/bin/clang", "product/bin/ld.lld", "product/lib/clang/22/include/stddef.h", "product/SOURCE"] { + write(rel); + } + for rel in ["c/include/stdio.h", "c/include/c++/v1/vector", "c/lib/libtoyos_c.a", "c/toolchain.cmake", "LICENSE.TXT"] { + write(rel); + } + let (product, c) = (scratch.join("product"), scratch.join("c")); + let licence = scratch.join("LICENSE.TXT"); + let first = pack(&product, &c, &licence, "22.1.8", &scratch.join("one")); + assert_eq!(first.archive.file_name().unwrap(), "clang-v22.1.8-toyos-x86_64.tar.gz"); + + let gz = fs::File::open(&first.archive).unwrap(); + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(gz)); + let mut got = Vec::new(); + for entry in tar.entries().unwrap() { + let mut entry = entry.unwrap(); + assert_eq!(&entry.header().as_bytes()[257..265], b"ustar\x0000", "a header that is not POSIX ustar"); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + let held = match entry.link_name().unwrap() { + Some(target) => format!("-> {}", target.display()), + None => { + let mut text = String::new(); + std::io::Read::read_to_string(&mut entry, &mut text).unwrap(); + text + } + }; + got.push((path, held)); + } + let want = [ + ("clang/LICENSE.TXT", "LICENSE.TXT"), + ("clang/bin/clang", "product/bin/clang"), + ("clang/bin/ld.lld", "product/bin/ld.lld"), + ("clang/bin/x86_64-unknown-toyos.cfg", "--sysroot=/../sysroot\n"), + ("clang/lib/clang/22/include/stddef.h", "product/lib/clang/22/include/stddef.h"), + ("clang/sysroot/include/c++/v1/vector", "c/include/c++/v1/vector"), + ("clang/sysroot/include/stdio.h", "c/include/stdio.h"), + ("clang/sysroot/lib/libtoyos_c.a", "c/lib/libtoyos_c.a"), + ("clang/bin/cc", "-> clang"), + ("clang/bin/clang++", "-> clang"), + ]; + let want: Vec<(String, String)> = want.iter().map(|(p, h)| (p.to_string(), h.to_string())).collect(); + assert_eq!(got, want); + + let digest = |sums: &Path| fs::read_to_string(sums).unwrap(); + let again = pack(&product, &c, &licence, "22.1.8", &scratch.join("two")); + assert_eq!(digest(&first.sums), digest(&again.sums), "one input packed to two digests"); + assert!(digest(&first.sums).ends_with(" clang-v22.1.8-toyos-x86_64.tar.gz\n")); } /// **A product missing a binary or clang's headers is not whole**, nor one diff --git a/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md b/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md index e49d25894e1..70b9c72aaa9 100644 --- a/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md +++ b/issues/a-package-is-a-directory-under-apps-and-the-installer-is-a-program.md @@ -84,7 +84,10 @@ The storage track's users and mount-protocol stages do not block this one. binaries a launch starts and nothing else, so a writable `/apps` cannot name a device or a right. The stage-then-commit above amends it: `pkg` writes `/apps//` in place today, its `manifest.toml` last - (`system/pkg/src/main.rs`). + (`system/pkg/src/main.rs`). It streams an archive of any size that fits + the room `/apps`'s volume has, carries links that resolve inside the + package, and launches `/` or `/bin/`; + `hosted_clang_hello` installs the toolchain's `clang` package with it. 2. The HTTPS fetch: TLS client under `pkg`, the GitHub redirect, the signed repository's files and the archives its targets names. This is the internet-client track's last diff --git a/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md b/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md index dfc8adadaa8..e190d729449 100644 --- a/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md +++ b/issues/process-memory-is-2-mib-pages-and-that-caps-the-process-count.md @@ -18,6 +18,16 @@ IOMMU's superpages, the kernel's own mappings. One paging design serves both, on x86-64 and on the ARM64 the tree is kept portable for (`issues/toyos-runs-on-arm64.md`). +**A large program spawned from a large parent is refused.** A program a file +server holds is spawned from a copy in one memory object, and `SYS_SHM_CREATE` +allocates that object physically contiguous (`kernel/src/object/shm.rs`, +`alloc_contiguous`): about 148 MB for the hosted clang. A small std parent +spawns `ld.lld` that way, but clang's driver, starting `cc1` and `ld.lld` from +`/apps/clang`, is refused `ResourceExhausted` (`ENOMEM` in libc), so +`hosted_clang_hello` compiles and cannot link. That the contiguous allocation +is the one refused is inferred: the errno is measured, the allocation is +not. Removing contiguous user memory is the exit's; this row is its check. + **The constraint that makes this a track, measured on the T14** (run 29 readback, `PMM: 100/16038MB used` at 11 s with four user processes and three kernel threads live; the same record's rows: stack 16 pages held, init-tls 5, diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index eb7dab5e8fb..b06f59d84e4 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -64,73 +64,84 @@ An owner is an issue under `issues/`, or this track. | 2 | C reaches every file a process holds: descriptors (#975, #978) and served files through `toyos::files` (#988) | done | this track | — | | 3 | `pkg` installs a toolchain: a streamed archive bounded by its volume's room, and links. #970 installs the clang package (83.8 MB archive, 279 MB installed) on a 1 GiB DATA in QEMU | in progress | `a-package-is-a-directory-under-apps-and-the-installer-is-a-program` | — | | 4 | Room for a toolchain on DATA, durable. DATA serves `/apps`, `/config`, `/home` and `/state`; the full toolchain is about 1 GB | in progress | `storage-is-layers-and-a-role-is-a-filesystem` | — | -| 5 | **M2 exit**: a `pkg`-installed clang compiles, links and runs `hello.c` (#970's guest row), as `hosted_clang_hello` already does with a clang staged on ROOT | in progress | this track, `toyos-has-no-c-compiler-inside-it` | 2–4 | +| 5 | User memory without physical contiguity. clang's driver, spawning `cc1` from `/apps`, is refused `ENOMEM` (measured), where a small parent spawns the same images. The likely refusal is the image's memory object, which `SYS_SHM_CREATE` allocates physically contiguous (`kernel/src/object/shm.rs`), about 148 MB for clang; that it is this allocation and not the spawn's own is inferred. Memory stages 1–3 remove contiguous user memory | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count` | engineering: item 13's design, stages 1–3 | +| 6 | **M2 exit**: a `pkg`-installed clang compiles, links and runs `hello.c` (#970's guest row, which compiles and stops at the link on 5), as `hosted_clang_hello` did with a clang staged on ROOT | in progress | this track, `toyos-has-no-c-compiler-inside-it` | 2–5 | **M3: an LLVM-backed rustc.** | # | Item | Status | Owner | Blocked on | |---|---|---|---|---| -| 6 | Threads mature enough for LLVM | not started | `std-and-libc-drop-the-answer-thread-join-gives` | — | -| 7 | Concurrent reads and writes of one file: `pread` and `pwrite`. Without them LLVM reads a file slice with `lseek` and `read`, which moves an offset other threads share | not started | `libc-has-no-pread-or-pwrite` | — | -| 8 | LLVM, clang and lld built by bootstrap for a ToyOS host, and the libc names they call | not started | `bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host`, `rustc-llvm-cannot-build-for-a-toyos-host`, `libc-lacks-names-llvm-for-a-toyos-host-calls`, `a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine` | — | -| 9 | A std binary links the C++ runtime | not started | `a-rust-std-binary-cannot-link-the-cxx-runtime` | 8 | -| 10 | A package ships its own libraries (`librustc_driver`); moot for M2's static PIEs | not started | `a-package-cannot-ship-its-own-libraries` | 3 | -| 11 | **M3 exit**: the hosted rustc, which runs LLVM on several threads, builds and runs a program in the guest, linking with the guest's `ld.lld` | not started | `nothing-builds-the-toyos-hosted-rustc`, `the-hosted-rustc-names-a-linker-toyos-does-not-have`, `the-hosted-rustcs-stage2-carries-seventeen-proc-macro-libraries-no-image-needs` | 5–10 | +| 7 | Threads mature enough for LLVM | not started | `std-and-libc-drop-the-answer-thread-join-gives` | — | +| 8 | Concurrent reads and writes of one file: `pread` and `pwrite`. Without them LLVM reads a file slice with `lseek` and `read`, which moves an offset other threads share | not started | `libc-has-no-pread-or-pwrite` | — | +| 9 | LLVM, clang and lld built by bootstrap for a ToyOS host, and the libc names they call | not started | `bootstrap-cannot-build-llvm-clang-and-lld-for-a-toyos-host`, `rustc-llvm-cannot-build-for-a-toyos-host`, `libc-lacks-names-llvm-for-a-toyos-host-calls`, `a-toyos-hosted-llvm-is-configured-as-running-on-the-build-machine` | — | +| 10 | A std binary links the C++ runtime | not started | `a-rust-std-binary-cannot-link-the-cxx-runtime` | 9 | +| 11 | A package ships its own libraries (`librustc_driver`); moot for M2's static PIEs | not started | `a-package-cannot-ship-its-own-libraries` | 3 | +| 12 | **M3 exit**: the hosted rustc, which runs LLVM on several threads, builds and runs a program in the guest, linking with the guest's `ld.lld` | not started | `nothing-builds-the-toyos-hosted-rustc`, `the-hosted-rustc-names-a-linker-toyos-does-not-have`, `the-hosted-rustcs-stage2-carries-seventeen-proc-macro-libraries-no-image-needs` | 6–11 | **M4: cargo, rustc and clang build ToyOS in the guest, byte-identical.** | # | Item | Status | Owner | Blocked on | |---|---|---|---|---| -| 12 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count`, `one-mapping-inventory-nobody-can-read`, `anonymous-mmap-is-not-demand-paged`, `nothing-charges-kernel-memory-to-a-process` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | -| 13 | Record locks the file server serves, and std's and cargo's locks on them | direction set | `std-file-lock-answers-ok-and-locks-nothing`, `libc-refuses-what-toyos-cannot-yet-answer` | engineering: the lock design, part of 12's | -| 14 | An unlinked open file lives until its last close or unmap | direction set | `an-unlinked-open-file-answers-stale` | 12 for the mapping | -| 15 | An exclusive create on `/tmp` (`O_CREAT\|O_EXCL`, `create_new`) | not started | `create-new-on-a-kernel-path-is-not-exclusive` | — | -| 16 | libc tells the truth: `sigaction` refuses what it does not keep, `st_nlink` is 1, and the user and group ids are the process's own | not started | `libc-sigaction-answers-success-and-keeps-nothing`, `libc-answers-st-nlink-0`, `libc-answers-uid-0-for-every-process` | — | -| 17 | libc gains `select`, `setvbuf` and `socketpair`. A descriptor is a small table number since #978, so `FD_SETSIZE` holds | not started | this track | — | -| 18 | The `libc` crate gains a ToyOS module, checked by `ctest` against `sdk/libc`'s headers | not started | this track | 17 | -| 19 | Children: `SIGCHLD`, sessions, a quit notice and Ctrl+C, the process track's stages 4–7 | not started | `a-childs-end-is-an-event-and-a-parent-takes-its-children-down` | — | -| 20 | Sockets shared by their holders and inherited by a child, which curl, libssh2 and every inheriting server need | direction set | `a-stream-cannot-move-between-processes`, `libc-close-of-a-socket-is-unproven-on-a-connected-socket` | engineering: the sharing and recovery design, out of the tree, under review | -| 21 | cargo's eight C libraries cross-built: curl, libgit2, libssh2, OpenSSL, SQLite, nghttp2, zlib, blake3; five stopped on headers, types and functions libc lacked at `15625e0cb` | not started | this track | 17, 18 | -| 22 | cargo's TLS: OpenSSL through its Perl `Configure`, or curl's rustls backend on `ToyOSOrg/ring`. A build from vendored sources needs neither, and neither needs the `toyos-crypto` track | not started, choice open | this track, `the-internet-clients-work-unchanged` | 21; HTTPS on `the-lan-is-not-yet-production-grade` | -| 23 | cargo at run time: its SQLite and its locks, sockets and children | not started | this track, `sqlite-and-postgresql-run-on-toyos` | 13–16, 19–21 | -| 24 | The tools a build runs: Perl and Python started by spawn, brush as `sh`, uutils, make, CMake with libuv on spawn, awk, and a script started by its interpreter line | not started | `ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have`, `nothing-launches-a-script-by-its-interpreter-line` | 19 for the C tools | -| 25 | git in the guest | not started | this track | 21 | -| 26 | The toolchain as `pkg` packages, none of it in the image. #970 makes the clang package; how the rest splits, and how a self-host test's guest reaches them, is open | in progress | this track | 3, 10 | -| 27 | Storage durable and fast enough for an LLVM build tree, unmeasured | not started | `storage-is-layers-and-a-role-is-a-filesystem`, `an-fsync-of-a-served-file-syncs-its-whole-volume` | 4 | -| 28 | A build reproduces: an image's GUIDs from the commit, and one checkout building one kernel | not started | `an-images-guids-are-drawn-from-the-hosts-random-source`, `one-checkout-of-one-tree-builds-two-kernels` | — | -| 29 | **M4 exit**: the guest builds a userland program and then the kernel, and the hashes equal the host build's of the same commit | not started | this track | 11–28 | +| 13 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-is-2-mib-pages-and-that-caps-the-process-count`, `one-mapping-inventory-nobody-can-read`, `anonymous-mmap-is-not-demand-paged`, `nothing-charges-kernel-memory-to-a-process` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | +| 14 | Record locks the file server serves, and std's and cargo's locks on them | direction set | `std-file-lock-answers-ok-and-locks-nothing`, `libc-refuses-what-toyos-cannot-yet-answer` | engineering: the lock design, part of 13's | +| 15 | An unlinked open file lives until its last close or unmap | direction set | `an-unlinked-open-file-answers-stale` | 13 for the mapping | +| 16 | An exclusive create on `/tmp` (`O_CREAT\|O_EXCL`, `create_new`) | not started | `create-new-on-a-kernel-path-is-not-exclusive` | — | +| 17 | libc tells the truth: `sigaction` refuses what it does not keep, `st_nlink` is 1, and the user and group ids are the process's own | not started | `libc-sigaction-answers-success-and-keeps-nothing`, `libc-answers-st-nlink-0`, `libc-answers-uid-0-for-every-process` | — | +| 18 | libc gains `select`, `setvbuf` and `socketpair`. A descriptor is a small table number since #978, so `FD_SETSIZE` holds | not started | this track | — | +| 19 | The `libc` crate gains a ToyOS module, checked by `ctest` against `sdk/libc`'s headers | not started | this track | 18 | +| 20 | Children: `SIGCHLD`, sessions, a quit notice and Ctrl+C, the process track's stages 4–7 | not started | `a-childs-end-is-an-event-and-a-parent-takes-its-children-down` | — | +| 21 | Sockets shared by their holders and inherited by a child, which curl, libssh2 and every inheriting server need | direction set | `a-stream-cannot-move-between-processes`, `libc-close-of-a-socket-is-unproven-on-a-connected-socket` | engineering: the sharing and recovery design, out of the tree, under review | +| 22 | cargo's eight C libraries cross-built: curl, libgit2, libssh2, OpenSSL, SQLite, nghttp2, zlib, blake3; five stopped on headers, types and functions libc lacked at `15625e0cb` | not started | this track | 18, 19 | +| 23 | cargo's TLS: OpenSSL through its Perl `Configure`, or curl's rustls backend on `ToyOSOrg/ring`. A build from vendored sources needs neither, and neither needs the `toyos-crypto` track | not started, choice open | this track, `the-internet-clients-work-unchanged` | 22; HTTPS on `the-lan-is-not-yet-production-grade` | +| 24 | cargo at run time: its SQLite and its locks, sockets and children | not started | this track, `sqlite-and-postgresql-run-on-toyos` | 14–17, 20–22 | +| 25 | The tools a build runs: Perl and Python started by spawn, brush as `sh`, uutils, make, CMake with libuv on spawn, awk, and a script started by its interpreter line | not started | `ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have`, `nothing-launches-a-script-by-its-interpreter-line` | 20 for the C tools | +| 26 | git in the guest | not started | this track | 22 | +| 27 | The toolchain as `pkg` packages, none of it in the image. #970 makes the clang package; how the rest splits, and how a self-host test's guest reaches them, is open | in progress | this track | 3, 11 | +| 28 | Storage durable and fast enough for an LLVM build tree, unmeasured | not started | `storage-is-layers-and-a-role-is-a-filesystem`, `an-fsync-of-a-served-file-syncs-its-whole-volume` | 4 | +| 29 | A build reproduces: an image's GUIDs from the commit, and one checkout building one kernel | not started | `an-images-guids-are-drawn-from-the-hosts-random-source`, `one-checkout-of-one-tree-builds-two-kernels` | — | +| 30 | **M4 exit**: the guest builds a userland program and then the kernel, and the hashes equal the host build's of the same commit | not started | this track | 12–29 | What the items rest on. The compiles were made against libc at `15625e0cb` (2026-10-01), before #975, #978 and #988 changed it: -- **3, 26, measured:** the old `pkg` inflated a whole archive in memory under +- **3, 27, measured:** the old `pkg` inflated a whole archive in memory under `MAX_INFLATED`, 256 MiB, and refused every link (`system/pkg/src/main.rs`, `archive.rs`), which #970 replaces. Measured on the Linux-host release `toolchain-linux-x86_64-48dd24f826263d6c`, stripped: `librustc_driver` 184.7 MB, clang 122.8 MB, lld 78.6 MB, and 181.1 MB of `x86_64-unknown-toyos` libraries. -- **10, read:** rustc's launcher names `librustc_driver` as needed +- **11, read:** rustc's launcher names `librustc_driver` as needed (`llvm-readobj --needed-libs`). -- **17, read:** curl 8.21.0, which `curl-sys` 0.4.90 builds, does not compile +- **18, read:** curl 8.21.0, which `curl-sys` 0.4.90 builds, does not compile without `select` (`lib/curlx/wait.c`), and waits through it, since `curl-sys` does not define `HAVE_POLL`. curl's multi handle wakes through a `socketpair`, and its TLS key log sets its buffering through `setvbuf`. -- **18, read:** `libc` 0.2.189 has no ToyOS module (`rg -i toyos` over `src`) +- **19, read:** `libc` 0.2.189 has no ToyOS module (`rg -i toyos` over `src`) and is empty for an OS it does not know; `ToyOSOrg/libc` does not exist. cargo's `curl-sys`, `libssh2-sys` and `libgit2-sys` import its C types unconditionally. -- **21, compiled at `15625e0cb`:** nghttp2, zlib and blake3 build with nothing +- **22, compiled at `15625e0cb`:** nghttp2, zlib and blake3 build with nothing undefined; curl, libgit2, libssh2, OpenSSL and SQLite stop on headers, types or functions libc lacked. -- **21 and 23, read:** SQLite's unix VFS takes `fcntl` record locks keyed on +- **22 and 24, read:** SQLite's unix VFS takes `fcntl` record locks keyed on `st_dev` and `st_ino`; libgit2 opens every repository through `realpath` and reads packs through a file `mmap`; curl and libssh2 wait on non-blocking sockets. `openssl-src` 300.6.1 refuses a target it does not know (`src/lib.rs`), and `openssl-sys` is a `cfg(unix)` dependency of `curl-sys` and `libssh2-sys`, which ToyOS is not. That cargo bundles SQLite is read from its dependencies, not run. -- **24, read:** Perl 5.44.0's sources never call `posix_spawn`. Which Python, +- **25, read:** Perl 5.44.0's sources never call `posix_spawn`. Which Python, which make and which awk is open. +Owed by the `clang` package (item 3), and needed by no exit: +- **Licences.** The package carries LLVM's `LICENSE.TXT` and none of the + licences of what `libtoyos_c.a` links besides `sdk/libc`, though + `build/hostedclang.rs` packs `sysroot/lib`; a package published beyond a + test's ROOT owes them. +- **A command by its name.** A shell finds no installed package's command by + name: its `PATH` is `/system/bin` (`system/shell/src/main.rs`), so + `clang hello.c` is typed `/apps/clang/bin/clang hello.c`. How an installed + command is reached by its name is open. + Beside the path, and needed by none of its exits: - **The boot handoff** (owner, 2026-10-21) removes the kernel's ROOT mount: the loader hands the supervisor and the system file server over as signed @@ -149,5 +160,5 @@ Beside the path, and needed by none of its exits: - Every AArch64 stage lands one step behind x86-64, on `toyos-runs-on-arm64`. **No image ships a linker** (owner, 2026-10-04: "Just get rid of toyos ld"). -Until item 5 lands, `ld.lld` links inside ToyOS only where `hosted_clang_hello` +Until item 6 lands, `ld.lld` links inside ToyOS only where `hosted_clang_hello` stages it. diff --git a/lib/bcachefs/src/alloc_bitmap.rs b/lib/bcachefs/src/alloc_bitmap.rs index af76e98260c..04e216f8277 100644 --- a/lib/bcachefs/src/alloc_bitmap.rs +++ b/lib/bcachefs/src/alloc_bitmap.rs @@ -141,7 +141,7 @@ impl BitmapAllocator { /// What an allocation may take now: every free block, but for those /// [`NODE_RESERVE`] keeps from all but an operation that draws on it. - fn spare(&self) -> u64 { + pub(crate) fn spare(&self) -> u64 { match !self.shadow || self.op.as_ref().is_some_and(|op| op.reserve == Reserve::Draw) { true => self.free_blocks, false => self.free_blocks.saturating_sub(NODE_RESERVE), diff --git a/lib/bcachefs/src/fs.rs b/lib/bcachefs/src/fs.rs index 28355c73d84..418cee4118e 100644 --- a/lib/bcachefs/src/fs.rs +++ b/lib/bcachefs/src/fs.rs @@ -607,6 +607,12 @@ impl Mounted { fs.alloc.count_free(&fs.io)?; Ok(fs) } + + /// The bytes a change may still take: every free block but those + /// [`crate::alloc_bitmap::NODE_RESERVE`] keeps for a delete. + pub fn free_bytes(&self) -> u64 { + self.alloc.spare() * BLOCK_SIZE as u64 + } } impl Mounted { diff --git a/lib/manifest/src/package.rs b/lib/manifest/src/package.rs index b616fc62b24..039fc092c7f 100644 --- a/lib/manifest/src/package.rs +++ b/lib/manifest/src/package.rs @@ -36,8 +36,8 @@ pub struct Package { /// The archive this directory was unpacked from, as the release's own /// `SHA256SUMS` spelled it. pub digest: String, - /// The whole guest path a launch starts, always a file directly inside - /// this package's own directory. + /// The whole guest path a launch starts, always a file inside this + /// package's own directory. pub program: String, } @@ -181,13 +181,14 @@ pub fn package_of(path: &str) -> Option<&str> { Some(name) } -/// Whether `program` names a file directly inside package `name`'s directory. +/// Whether `program` names a file inside package `name`'s directory, by a path +/// with no empty, `.` or `..` component. pub fn program_is_inside(name: &str, program: &str) -> bool { if name_is_bad(name) { return false; } let Some(file) = program.strip_prefix(&format!("{DIR}/{name}/")) else { return false }; - !file.is_empty() && !file.contains('/') && file != "." && file != ".." + file.split('/').all(|part| !part.is_empty() && part != "." && part != "..") } fn name_is_bad(name: &str) -> bool { @@ -249,13 +250,17 @@ mod tests { "/system/bin/supervisor", "/apps/other/other", "/apps/gbae/../../system/bin/supervisor", - "/apps/gbae/sub/x", + "/apps/gbae/bin/../../other/other", + "/apps/gbae/bin//gbae", + "/apps/gbae/./gbae", + "/apps/gbae/bin/", "/apps/gbae/", ] { let bad = Package { program: program.into(), ..sample() }; assert!(bad.render().is_err(), "{program} was accepted"); } assert!(sample().render().is_ok()); + assert!(Package { program: "/apps/gbae/bin/gbae".into(), ..sample() }.render().is_ok()); } #[test] diff --git a/sdk/toyos/src/fs.rs b/sdk/toyos/src/fs.rs index dc0f7f86e34..deb14ef00df 100644 --- a/sdk/toyos/src/fs.rs +++ b/sdk/toyos/src/fs.rs @@ -92,6 +92,8 @@ pub const SEEK: u32 = 21; /// carries its other end. It spends a served slot of this connection's share /// from here, and names no path. pub const EXPORT: u32 = 22; +/// The free space of the volume behind the capability ([`Room`]). +pub const ROOM: u32 = 23; pub const SEEK_START: u64 = 0; pub const SEEK_CURRENT: u64 = 1; @@ -453,6 +455,16 @@ pub struct Stat { pub node: u64, } +/// How much more the volume behind a capability holds: `free` bytes, taken in +/// allocation units of `unit` bytes, so a file of `n` bytes takes `n` rounded +/// up to `unit`. A volume's own structures take room too, so it is a bound and +/// never a promise. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Room { + pub free: u64, + pub unit: u64, +} + /// A write answered. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Written { @@ -743,6 +755,18 @@ impl Dir { }) } + /// The volume's [`Room`]. Like [`Self::sync`], it names no path. + pub fn room(&mut self) -> Result { + let reply = self.path_call(ROOM, Request::new(), "", "").map_err(|refused| match refused { + Refused::Error(e) => e, + Refused::Link(_) => Error::Malformed, + })?; + match reply.value2 { + 0 => Err(Error::Malformed), + unit => Ok(Room { free: reply.value, unit }), + } + } + /// A pipe whose bytes the server writes to this file at the description's /// offset, which they move, for a program's standard output: the write end. pub fn stream(&mut self, fid: u64, generation: u64) -> Result { diff --git a/system/fileserver/src/absent.rs b/system/fileserver/src/absent.rs index 6b26b433450..da923e553da 100644 --- a/system/fileserver/src/absent.rs +++ b/system/fileserver/src/absent.rs @@ -102,6 +102,10 @@ impl Volume for Absent { Ok(Vec::new()) } + fn room(&mut self) -> Result { + Ok(toyos::fs::Room { free: 0, unit: 1 }) + } + fn describe(&self) -> String { format!("absent: {}", self.why) } diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index c49fbca2ef0..e587f828422 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -769,6 +769,10 @@ impl Volume for DataVolume { Ok(unwritten) } + fn room(&mut self) -> Result { + Ok(toyos::fs::Room { free: self.fs.free_bytes(), unit: BLOCK as u64 }) + } + fn describe(&self) -> String { let c = self.cache.counts(); format!( @@ -1420,4 +1424,19 @@ mod tests { assert_eq!(names(&mut v, "home/staged"), ["a", "z"]); assert_eq!(v.lstat(&to), Err(Error::NotFound)); } + + /// What a write takes is what the room answers less, block for block, + /// and the room never names the blocks the reserve keeps for a delete. + #[test] + fn a_write_takes_its_blocks_out_of_the_room() { + let mut v = vol(); + let n = v.open("apps/x", CREATE).unwrap(); + let before = v.room().unwrap(); + assert_eq!(before.unit, BLOCK as u64); + v.write(n, 0, &vec![7u8; 3 * BLOCK + 1]).unwrap(); + assert_eq!(v.room().unwrap().free, before.free - 4 * BLOCK as u64); + v.close(n).unwrap(); + let free = free_blocks(&mut v); + assert!(v.room().unwrap().free < free * BLOCK as u64, "the reserve's blocks were counted as room"); + } } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 7b08554a747..ec48e11ce18 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -458,6 +458,15 @@ impl Volume for FatVolume { Ok(unlevel) } + fn room(&mut self) -> Result { + let unit = u64::from(self.fs.geometry().bytes_per_cluster()); + if !self.writable { + return Ok(fs::Room { free: 0, unit }); + } + let free = self.fs.free_bytes().map_err(|e| logged("free space", "", e))?; + Ok(fs::Room { free, unit }) + } + fn describe(&self) -> String { let c = self.cache.counts(); format!( diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 8a5657ccdd2..b3d4a20fad8 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -761,6 +761,10 @@ impl Server { self.sync(Some(node)) } SYNC => self.sync(None), + ROOM => { + let room = self.volume.room()?; + Ok(Answer::Reply(Reply { value: room.free, value2: room.unit, ..Reply::ok() })) + } STREAM => { let file = self.fid(id, r.fid)?; if !file.borrow().write { diff --git a/system/fileserver/src/rights.rs b/system/fileserver/src/rights.rs index eba3a0afa0b..a8a6186823b 100644 --- a/system/fileserver/src/rights.rs +++ b/system/fileserver/src/rights.rs @@ -13,7 +13,7 @@ use toyos::fs::*; pub fn changes(op: u32, flags: u64) -> Option { match op { OPEN => Some(flags & (O_WRITE | O_APPEND | O_CREATE | O_TRUNCATE | O_CREATE_NEW) != 0), - HELLO | CLOSE | READ | STAT | LSTAT | FSTAT | FSYNC | SYNC | READDIR | READLINK | DUP | SEEK | EXPORT => { + HELLO | CLOSE | READ | STAT | LSTAT | FSTAT | FSYNC | SYNC | READDIR | READLINK | DUP | SEEK | EXPORT | ROOM => { Some(false) } WRITE | TRUNCATE | MKDIR | RMDIR | UNLINK | RENAME | SYMLINK | STREAM => Some(true), @@ -27,7 +27,7 @@ mod tests { /// Every request the wire has, by what it does to the directory: the /// independent spelling `changes` is held to. - const READS: [u32; 13] = [HELLO, CLOSE, READ, STAT, LSTAT, FSTAT, FSYNC, SYNC, READDIR, READLINK, DUP, SEEK, EXPORT]; + const READS: [u32; 14] = [HELLO, CLOSE, READ, STAT, LSTAT, FSTAT, FSYNC, SYNC, READDIR, READLINK, DUP, SEEK, EXPORT, ROOM]; const WRITES: [u32; 8] = [WRITE, TRUNCATE, MKDIR, RMDIR, UNLINK, RENAME, SYMLINK, STREAM]; #[test] @@ -41,7 +41,7 @@ mod tests { // Every request number the wire has is one of the two, or `OPEN`. let mut named: Vec = READS.iter().chain(&WRITES).copied().chain([OPEN]).collect(); named.sort_unstable(); - assert_eq!(named, (HELLO..=EXPORT).collect::>()); + assert_eq!(named, (HELLO..=ROOM).collect::>()); } /// An open changes the directory by any one flag that writes, creates or @@ -60,7 +60,7 @@ mod tests { /// client is let go whatever its grant. #[test] fn a_request_the_wire_does_not_have_is_neither() { - for op in [0, EXPORT + 1, REPLY, LINK, u32::MAX] { + for op in [0, ROOM + 1, REPLY, LINK, u32::MAX] { assert_eq!(changes(op, 0), None, "request {op}"); } } diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 2fb7c53b4df..0278b563465 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -132,6 +132,9 @@ pub trait Volume { /// volume's own flush refused. fn sync(&mut self) -> Result, Error>; + /// What more can be written to the volume: none, where nothing can be. + fn room(&mut self) -> Result; + /// One line of what the volume and its cache have done, for a log. fn describe(&self) -> String; } diff --git a/system/pkg/Cargo.toml b/system/pkg/Cargo.toml index 20f281bede5..5f9be099a3c 100644 --- a/system/pkg/Cargo.toml +++ b/system/pkg/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pkg" -description = "The installer: an archive checked against its release's SHA256SUMS and unpacked as one directory under /apps; the decoders are a library tested on the host." +description = "The installer: an archive checked against its release's SHA256SUMS and streamed into one directory under /apps within the room its volume has; the decoders are a library tested on the host." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" @@ -12,10 +12,18 @@ license = "MIT OR Apache-2.0" doctest = false [dependencies] +# The room `/apps`'s volume has, which only its file server answers. +toyos = { path = "../../sdk/toyos" } toyos-manifest = { path = "../../lib/manifest" } +toyos-sha2 = { path = "../../lib/sha2" } toyos-sha2-hw = { path = "../../lib/sha2-hw" } flate2 = { version = "1", default-features = false, features = ["rust_backend"] } [dev-dependencies] +# Archives made by a writer that is not this crate's reader. +tar = { version = "0.4.46", default-features = false } # Test scratch that is gone on every way out; `--ci host` reds on any left behind. toyos-tmpdir = { path = "../../lib/tmpdir" } + +[package.metadata.toyos.host] +exempt.manages = "ToyOS's installed packages: the directories under /apps the supervisor launches from" diff --git a/system/pkg/src/archive.rs b/system/pkg/src/archive.rs index aeb799cccfc..b7cd77b47ca 100644 --- a/system/pkg/src/archive.rs +++ b/system/pkg/src/archive.rs @@ -1,41 +1,48 @@ -//! The ustar half of a `.tar.gz`, decoded as far as unpacking a release -//! archive needs and no further. +//! The ustar half of a `.tar.gz`, read as a stream: one header at a time, and +//! the entry's data after it, so no archive is ever held whole. //! //! **Every field is the archive's claim about itself**, and this is the one //! place a downloaded file decides where bytes are written. A name that leaves -//! the package directory, a header whose own checksum does not match, a size -//! that runs off the end and every type that is not a plain file or a -//! directory are refused by name rather than skipped. +//! the package directory, a header whose own checksum does not match, an +//! archive that ends inside an entry and every type that is not a plain file, +//! a directory or a symlink are refused by name rather than skipped. Where a +//! symlink may point is the package's business (`crate::Planner`). //! -//! GNU long names, pax headers, links and device nodes are refused rather than -//! implemented: an archive that needs one is one this installer does not +//! GNU long names, pax headers, hard links and device nodes are refused rather +//! than implemented: an archive that needs one is one this installer does not //! install, said out loud. +use std::io::Read; + const BLOCK: usize = 512; const TYPE_FLAG: usize = 156; const CHKSUM: usize = 148; const CHKSUM_LEN: usize = 8; +const LINK_NAME: usize = 157; +const MAGIC: usize = 257; /// What a tar entry may be here. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum Kind { File, Dir, + /// A symlink, holding its target as the archive wrote it. + Symlink(String), } -/// One entry, borrowing the decompressed archive. -#[derive(Debug)] -pub struct Entry<'a> { +/// One entry's header. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Header { /// Slash-separated, relative, and already checked to stay inside itself. pub path: String, pub kind: Kind, - /// Empty for a directory. - pub data: &'a [u8], + /// The data that follows the header: 0 but for a file. + pub size: u64, /// The low nine permission bits; the rest of the mode is ignored. pub mode: u32, } -impl Entry<'_> { +impl Header { /// Whether the archive marked this entry executable for its owner. pub fn executable(&self) -> bool { self.mode & 0o100 != 0 @@ -48,49 +55,100 @@ impl Entry<'_> { } } -/// Decode every entry of an uncompressed tar. -pub fn entries(tar: &[u8]) -> Result>, String> { - let mut out = Vec::new(); - let mut at = 0; - while at + BLOCK <= tar.len() { - let header = &tar[at..at + BLOCK]; +/// An uncompressed tar, an entry at a time. +pub struct Reader { + inner: R, + /// The current entry's data not yet read. + left: u64, + /// The zeros that pad it to a block. + pad: u64, + /// The path of the current entry, for a refusal that names it. + current: String, +} + +impl Reader { + pub fn new(inner: R) -> Self { + Self { inner, left: 0, pad: 0, current: String::new() } + } + + /// The next entry, past whatever of the last one was not read, or `None` at + /// the block of zeros that ends the archive. + pub fn next_entry(&mut self) -> Result, String> { + self.skip(self.left + self.pad)?; + self.left = 0; + self.pad = 0; + let mut header = [0u8; BLOCK]; + self.inner.read_exact(&mut header).map_err(|e| match e.kind() { + std::io::ErrorKind::UnexpectedEof => String::from("tar: the archive ends inside a header"), + _ => format!("tar: {e}"), + })?; if header.iter().all(|&b| b == 0) { - return Ok(out); + return Ok(None); } - checksum_matches(header)?; - let magic = &header[257..262]; - if magic != b"ustar" { - return Err(format!("tar: block at {at} is not ustar")); + let header = decode(&header)?; + self.current = header.path.clone(); + self.left = header.size; + self.pad = header.size.next_multiple_of(BLOCK as u64) - header.size; + Ok(Some(header)) + } + + /// Up to `buf.len()` bytes of the current entry's data: 0 once it is all + /// read, and a refusal where the archive ends before it does. + pub fn read_data(&mut self, buf: &mut [u8]) -> Result { + let want = buf.len().min(usize::try_from(self.left).unwrap_or(usize::MAX)); + if want == 0 { + return Ok(0); } - let size = octal(header, 124, 12, "size")?; - let mode = octal(header, 100, 8, "mode")? as u32; - let kind = match header[TYPE_FLAG] { - b'0' | 0 => Kind::File, - b'5' => Kind::Dir, - other => { - return Err(format!( - "tar: entry {:?} is type {:?}, and only a file or a directory is unpacked", - field(header, 0, 100), - other as char - )) - } - }; - let path = path_of(header)?; - at += BLOCK; - let data = match kind { - Kind::Dir => &tar[at..at], - Kind::File => { - let end = at - .checked_add(size as usize) - .filter(|end| *end <= tar.len()) - .ok_or_else(|| format!("tar: {path} claims {size} bytes the archive lacks"))?; - &tar[at..end] + let n = self.inner.read(&mut buf[..want]).map_err(|e| format!("tar: {}: {e}", self.current))?; + if n == 0 { + return Err(format!("tar: the archive ends inside {}", self.current)); + } + self.left -= n as u64; + Ok(n) + } + + /// Discard up to `bytes` of the stream. One that ends first is refused by + /// the header read after it. + fn skip(&mut self, bytes: u64) -> Result<(), String> { + std::io::copy(&mut (&mut self.inner).take(bytes), &mut std::io::sink()) + .map(drop) + .map_err(|e| format!("tar: {}: {e}", self.current)) + } +} + +fn decode(header: &[u8; BLOCK]) -> Result { + checksum_matches(header)?; + // POSIX ustar is `ustar\0` then `00`; GNU tar's own format is `ustar \0`, + // and keeps other fields where ustar has the name's prefix. + let posix = match &header[MAGIC..MAGIC + 8] { + b"ustar\x0000" => true, + b"ustar \0" => false, + _ => return Err(format!("tar: {:?} is not a ustar header", field(header, 0, 100))), + }; + let mode = octal(header, 100, 8, "mode")? as u32 & 0o777; + let size = octal(header, 124, 12, "size")?; + let path = path_of(header, posix)?; + let kind = match header[TYPE_FLAG] { + b'0' | 0 => Kind::File, + b'5' => Kind::Dir, + b'2' => { + let target = field(header, LINK_NAME, 100); + if target.is_empty() { + return Err(format!("tar: the symlink {path} holds no target")); } - }; - at += (size as usize).div_ceil(BLOCK) * BLOCK; - out.push(Entry { path, kind, data, mode }); + Kind::Symlink(target) + } + other => { + return Err(format!( + "tar: entry {path:?} is type {:?}, and only a file, a directory or a symlink is unpacked", + other as char + )) + } + }; + if kind != Kind::File && size != 0 { + return Err(format!("tar: {path} claims {size} bytes of data, which only a file carries")); } - Err(String::from("tar: the archive ends inside a header")) + Ok(Header { path, kind, size, mode }) } /// The header's own checksum, which is what makes a truncated or misaligned @@ -113,9 +171,9 @@ fn checksum_matches(header: &[u8]) -> Result<(), String> { /// `prefix/name`, refused unless it stays inside the archive's own tree: a /// leading `/` or a `..` component is how an archive writes outside the /// directory it was unpacked into. -fn path_of(header: &[u8]) -> Result { +fn path_of(header: &[u8], posix: bool) -> Result { let name = field(header, 0, 100); - let prefix = field(header, 345, 155); + let prefix = if posix { field(header, 345, 155) } else { String::new() }; let joined = if prefix.is_empty() { name.clone() } else { format!("{prefix}/{name}") }; let path = joined.trim_end_matches('/'); if path.is_empty() || path.starts_with('/') { @@ -146,29 +204,35 @@ fn octal(header: &[u8], at: usize, len: usize, what: &str) -> Result Vec { + /// One POSIX ustar header as a writer would build it, so the refusals below + /// are tested against archives that are otherwise valid. + pub(crate) fn header(name: &str, kind: u8, size: u64, mode: u32, link: &str) -> Vec { let mut h = vec![0u8; BLOCK]; h[..name.len()].copy_from_slice(name.as_bytes()); h[100..108].copy_from_slice(format!("{mode:07o}\0").as_bytes()); h[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); h[TYPE_FLAG] = kind; - h[257..263].copy_from_slice(b"ustar\0"); - h[263..265].copy_from_slice(b"00"); + h[LINK_NAME..LINK_NAME + link.len()].copy_from_slice(link.as_bytes()); + h[MAGIC..MAGIC + 8].copy_from_slice(b"ustar\x0000"); + seal(&mut h); + h + } + + /// Write the header's checksum over what it holds now. + pub(crate) fn seal(h: &mut [u8]) { h[CHKSUM..CHKSUM + CHKSUM_LEN].copy_from_slice(b" "); let sum: u64 = h.iter().map(|&b| u64::from(b)).sum(); h[CHKSUM..CHKSUM + CHKSUM_LEN].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - h } - fn archive(parts: &[(&str, u8, &[u8], u32)]) -> Vec { + /// `(name, type, data, mode, link target)` each, then the end. + pub(crate) fn archive(parts: &[(&str, u8, &[u8], u32, &str)]) -> Vec { let mut out = Vec::new(); - for (name, kind, data, mode) in parts { - out.extend(header(name, *kind, data.len() as u64, *mode)); + for (name, kind, data, mode, link) in parts { + out.extend(header(name, *kind, data.len() as u64, *mode, link)); out.extend(*data); out.resize(out.len().div_ceil(BLOCK) * BLOCK, 0); } @@ -176,54 +240,116 @@ mod tests { out } + /// Every entry with its data, read the way the installer reads them. + fn all(tar: &[u8]) -> Result)>, String> { + let mut reader = Reader::new(tar); + let mut out = Vec::new(); + while let Some(h) = reader.next_entry()? { + let mut data = Vec::new(); + let mut buf = [0u8; 3]; + loop { + match reader.read_data(&mut buf)? { + 0 => break, + n => data.extend_from_slice(&buf[..n]), + } + } + out.push((h, data)); + } + Ok(out) + } + #[test] - fn a_directory_and_two_files_come_back_whole() { + fn a_directory_two_files_and_a_link_come_back_whole() { let tar = archive(&[ - ("gbae/", b'5', b"", 0o755), - ("gbae/gbae", b'0', b"ELF...", 0o755), - ("gbae/LICENSE", b'0', b"MIT", 0o644), + ("gbae/", b'5', b"", 0o755, ""), + ("gbae/gbae", b'0', b"ELF...", 0o755, ""), + ("gbae/LICENSE", b'0', b"MIT", 0o644, ""), + ("gbae/run", b'2', b"", 0o777, "gbae"), ]); - let got = entries(&tar).expect("entries"); - assert_eq!(got.len(), 3); - assert_eq!(got[0].path, "gbae"); - assert_eq!(got[0].kind, Kind::Dir); - assert_eq!(got[1].path, "gbae/gbae"); - assert_eq!(got[1].data, b"ELF..."); - assert!(got[1].executable()); - assert_eq!(got[2].data, b"MIT"); - assert!(!got[2].executable()); - assert!(got.iter().all(|e| e.top() == "gbae")); + let got = all(&tar).expect("entries"); + assert_eq!(got.len(), 4); + assert_eq!((got[0].0.path.as_str(), &got[0].0.kind), ("gbae", &Kind::Dir)); + assert_eq!((got[1].0.path.as_str(), got[1].1.as_slice()), ("gbae/gbae", &b"ELF..."[..])); + assert!(got[1].0.executable()); + assert_eq!(got[2].1, b"MIT"); + assert!(!got[2].0.executable()); + assert_eq!(got[3].0.kind, Kind::Symlink("gbae".into())); + assert!(got.iter().all(|(h, _)| h.top() == "gbae")); + } + + /// An entry whose data the reader never asked for is stepped over, padding + /// and all, so the next header is where the archive put it. + #[test] + fn data_left_unread_is_skipped_to_the_next_header() { + let tar = archive(&[("a/x", b'0', &[1u8; 700], 0o644, ""), ("a/y", b'0', b"y", 0o644, "")]); + let mut reader = Reader::new(tar.as_slice()); + assert_eq!(reader.next_entry().unwrap().unwrap().path, "a/x"); + assert_eq!(reader.next_entry().unwrap().unwrap().path, "a/y"); + let mut buf = [0u8; 8]; + assert_eq!(reader.read_data(&mut buf).unwrap(), 1); + assert_eq!(reader.next_entry().unwrap(), None); } /// The refusal the installer's write path rests on. #[test] fn a_name_that_leaves_the_archive_is_refused() { - for name in ["/etc/passwd", "gbae/../../system/bin/supervisor", "../x", "./x", ""] { - let tar = archive(&[(name, b'0', b"x", 0o644)]); - assert!(entries(&tar).is_err(), "{name:?} was accepted"); + for name in ["/etc/passwd", "gbae/../../system/bin/supervisor", "../x", "./x", "a//b", ""] { + let tar = archive(&[(name, b'0', b"x", 0o644, "")]); + assert!(all(&tar).is_err(), "{name:?} was accepted"); } } /// A type silently skipped is a package missing a file the archive carried. #[test] - fn every_type_but_a_file_and_a_directory_is_refused() { - for kind in [b'1', b'2', b'3', b'4', b'6', b'L', b'x', b'g'] { - let tar = archive(&[("gbae/x", kind, b"", 0o644)]); - assert!(entries(&tar).is_err(), "type {} was accepted", kind as char); + fn every_type_but_a_file_a_directory_and_a_symlink_is_refused() { + for kind in *b"13467LKxg" { + let tar = archive(&[("gbae/x", kind, b"", 0o644, "y")]); + let why = all(&tar).expect_err(&format!("type {} was accepted", kind as char)); + assert!(why.contains("only a file, a directory or a symlink"), "{why}"); + } + assert!(all(&archive(&[("gbae/x", b'2', b"", 0o777, "")])).is_err(), "a link to nothing"); + let link_with_data = archive(&[("gbae/x", b'2', b"data", 0o777, "y")]); + assert!(all(&link_with_data).is_err(), "a link that carries data"); + } + + #[test] + fn a_truncated_archive_is_refused_wherever_it_ends() { + let whole = archive(&[("gbae/gbae", b'0', &[7u8; 1000], 0o755, "")]); + // Inside the header, inside the data, inside the padding, and with no + // end block at all. + for cut in [1, BLOCK - 1, BLOCK + 1, BLOCK + 999, BLOCK + 1000, 3 * BLOCK] { + let why = all(&whole[..cut]).expect_err(&format!("cut at {cut} was accepted")); + assert!(why.contains("ends inside"), "cut at {cut}: {why}"); } + assert_eq!(all(&whole).unwrap().len(), 1); + assert_eq!(all(&[0u8; BLOCK]).expect("an end block ends it").len(), 0); } #[test] - fn a_header_that_does_not_sum_and_a_size_past_the_end_are_both_refused() { - let mut tampered = archive(&[("gbae/gbae", b'0', b"ELF", 0o755)]); + fn a_header_that_does_not_sum_or_is_not_ustar_is_refused() { + let mut tampered = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); tampered[3] ^= 0x20; - assert!(entries(&tampered).is_err()); + assert!(all(&tampered).is_err()); + + let mut v7 = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); + v7[MAGIC..MAGIC + 8].fill(0); + seal(&mut v7[..BLOCK]); + assert!(all(&v7).unwrap_err().contains("not a ustar header")); + } - let mut short = archive(&[("gbae/gbae", b'0', b"ELF", 0o755)]); - short.truncate(BLOCK + 1); - assert!(entries(&short).is_err()); + /// GNU tar's format keeps times where POSIX keeps the prefix, so its + /// prefix field is never part of the name. + #[test] + fn a_gnu_header_s_prefix_field_is_not_its_name() { + let mut tar = archive(&[("gbae/gbae", b'0', b"ELF", 0o755, "")]); + tar[MAGIC..MAGIC + 8].copy_from_slice(b"ustar \0"); + tar[345..352].copy_from_slice(b"1234567"); + seal(&mut tar[..BLOCK]); + assert_eq!(all(&tar).unwrap()[0].0.path, "gbae/gbae"); - assert!(entries(&[0u8; 7]).is_err()); - assert_eq!(entries(&[0u8; BLOCK]).expect("an end block ends it").len(), 0); + let mut posix = archive(&[("gbae", b'0', b"ELF", 0o755, "")]); + posix[345..349].copy_from_slice(b"deep"); + seal(&mut posix[..BLOCK]); + assert_eq!(all(&posix).unwrap()[0].0.path, "deep/gbae"); } } diff --git a/system/pkg/src/lib.rs b/system/pkg/src/lib.rs index b6069eadc71..dc1e3324ce0 100644 --- a/system/pkg/src/lib.rs +++ b/system/pkg/src/lib.rs @@ -2,96 +2,168 @@ //! //! **A package is a directory**, so installing one is: agree with the archive //! on a single top-level name, find the program inside it that name promises, -//! and write the manifest recording where the bytes came from. +//! check that it fits the room the volume has, and write the manifest +//! recording where the bytes came from. The archive is planned as it streams +//! past ([`Planner`]), and only a plan that finished is written. pub mod archive; pub mod sums; -use archive::{Entry, Kind}; +use std::collections::BTreeMap; + +use archive::{Header, Kind}; use toyos_manifest::package::{self, Package}; /// The suffix an installable archive carries. Not a guess about compression — /// the version is read out of the name, so the name has a shape or it has none. pub const SUFFIX: &str = ".tar.gz"; +/// The most entries one archive installs. Policy on the primitive: the plan +/// holds every entry's header until the archive has been read to its end, so +/// this bounds the installer's memory. A toolchain is a few thousand. +pub const MAX_ENTRIES: usize = 1 << 16; + /// Everything the writer needs, and nothing it has to decide. -pub struct Plan<'a> { +#[derive(Debug)] +pub struct Plan { pub manifest: Package, - /// Directories to create, shallowest first. - pub dirs: Vec, - /// `(whole guest path, contents)`. **No mode travels with them**: this + /// Every entry, in the archive's order, which the second read of the same + /// archive must meet again header for header. **No mode is written**: this /// filesystem has no permission bits, so the archive's mode is read as its /// statement about which file is the program and is never written. - pub files: Vec<(String, &'a [u8])>, + pub entries: Vec
, + /// The room the files take on the volume, each rounded up to its unit. + pub need: u64, +} + +/// A plan, made one header at a time. +pub struct Planner<'a> { + file_name: &'a str, + free: u64, + unit: u64, + entries: Vec
, + /// Every path the archive names or implies, and whether it is a directory. + /// One path is one thing: a path named twice, or a path beneath a file or a + /// link, would decide by write order what is at it, or write through a + /// link to somewhere the plan never checked. + seen: BTreeMap, + need: u64, } -/// What installing `file_name` would do, or why it would not. -pub fn plan<'a>( - file_name: &str, - digest: &str, - entries: &[Entry<'a>], -) -> Result, String> { - let name = single_top(entries)?; - let version = version_of(file_name, name)?; - let program = format!("{}/{name}", Package::dir(name)); - let inside = format!("{name}/{name}"); - match entries.iter().find(|e| e.path == inside) { - Some(e) if e.kind == Kind::File && e.executable() => {} - Some(_) => { - return Err(format!("pkg: {file_name}'s {inside} is not an executable file")); +impl<'a> Planner<'a> { + /// A plan for `file_name` on a volume with `free` bytes in units of `unit`. + pub fn new(file_name: &'a str, free: u64, unit: u64) -> Self { + Self { file_name, free, unit: unit.max(1), entries: Vec::new(), seen: BTreeMap::new(), need: 0 } + } + + /// The next header, refused the moment it breaks the package's shape or + /// takes the plan past the room: what an archive claims about its size is + /// refused before any of that size is read. + pub fn add(&mut self, header: Header) -> Result<(), String> { + let name = self.file_name; + if self.entries.len() == MAX_ENTRIES { + return Err(format!("pkg: {name} holds more than {MAX_ENTRIES} entries")); } - None => { + if let Some(first) = self.entries.first() { + if header.top() != first.top() { + return Err(format!( + "pkg: the archive holds both {:?} and {:?} at its top level", + first.top(), + header.top() + )); + } + } + if header.path == format!("{}/{}", header.top(), package::FILE) { + return Err(format!("pkg: {name} carries {}, which is the installer's to write", header.path)); + } + if let Kind::Symlink(target) = &header.kind { + link_stays_inside(&header.path, target) + .map_err(|why| format!("pkg: {name}'s link {} -> {target:?} {why}", header.path))?; + } + let parts: Vec<&str> = header.path.split('/').collect(); + for depth in 1..parts.len() { + let parent = parts[..depth].join("/"); + if !*self.seen.entry(parent.clone()).or_insert(true) { + return Err(format!("pkg: {name} puts {} beneath {parent}, which is not a directory", header.path)); + } + } + let is_dir = header.kind == Kind::Dir; + match self.seen.insert(header.path.clone(), is_dir) { + Some(true) if is_dir => {} + Some(_) => return Err(format!("pkg: {name} names {} twice", header.path)), + None => {} + } + self.need = self.need.saturating_add(header.size.div_ceil(self.unit).saturating_mul(self.unit)); + if self.need > self.free { return Err(format!( - "pkg: {file_name} carries no {inside}, so there is no program to launch" - )) + "pkg: {name} needs more than the {} bytes free under {}", + self.free, + package::DIR + )); } + self.entries.push(header); + Ok(()) } - let mut dirs = vec![Package::dir(name)]; - let mut files = Vec::new(); - for entry in entries { - let dest = format!("{}/{}", package::DIR, entry.path); - match entry.kind { - Kind::Dir => dirs.push(dest), - Kind::File => files.push((dest, entry.data)), - } + /// The plan, once the archive has ended, with the archive's `digest`. + pub fn finish(self, digest: &str) -> Result { + let file_name = self.file_name; + let name = + self.entries.first().ok_or_else(|| String::from("pkg: the archive is empty"))?.top().to_string(); + let version = version_of(file_name, &name)?; + let inside = program_of(&self.entries, &name) + .map_err(|why| format!("pkg: {file_name} {why}"))?; + let manifest = Package { + name, + version, + digest: digest.to_string(), + program: format!("{}/{inside}", package::DIR), + }; + // The render is the check: it is what refuses a name, a version or a + // program that would not come back out of the file the same way. + manifest.render()?; + Ok(Plan { manifest, entries: self.entries, need: self.need }) } - dirs.sort(); - dirs.dedup(); - // One path, one kind, over both lists: the write order is directories then - // files, so a path named as both would decide by that order what is at it. - let mut seen: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); - seen.extend(dirs.iter().map(String::as_str)); - seen.sort_unstable(); - let count = seen.len(); - seen.dedup(); - if seen.len() != count { - return Err(format!("pkg: {file_name} names one path twice")); - } - - let manifest = Package { - name: name.to_string(), - version, - digest: digest.to_string(), - program, - }; - // The render is the check: it is what refuses a name, a version or a - // program that would not come back out of the file the same way. - manifest.render()?; - Ok(Plan { manifest, dirs, files }) } -/// The one directory every entry is inside, which is the package's name. An -/// archive that spreads over two top-level names installs neither: picking the -/// first would install half of it under a name it never chose. -fn single_top<'a>(entries: &'a [Entry<'a>]) -> Result<&'a str, String> { - let first = entries.first().ok_or_else(|| String::from("pkg: the archive is empty"))?.top(); - match entries.iter().map(Entry::top).find(|top| *top != first) { - Some(other) => { - Err(format!("pkg: the archive holds both {first:?} and {other:?} at its top level")) - } - None => Ok(first), +/// The program a package launches: `/`, as a package of one +/// program has it, or `/bin/`, as a toolchain does, an executable +/// file either way. Both is a package that has not said which. +fn program_of(entries: &[Header], name: &str) -> Result { + let candidates = [format!("{name}/{name}"), format!("{name}/bin/{name}")]; + let found: Vec<&Header> = + entries.iter().filter(|e| candidates.contains(&e.path)).collect(); + match found[..] { + [one] if one.kind == Kind::File && one.executable() => Ok(one.path.clone()), + [one] => Err(format!("has {} and it is not an executable file", one.path)), + [] => Err(format!("carries neither {} nor {}, so there is no program to launch", candidates[0], candidates[1])), + _ => Err(format!("carries both {} and {}, so which one launches is not said", candidates[0], candidates[1])), + } +} + +/// Whether the symlink at `path` reaching `target` lands inside the package. +/// +/// **Only `..` first and names after**: each leading `..` climbs one real +/// directory of the link's own path, which [`Planner::add`] has made sure is a +/// directory and not a link, and the climb may not pass the package's own +/// directory. A `..` after a name would climb from wherever that name, perhaps +/// itself a link, resolves to, which no reading of the archive alone can say. +fn link_stays_inside(path: &str, target: &str) -> Result<(), &'static str> { + if target.starts_with('/') { + return Err("is absolute"); + } + let parts: Vec<&str> = target.split('/').collect(); + let climbs = parts.iter().take_while(|p| **p == "..").count(); + if parts[climbs..].iter().any(|p| p.is_empty() || *p == "." || *p == "..") { + return Err("has an empty, `.` or `..` component after a name"); + } + // `top/a/link` is in two directories of the package, `top/a` and `top`; + // `top` itself is in none. + let depth = path.split('/').count().checked_sub(2).ok_or("is the package's own directory")?; + if climbs > depth { + return Err("climbs out of the package"); } + Ok(()) } /// The version, read out of the release's own file name. @@ -117,69 +189,173 @@ mod tests { const DIGEST: &str = "99fcd8a7263b5c25cd90cead1baaa7200ef272100fc2226e008a4e8205ba2916"; const ASSET: &str = "gbae-v0.2.0-toyos-x86_64.tar.gz"; + const ROOMY: u64 = 1 << 40; - fn entry(path: &str, kind: Kind, data: &'static [u8], mode: u32) -> Entry<'static> { - Entry { path: path.to_string(), kind, data, mode } + fn entry(path: &str, kind: Kind, size: u64, mode: u32) -> Header { + Header { path: path.to_string(), kind, size, mode } } - fn gbae() -> Vec> { + fn link(path: &str, target: &str) -> Header { + entry(path, Kind::Symlink(target.to_string()), 0, 0o777) + } + + fn gbae() -> Vec
{ vec![ - entry("gbae", Kind::Dir, b"", 0o755), - entry("gbae/gbae", Kind::File, b"ELF...", 0o755), - entry("gbae/README.md", Kind::File, b"# gbae", 0o644), - entry("gbae/LICENSE", Kind::File, b"MIT", 0o644), + entry("gbae", Kind::Dir, 0, 0o755), + entry("gbae/gbae", Kind::File, 6, 0o755), + entry("gbae/README.md", Kind::File, 6, 0o644), + entry("gbae/LICENSE", Kind::File, 3, 0o644), ] } + fn plan_of(file_name: &str, entries: Vec
, free: u64) -> Result { + let mut planner = Planner::new(file_name, free, 4096); + for e in entries { + planner.add(e)?; + } + planner.finish(DIGEST) + } + #[test] fn the_real_asset_plans_the_directory_the_track_describes() { - let plan = plan(ASSET, DIGEST, &gbae()).expect("plan"); + let plan = plan_of(ASSET, gbae(), ROOMY).expect("plan"); assert_eq!(plan.manifest.name, "gbae"); assert_eq!(plan.manifest.version, "0.2.0"); assert_eq!(plan.manifest.digest, DIGEST); assert_eq!(plan.manifest.program, "/apps/gbae/gbae"); - assert_eq!(plan.dirs, ["/apps/gbae"]); - let paths: Vec<&str> = plan.files.iter().map(|(p, _)| p.as_str()).collect(); - assert_eq!(paths, ["/apps/gbae/gbae", "/apps/gbae/README.md", "/apps/gbae/LICENSE"]); + let paths: Vec<&str> = plan.entries.iter().map(|e| e.path.as_str()).collect(); + assert_eq!(paths, ["gbae", "gbae/gbae", "gbae/README.md", "gbae/LICENSE"]); + assert_eq!(plan.need, 3 * 4096); } + /// A toolchain's shape: its program in `bin`, beside a link to it, and no + /// explicit directory entries at all. #[test] - fn an_archive_that_is_not_one_directory_installs_nothing() { + fn a_toolchain_launches_its_bin_and_carries_its_links() { + let entries = vec![ + entry("clang/bin/clang", Kind::File, 10_000, 0o755), + link("clang/bin/cc", "clang"), + link("clang/lib/clang/22/x", "../../../bin/clang"), + entry("clang/sysroot/lib/libtoyos_c.a", Kind::File, 1, 0o644), + ]; + let plan = plan_of("clang-v22.1.0-toyos-x86_64.tar.gz", entries, ROOMY).expect("plan"); + assert_eq!(plan.manifest.program, "/apps/clang/bin/clang"); + assert_eq!(plan.manifest.version, "22.1.0"); + assert_eq!(plan.need, 3 * 4096 + 4096); + } + + #[test] + fn an_archive_that_is_not_one_directory_or_has_no_one_program_installs_nothing() { let mut two = gbae(); - two.push(entry("other/x", Kind::File, b"x", 0o644)); - assert!(plan(ASSET, DIGEST, &two).is_err()); - assert!(plan(ASSET, DIGEST, &[]).is_err()); + two.push(entry("other/x", Kind::File, 1, 0o644)); + assert!(plan_of(ASSET, two, ROOMY).unwrap_err().contains("at its top level")); + assert!(plan_of(ASSET, Vec::new(), ROOMY).is_err()); - let no_program = vec![ - entry("gbae", Kind::Dir, b"", 0o755), - entry("gbae/README.md", Kind::File, b"# gbae", 0o644), - ]; - assert!(plan(ASSET, DIGEST, &no_program).is_err()); + let no_program = vec![entry("gbae", Kind::Dir, 0, 0o755), entry("gbae/README.md", Kind::File, 6, 0o644)]; + assert!(plan_of(ASSET, no_program, ROOMY).unwrap_err().contains("no program to launch")); // A directory where the program should be is not a program, and - // neither is a file the archive never marked executable. - let dir_instead = - vec![entry("gbae", Kind::Dir, b"", 0o755), entry("gbae/gbae", Kind::Dir, b"", 0o755)]; - assert!(plan(ASSET, DIGEST, &dir_instead).is_err()); - let not_executable = - vec![entry("gbae", Kind::Dir, b"", 0o755), entry("gbae/gbae", Kind::File, b"x", 0o644)]; - assert!(plan(ASSET, DIGEST, ¬_executable).is_err()); + // neither is a file the archive never marked executable, nor a link. + for program in [ + entry("gbae/gbae", Kind::Dir, 0, 0o755), + entry("gbae/gbae", Kind::File, 1, 0o644), + link("gbae/gbae", "README.md"), + ] { + let why = plan_of(ASSET, vec![entry("gbae", Kind::Dir, 0, 0o755), program], ROOMY).unwrap_err(); + assert!(why.contains("not an executable file"), "{why}"); + } + + let mut both = gbae(); + both.push(entry("gbae/bin/gbae", Kind::File, 1, 0o755)); + assert!(plan_of(ASSET, both, ROOMY).unwrap_err().contains("which one launches")); } #[test] - fn a_path_the_archive_names_as_both_a_file_and_a_directory_is_refused() { - let mut both = gbae(); - both.push(entry("gbae/README.md", Kind::Dir, b"", 0o755)); - let why = plan(ASSET, DIGEST, &both).err().expect("a path named twice was planned"); - assert!(why.contains("names one path twice"), "{why}"); + fn a_path_named_twice_or_beneath_a_file_or_a_link_is_refused() { + for extra in [ + entry("gbae/README.md", Kind::Dir, 0, 0o755), + entry("gbae/LICENSE", Kind::File, 5, 0o644), + link("gbae/LICENSE", "README.md"), + entry("gbae/gbae", Kind::Dir, 0, 0o755), + entry("gbae", Kind::File, 1, 0o644), + ] { + let mut twice = gbae(); + twice.push(extra.clone()); + let why = plan_of(ASSET, twice, ROOMY).expect_err(&format!("{extra:?} was planned")); + assert!(why.contains("twice"), "{why}"); + } + let mut manifest = gbae(); + manifest.push(entry("gbae/manifest.toml", Kind::File, 1, 0o644)); + assert!(plan_of(ASSET, manifest, ROOMY).unwrap_err().contains("the installer's to write")); + + // A directory given again is the same directory. + let mut again = gbae(); + again.push(entry("gbae", Kind::Dir, 0, 0o755)); + assert!(plan_of(ASSET, again, ROOMY).is_ok()); - let mut twice = gbae(); - twice.push(entry("gbae/LICENSE", Kind::File, b"other", 0o644)); - assert!(plan(ASSET, DIGEST, &twice).is_err()); + // Written beneath a link, a file lands wherever the link points. + for beneath in [ + vec![link("gbae/lib", "share"), entry("gbae/lib/x", Kind::File, 1, 0o644)], + vec![link("gbae/a/up", ".."), entry("gbae/a/up/b/x", Kind::File, 1, 0o644)], + vec![entry("gbae/LICENSE/x", Kind::File, 1, 0o644)], + ] { + let mut entries = gbae(); + entries.extend(beneath); + let why = plan_of(ASSET, entries, ROOMY).unwrap_err(); + assert!(why.contains("which is not a directory"), "{why}"); + } + } - let mut program = gbae(); - program.push(entry("gbae/gbae", Kind::Dir, b"", 0o755)); - assert!(plan(ASSET, DIGEST, &program).is_err()); + /// The links a package may carry are the ones that resolve inside it, + /// whatever the links around them resolve to. + #[test] + fn a_link_whose_target_leaves_the_package_is_refused() { + let top = plan_of(ASSET, vec![link("gbae", "elsewhere")], ROOMY).unwrap_err(); + assert!(top.contains("the package's own directory"), "{top}"); + for (path, target) in [ + ("gbae/x", "/system/bin/supervisor"), + ("gbae/x", ".."), + ("gbae/x", "../other/other"), + ("gbae/a/x", "../../.."), + // `a/..` is `gbae` read as text, and wherever `a`'s link points + // read as the filesystem does. + ("gbae/x", "a/../.."), + ("gbae/x", "a/.."), + ("gbae/x", "./gbae"), + ("gbae/x", "a//b"), + ] { + let mut entries = gbae(); + entries.push(link(path, target)); + let why = plan_of(ASSET, entries, ROOMY).expect_err(&format!("{path} -> {target} was planned")); + assert!(why.contains("link"), "{why}"); + } + for (path, target) in [("gbae/x", "gbae"), ("gbae/a/x", ".."), ("gbae/a/b/x", "../../gbae"), ("gbae/a/x", "../b/c")] + { + let mut entries = gbae(); + entries.push(link(path, target)); + assert!(plan_of(ASSET, entries, ROOMY).is_ok(), "{path} -> {target} was refused"); + } + } + + /// What does not fit is refused at the header that claims it, before a + /// byte of its data is read. + #[test] + fn an_entry_past_the_room_or_past_the_count_is_refused_at_its_header() { + let mut planner = Planner::new(ASSET, 2 * 4096, 4096); + planner.add(entry("gbae/gbae", Kind::File, 4096, 0o755)).unwrap(); + planner.add(entry("gbae/a", Kind::File, 1, 0o644)).unwrap(); + let why = planner.add(entry("gbae/b", Kind::File, 1, 0o644)).unwrap_err(); + assert!(why.contains("needs more than the 8192 bytes free"), "{why}"); + + let huge = entry("gbae/gbae", Kind::File, u64::MAX / 2, 0o755); + assert!(Planner::new(ASSET, ROOMY, 4096).add(huge).is_err()); + + let mut many = Planner::new(ASSET, ROOMY, 4096); + many.add(entry("gbae", Kind::Dir, 0, 0o755)).unwrap(); + for i in 1..MAX_ENTRIES { + many.add(entry(&format!("gbae/{i}"), Kind::Dir, 0, 0o755)).unwrap(); + } + assert!(many.add(entry("gbae/last", Kind::Dir, 0, 0o755)).unwrap_err().contains("more than")); } #[test] @@ -196,7 +372,13 @@ mod tests { /// producing a directory the supervisor cannot resolve. #[test] fn a_digest_the_manifest_cannot_carry_stops_the_install() { - assert!(plan(ASSET, "not-a-digest", &gbae()).is_err()); - assert!(plan(ASSET, &DIGEST.to_uppercase(), &gbae()).is_err()); + let planned = |digest: &str| { + let mut planner = Planner::new(ASSET, ROOMY, 4096); + gbae().into_iter().try_for_each(|e| planner.add(e)).unwrap(); + planner.finish(digest) + }; + assert!(planned(DIGEST).is_ok()); + assert!(planned("not-a-digest").is_err()); + assert!(planned(&DIGEST.to_uppercase()).is_err()); } } diff --git a/system/pkg/src/main.rs b/system/pkg/src/main.rs index e1dc2a89bed..5133a1660e9 100644 --- a/system/pkg/src/main.rs +++ b/system/pkg/src/main.rs @@ -6,27 +6,34 @@ //! `SHA256SUMS`, ask, unpack, and write the `manifest.toml` that records which //! binary the directory launches. //! +//! **An archive is read twice and never held.** The first read hashes it and +//! plans it within the room `/apps`'s volume has; the second writes it, and has +//! to meet the plan header for header and the digest again at its end, so an +//! archive changed between the two leaves nothing installed. +//! //! The decisions are `lib.rs`'s and are made on the host; what is left here is //! the filesystem and the question put to the user. use std::fs; -use std::io::{BufRead, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::Path; -use flate2::read::GzDecoder; -use pkg::{archive, sums}; +use flate2::bufread::MultiGzDecoder; +use pkg::archive::{self, Kind}; +use pkg::{sums, Plan, Planner}; use toyos_manifest::package::{self, Package}; /// Answers the consent prompt in advance, for a caller with no terminal. /// Spelled out rather than `-y`: installing without asking says so in full. const ASSUME_YES: &str = "--yes"; -const MAX_INFLATED: u64 = 256 * 1024 * 1024; - /// The release publishes one covering every asset, so it is found by sitting /// beside the archive rather than by being named on the command line. const SUMS: &str = "SHA256SUMS"; +/// How much of an archive one read takes, and of a file one write. +const CHUNK: usize = 1 << 20; + fn main() -> std::process::ExitCode { let args: Vec = std::env::args().skip(1).collect(); let words: Vec<&str> = args.iter().map(String::as_str).collect(); @@ -64,20 +71,18 @@ fn install(file: &Path, assume_yes: bool) -> Result<(), String> { .map_err(|e| format!("pkg: cannot read {}: {e}", beside.display()))?; let want = sums::digest_for(&sums_text, &name)?; - let bytes = fs::read(file).map_err(|e| format!("pkg: cannot read {}: {e}", file.display()))?; - let got = sums::hex(&toyos_sha2_hw::sha256_digest(&bytes)); + let mut apps = apps()?; + let room = apps.room().map_err(|e| format!("pkg: {} answers no room: {e:?}", package::DIR))?; + let (planner, got) = survey(file, Planner::new(&name, room.free, room.unit))?; if got != want { return Err(format!("pkg: {name} hashes to {got} and {SUMS} says {want}")); } println!("pkg: verified {name} against {SUMS} ({got})"); - - let tar = inflate(&name, &bytes, MAX_INFLATED)?; - let entries = archive::entries(&tar)?; - let plan = pkg::plan(&name, &got, &entries)?; + let plan = planner.finish(&got)?; let package = &plan.manifest; let dir = Package::dir(&package.name); - if fs::metadata(&dir).is_ok() { + if fs::symlink_metadata(&dir).is_ok() { return Err(format!( "pkg: {dir} is already there — remove it first, this installer does not replace" )); @@ -85,73 +90,152 @@ fn install(file: &Path, assume_yes: bool) -> Result<(), String> { if !assume_yes && !consented(package, file)? { return Err(format!("pkg: not installing {}", package.name)); } - - // The manifest is written last, so a directory carrying one is a finished - // install: the supervisor and the launcher both reach a package through it. - write_package( - &dir, - &plan.dirs, - &plan.files, - &Package::path(&package.name), - &package.render()?, - )?; + let mut durable = || apps.sync().map_err(|e| format!("pkg: {}'s volume did not write what it was given: {e:?}", package::DIR)); + write_package(file, &plan, Path::new(package::DIR), &mut durable)?; println!( - "pkg: installed {} {} at {dir}, launching {}", - package.name, package.version, package.program + "pkg: installed {} {} at {dir} ({} bytes), launching {}", + package.name, package.version, plan.need, package.program ); Ok(()) } -/// Inflate `bytes`, refusing an archive that claims more than `ceiling`. -/// -/// **The inflated size is the archive's claim and `SHA256SUMS` says nothing -/// about it**, so the reader is bounded rather than the output measured after: -/// measuring is the allocation this refuses. -fn inflate(name: &str, bytes: &[u8], ceiling: u64) -> Result, String> { - use std::io::Read; - let mut tar = Vec::new(); - GzDecoder::new(bytes) - .take(ceiling + 1) - .read_to_end(&mut tar) - .map_err(|e| format!("pkg: {name} is not gzip: {e}"))?; - if tar.len() as u64 > ceiling { - return Err(format!("pkg: {name} inflates past {ceiling} bytes and is refused unread")); +/// `/apps`'s own capability, for what std has no word for: the room on its +/// volume, and a sync of the whole volume that answers for every file in it. +fn apps() -> Result { + let names = toyos::endow::namespace() + .ok_or_else(|| String::from("pkg: this process holds no directory capabilities"))?; + let name = format!("{}{}", toyos::fs::CAPABILITY_PREFIX, package::DIR); + toyos::fs::Dir::connect(names, &name).map_err(|e| format!("pkg: cannot reach {}: {e:?}", package::DIR)) +} + +/// The first read: every header through `planner`, and the whole file's +/// digest. +fn survey<'a>(file: &Path, mut planner: Planner<'a>) -> Result<(Planner<'a>, String), String> { + let mut hashed = Hashed::open(file)?; + let mut tar = archive::Reader::new(gunzip(&mut hashed)); + while let Some(header) = tar.next_entry()? { + planner.add(header)?; } - Ok(tar) + drop(tar); + Ok((planner, hashed.finish()?)) } -/// Write a package's directory, and take it back down if any write fails: a -/// verified archive can still run out of volume half way, and a directory with -/// no `manifest.toml` is a name `install` refuses for the volume's life. +/// Unpack `plan` from `file` under `root` and write its manifest last, or take +/// the package's directory back down: a verified archive can still run out of +/// volume half way, and a directory with no `manifest.toml` is a name `install` +/// refuses for the volume's life. Its directory is made first and alone, so a +/// name some other writer took is refused and never swept. +/// +/// **`durable` is asked before the manifest is written**, and refuses if the +/// volume holds any file it could not write: a file's last close can be +/// refused, and std's `File` drops that answer. fn write_package( - dir: &str, - dirs: &[String], - files: &[(String, &[u8])], - manifest: &str, - text: &str, + file: &Path, + plan: &Plan, + root: &Path, + durable: &mut dyn FnMut() -> Result<(), String>, ) -> Result<(), String> { - match write_all(dirs, files, manifest, text) { + let dir = root.join(&plan.manifest.name); + fs::create_dir(&dir).map_err(|e| format!("pkg: cannot create {}: {e}", dir.display()))?; + match unpack(file, plan, root).and_then(|()| durable()).and_then(|()| write_manifest(&dir, &plan.manifest)) { Ok(()) => Ok(()), - Err(why) => match remove_tree(Path::new(dir)) { - Ok(()) => Err(format!("{why}; {dir} was taken back down")), + Err(why) => match remove_tree(&dir) { + Ok(()) => Err(format!("{why}; {} was taken back down", dir.display())), Err(swept) => Err(format!("{why}; {swept}")), }, } } -fn write_all( - dirs: &[String], - files: &[(String, &[u8])], - manifest: &str, - text: &str, -) -> Result<(), String> { - for dir in dirs { - fs::create_dir_all(dir).map_err(|e| format!("pkg: cannot create {dir}: {e}"))?; +/// The second read, writing what the first one planned. +fn unpack(file: &Path, plan: &Plan, root: &Path) -> Result<(), String> { + let changed = || format!("pkg: {} changed since it was verified", file.display()); + let mut hashed = Hashed::open(file)?; + let mut tar = archive::Reader::new(gunzip(&mut hashed)); + let mut buf = vec![0u8; CHUNK]; + for want in &plan.entries { + if tar.next_entry()?.as_ref() != Some(want) { + return Err(changed()); + } + let dest = root.join(&want.path); + let parent = || match dest.parent() { + Some(up) => fs::create_dir_all(up), + None => Ok(()), + }; + let made = match &want.kind { + Kind::Dir => fs::create_dir_all(&dest).map_err(|e| e.to_string()), + // The one link call std has for every OS this compiles for: ToyOS, + // and the host its tests run on. + #[allow(deprecated)] + Kind::Symlink(target) => parent().and_then(|()| fs::soft_link(target, &dest)).map_err(|e| e.to_string()), + Kind::File => parent() + .and_then(|()| fs::File::create_new(&dest)) + .map_err(|e| e.to_string()) + .and_then(|out| copy_data(&mut tar, out, &mut buf)), + }; + made.map_err(|e| format!("pkg: cannot write {}: {e}", dest.display()))?; + } + if tar.next_entry()?.is_some() { + return Err(changed()); + } + drop(tar); + match hashed.finish()? == plan.manifest.digest { + true => Ok(()), + false => Err(changed()), + } +} + +/// The current entry's data, into `out`. +fn copy_data(tar: &mut archive::Reader, mut out: fs::File, buf: &mut [u8]) -> Result<(), String> { + loop { + match tar.read_data(buf)? { + 0 => return Ok(()), + n => out.write_all(&buf[..n]).map_err(|e| e.to_string())?, + } } - for (path, data) in files { - fs::write(path, data).map_err(|e| format!("pkg: cannot write {path}: {e}"))?; +} + +/// The manifest, made durable before the install is called done: written last, +/// so a directory carrying one is a finished install, which the supervisor and +/// the launcher both reach a package through. +fn write_manifest(dir: &Path, package: &Package) -> Result<(), String> { + let path = dir.join(package::FILE); + let text = package.render()?; + fs::File::create_new(&path) + .and_then(|mut f| f.write_all(text.as_bytes()).and_then(|()| f.sync_all())) + .map_err(|e| format!("pkg: cannot write {}: {e}", path.display())) +} + +fn gunzip(r: R) -> MultiGzDecoder> { + MultiGzDecoder::new(BufReader::with_capacity(CHUNK, r)) +} + +/// A file read through a SHA-256 of every byte read. +struct Hashed { + file: fs::File, + hash: toyos_sha2::Sha256, +} + +impl Hashed { + fn open(path: &Path) -> Result { + let file = fs::File::open(path).map_err(|e| format!("pkg: cannot read {}: {e}", path.display()))?; + Ok(Self { file, hash: toyos_sha2_hw::sha256() }) + } + + /// The whole file's digest: what was read, and the rest, read now. A tar + /// ends before its file does, in the zeros gzip and tar pad it with, and + /// `SHA256SUMS` covers every byte. + fn finish(mut self) -> Result { + std::io::copy(&mut self, &mut std::io::sink()).map_err(|e| format!("pkg: {e}"))?; + Ok(sums::hex(&self.hash.finalize())) + } +} + +impl Read for Hashed { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.file.read(buf)?; + self.hash.update(&buf[..n]); + Ok(n) } - fs::write(manifest, text).map_err(|e| format!("pkg: cannot write {manifest}: {e}")) } /// The question, asked where the user typed the command. A closed or empty @@ -195,8 +279,8 @@ fn remove(name: &str) -> Result<(), String> { Ok(()) } -/// How deep this walk goes. Policy on the primitive: any process can plant a -/// Delete a directory and everything under it. +/// Delete a directory and everything under it, a link as itself and never +/// what it points at. /// /// **Not `fs::remove_dir_all`**: that one empties a directory and leaves it /// (`issues/remove-dir-all-empties-a-directory-and-leaves-it.md`), @@ -261,21 +345,177 @@ mod tests { use super::*; use toyos_tmpdir::TempDir; + const ASSET: &str = "tool-v1.0-toyos-x86_64.tar.gz"; + + /// A gzipped tar of `(path, data)` files and `(path, target)` links, as a + /// writer outside this crate makes one. + fn gzipped(files: &[(&str, &[u8])], links: &[(&str, &str)]) -> Vec { + let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast())); + for (path, data) in files { + let mut header = tar::Header::new_ustar(); + header.set_size(data.len() as u64); + header.set_mode(if path.ends_with("tool") { 0o755 } else { 0o644 }); + tar.append_data(&mut header, path, *data).expect("a file"); + } + for (path, target) in links { + let mut header = tar::Header::new_ustar(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + tar.append_link(&mut header, path, target).expect("a link"); + } + tar.into_inner().and_then(|gz| gz.finish()).expect("an archive") + } + + /// A toolchain's shape: its program in `bin`, a link to it, and a file + /// past several reads, whose bytes `seed` decides. + fn tool(seed: u32) -> Vec { + let big: Vec = (0..3 * CHUNK as u32 + 7).map(|i| ((i + seed) * 31 % 251) as u8).collect(); + gzipped(&[("tool/bin/tool", b"ELF"), ("tool/share/big", &big)], &[("tool/bin/cc", "tool")]) + } + + /// The archive at `scratch/ASSET`, surveyed on a volume of `free` bytes. + fn surveyed(scratch: &Path, bytes: &[u8], free: u64) -> Result { + let file = scratch.join(ASSET); + fs::write(&file, bytes).unwrap(); + let (planner, digest) = survey(&file, Planner::new(ASSET, free, 4096))?; + planner.finish(&digest) + } + + #[test] + fn an_archive_installs_whole_its_files_its_links_and_its_manifest() { + let scratch = TempDir::new("pkg-install"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + assert_eq!(plan.manifest.program, "/apps/tool/bin/tool"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect("installed"); + + let big = fs::read(root.join("tool/share/big")).unwrap(); + assert_eq!(big.len(), 3 * CHUNK + 7); + assert!(big.iter().enumerate().all(|(i, &b)| b == (i as u32 * 31 % 251) as u8)); + assert_eq!(fs::read_link(root.join("tool/bin/cc")).unwrap(), Path::new("tool")); + assert_eq!(fs::read(root.join("tool/bin/cc")).unwrap(), b"ELF"); + let manifest = fs::read_to_string(root.join("tool/manifest.toml")).unwrap(); + assert_eq!(Package::parse(&manifest).unwrap(), plan.manifest); + } + + /// An archive replaced between the read that verified it and the one that + /// writes it installs nothing. + #[test] + fn an_archive_changed_after_its_survey_installs_nothing() { + let scratch = TempDir::new("pkg-changed"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + let other = gzipped(&[("tool/bin/tool", b"ELF"), ("tool/share/big", b"short")], &[]); + // The same headers over other bytes: only the digest tells. + for replaced in [other, tool(1)] { + fs::write(scratch.join(ASSET), &replaced).unwrap(); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("a changed archive installed"); + assert!(why.contains("was taken back down"), "{why}"); + assert!(!root.join("tool").exists(), "a changed archive left its directory: {why}"); + } + } + + /// The second read writes only what the first one checked: an archive + /// swapped for one that plants a link and writes through it reaches + /// nothing outside the package, though its digest is only known at its end. + #[test] + fn a_swapped_archive_writes_nothing_its_plan_did_not_check() { + let scratch = TempDir::new("pkg-swapped"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let (root, outside) = (scratch.join("apps"), scratch.join("outside")); + fs::create_dir(&root).unwrap(); + fs::create_dir(&outside).unwrap(); + let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast())); + let mut link = tar::Header::new_ustar(); + link.set_entry_type(tar::EntryType::Symlink); + link.set_size(0); + link.set_mode(0o777); + tar.append_link(&mut link, "tool/out", "../../outside").unwrap(); + let mut file = tar::Header::new_ustar(); + file.set_size(1); + file.set_mode(0o644); + tar.append_data(&mut file, "tool/out/planted", &b"x"[..]).unwrap(); + fs::write(scratch.join(ASSET), tar.into_inner().and_then(|gz| gz.finish()).unwrap()).unwrap(); + + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("a swapped archive installed"); + assert!(why.contains("changed since it was verified"), "{why}"); + assert!(!outside.join("planted").exists(), "the swapped archive wrote through its link"); + assert!(!root.join("tool").exists()); + } + + /// The digest is of every byte of the file, past the tar's end and the + /// gzip member's, which the archive's reader never asks for; a cut archive + /// is refused where it ends. #[test] - fn an_install_that_cannot_finish_takes_its_directory_back_down() { - let root = TempDir::new("pkg-unwind"); - let dir = root.join("gbae"); - let dir_s = dir.to_str().unwrap().to_string(); - let good = format!("{dir_s}/gbae"); - // `gbae` is a file, so a write to `gbae/inside` cannot succeed. - let doomed = format!("{good}/inside"); - let files: Vec<(String, &[u8])> = vec![(good.clone(), b"ELF".as_slice()), (doomed, b"x")]; - - let why = write_package(&dir_s, &[dir_s.clone()], &files, "unreached", "unreached") - .expect_err("the second write cannot succeed"); - assert!(why.contains("was taken back down"), "{why}"); - assert!(!dir.exists(), "{dir_s} survived a failed install"); - assert!(root.exists(), "the unwind went past the package's own directory"); + fn the_digest_covers_the_whole_file_and_a_cut_archive_is_refused() { + let scratch = TempDir::new("pkg-truncated"); + let file = scratch.join(ASSET); + let survey_of = |bytes: &[u8]| { + fs::write(&file, bytes).unwrap(); + survey(&file, Planner::new(ASSET, 1 << 30, 4096)).map(|(_, digest)| digest) + }; + let mut whole = tool(0); + whole.extend(vec![0u8; 3 * CHUNK]); + assert_eq!(survey_of(&whole).unwrap(), sums::hex(&toyos_sha2_hw::sha256_digest(&whole))); + assert!(survey_of(&whole[..tool(0).len() / 3]).unwrap_err().contains("tar:")); + assert!(survey_of(b"not gzip at all").is_err()); + } + + /// The room is the bound on what a survey inflates: a header claiming + /// more is refused at that header, and nothing past it is read. + #[test] + fn an_archive_bigger_than_the_room_is_refused_unread() { + let scratch = TempDir::new("pkg-room"); + let why = surveyed(&scratch, &tool(0), 64 * 1024).expect_err("3 MiB planned into 64 KiB"); + assert!(why.contains("needs more than the 65536 bytes free"), "{why}"); + + // A bomb: a 4 GiB file of zeros, a few megabytes compressed, refused + // at its header. + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); + let mut header = tar::Header::new_ustar(); + header.set_path("tool/bin/tool").unwrap(); + header.set_size(4 << 30); + header.set_mode(0o755); + header.set_cksum(); + gz.write_all(header.as_bytes()).unwrap(); + let zeros = vec![0u8; 1 << 20]; + for _ in 0..64 { + gz.write_all(&zeros).unwrap(); + } + let bomb = gz.finish().unwrap(); + let why = surveyed(&scratch, &bomb, 1 << 30).expect_err("a 4 GiB entry planned into 1 GiB"); + assert!(why.contains("needs more than"), "{why}"); + } + + /// A volume that could not write every file of the package installs + /// nothing, though every write was answered. + #[test] + fn a_volume_that_refuses_its_sync_installs_nothing() { + let scratch = TempDir::new("pkg-durable"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir(&root).unwrap(); + let mut refused = || Err(String::from("pkg: the volume did not write it")); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut refused).expect_err("installed unwritten"); + assert!(why.contains("did not write it") && why.contains("was taken back down"), "{why}"); + assert!(!root.join("tool").exists()); + } + + /// A name some other writer holds is refused and left as it was found: + /// only a directory this install made is one its unwind may take down. + #[test] + fn a_directory_already_at_the_name_is_refused_and_never_swept() { + let scratch = TempDir::new("pkg-taken"); + let plan = surveyed(&scratch, &tool(0), 1 << 30).expect("a plan"); + let root = scratch.join("apps"); + fs::create_dir_all(root.join("tool")).unwrap(); + fs::write(root.join("tool/theirs"), b"x").unwrap(); + let why = write_package(&scratch.join(ASSET), &plan, &root, &mut || Ok(())).expect_err("an existing directory"); + assert!(why.contains("cannot create"), "{why}"); + assert!(root.join("tool/theirs").exists(), "a directory the installer did not make was swept"); } /// 33 is the first depth the old bound refused; 256 is far past it. @@ -294,23 +534,16 @@ mod tests { // A second branch, so the walk is a tree and not a chain. fs::create_dir_all(root.join("wide/er")).expect("a second branch"); fs::write(root.join("wide/er/leaf"), b"y").expect("a second leaf"); + // A link to outside the tree goes as itself. + let outside = scratch.join("outside"); + fs::create_dir(&outside).unwrap(); + fs::write(outside.join("kept"), b"z").unwrap(); + #[allow(deprecated)] + fs::soft_link(&outside, root.join("wide/out")).unwrap(); remove_tree(&root).unwrap_or_else(|e| panic!("depth {depth}: {e}")); assert!(!root.exists(), "a tree {depth} deep survived its removal"); + assert!(outside.join("kept").exists(), "the removal followed a link"); } } - - #[test] - fn an_archive_that_inflates_past_the_ceiling_is_refused_unread() { - use std::io::Write; - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); - gz.write_all(&vec![0u8; 64 * 1024]).expect("deflate"); - let bomb = gz.finish().expect("a gzip stream"); - assert!(bomb.len() < 1024, "the fixture is not a bomb: {} bytes", bomb.len()); - - let why = inflate("bomb.tar.gz", &bomb, 4096).expect_err("64 KiB past a 4 KiB ceiling"); - assert!(why.contains("inflates past 4096 bytes"), "{why}"); - assert_eq!(inflate("bomb.tar.gz", &bomb, 128 * 1024).expect("under it").len(), 64 * 1024); - assert!(inflate("not.tar.gz", b"not gzip at all", 4096).is_err()); - } } diff --git a/tests/common/hostedclang.rs b/tests/common/hostedclang.rs index ffbf72ba931..3178859ed1e 100644 --- a/tests/common/hostedclang.rs +++ b/tests/common/hostedclang.rs @@ -1,20 +1,18 @@ -//! clang and LLD built for ToyOS (`build/hostedclang.rs`), run inside a guest: -//! M2 of `issues/toyos-builds-itself.md`. +//! clang and LLD built for ToyOS (`build/hostedclang.rs`), installed by `pkg` +//! and run inside a guest: M2 of `issues/toyos-builds-itself.md`. //! -//! The binaries, clang's resource headers and the C sysroot's headers and -//! `libtoyos_c.a` go on the ROOT of `tests/images/hostedclangcase.toml`, as fixtures -//! beside the image's own files: ROOT is memory the loader filled, so the -//! guest reads them with no disk and no server behind it. The job compiles -//! `hello.c` there and has the driver link it, which starts LLD as a child, -//! runs it, and compares each product with what the host's clang and LLD, of -//! the same LLVM commit, made of the same input on the same command line. +//! The package and its `SHA256SUMS` go on the ROOT of +//! `tests/images/hostedclangcase.toml` as fixtures, beside `hello.c` and what +//! the host's clang and LLD, of the same LLVM commit, made of it on the same +//! command line. The job installs the package into `/apps` on DATA with +//! `/system/bin/pkg`, compiles `hello.c` with the installed clang, has its +//! driver link it, which starts LLD as a child, runs it, and compares each +//! product with the host's. use std::fs; -use std::path::Path; use std::process::Command; use std::time::Duration; - use super::{compile, qemu}; use qemu::{BootOptions, QemuInstance}; @@ -27,51 +25,51 @@ const STAGED: &str = "share/hosted-clang"; /// What `hello.c` prints. const HELLO: &str = "hello from a C program compiled and linked inside ToyOS"; -/// The host's half: its clang and LLD on `hello.c`, the guest's on the same -/// file with the same arguments, and the guest's verdict on their bytes. +/// DATA's size: the installed toolchain is about 300 MB, which the profile's +/// 128 MiB does not hold. +const DATA_BYTES: u64 = 1 << 30; + +/// The host's half: the package, its clang and LLD on `hello.c`, and the +/// guest's verdict on the bytes its own made of the same file with the same +/// arguments. pub fn hosted_clang_hello() -> Result<(), String> { let hosted = toyos_build::hostedclang::ensure(&compile::repo_root()); - let c = compile::c_sysroot(); - let (case, sources) = (compile::repo_root().join("tests/images/hostedclangcase.toml"), compile::repo_root().join("tests/guest/c")); + let sources = compile::repo_root().join("tests/guest/c"); + let case = compile::repo_root().join("tests/images/hostedclangcase.toml"); let scratch = super::lane::dir().join(JOB); fs::create_dir_all(&scratch).map_err(|e| format!("create {}: {e}", scratch.display()))?; let (object, linked) = (scratch.join("hello.o"), scratch.join("a.out")); for (args, to) in [(&["-c"][..], &object), (&[][..], &linked)] { - let mut cc = Command::new(&c.clang); - cc.args(c.args()).args(args).arg("hello.c").arg("-o").arg(to).current_dir(&sources); + let mut cc = Command::new(&hosted.c.clang); + cc.args(hosted.c.args()).args(args).arg("hello.c").arg("-o").arg(to).current_dir(&sources); host_run(cc)?; } + let packaged = hosted.package(&scratch.join("package")); let mut staged: Vec<(String, Vec)> = Vec::new(); - let mut stage = |rel: &str, from: &Path| { - let bytes = fs::read(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())); + let mut stage = |rel: &str, from: &std::path::Path| -> Result<(), String> { + let bytes = fs::read(from).map_err(|e| format!("read {}: {e}", from.display()))?; staged.push((format!("{STAGED}/{rel}"), bytes)); + Ok(()) }; - stage("bin/clang", &hosted.dir.join("bin/clang")); - stage("bin/ld.lld", &hosted.dir.join("bin/ld.lld")); - stage("sysroot/lib/libtoyos_c.a", &c.dir.join("lib/libtoyos_c.a")); - stage("src/hello.c", &sources.join("hello.c")); + let archive = packaged.archive.file_name().and_then(|n| n.to_str()).expect("a package file name").to_string(); + stage(&archive, &packaged.archive)?; + stage("SHA256SUMS", &packaged.sums)?; + stage("src/hello.c", &sources.join("hello.c"))?; for (name, product) in [("hello.o", &object), ("a.out", &linked)] { let bytes = fs::read(product).map_err(|e| format!("read {}: {e}", product.display()))?; let sha256: String = toyos_sha2_hw::sha256_digest(&bytes).iter().map(|b| format!("{b:02x}")).collect(); eprintln!(" [{JOB}] the host's {name}: {} bytes, sha256 {sha256}", bytes.len()); - stage(&format!("host/{name}"), product); - } - // clang finds its resource headers beside itself, `../lib/clang`. - for (rel, from) in tree(&hosted.dir.join("lib/clang")) { - stage(&format!("lib/clang/{rel}"), &from); - } - // The C headers alone: `include/c++` is libc++'s, which no C compile reads. - for (rel, from) in tree(&c.dir.join("include")).into_iter().filter(|(rel, _)| !rel.starts_with("c++/")) { - stage(&format!("sysroot/include/{rel}"), &from); + stage(&format!("host/{name}"), product)?; } + eprintln!(" [{JOB}] the package: {archive}, {}", fs::read_to_string(&packaged.sums).unwrap_or_default().trim()); fs::remove_dir_all(&scratch).map_err(|e| format!("remove {}: {e}", scratch.display()))?; - let options = BootOptions { extra_root_files: staged, ..Default::default() }; + let options = BootOptions { extra_root_files: staged, data_bytes: Some(DATA_BYTES), ..Default::default() }; let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); let mut qemu = QemuInstance::boot_with_options(&case, &[], &[(JOB.to_string(), bin)], options); - let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED}"), Duration::from_secs(600)); + let result = qemu.run_test(&format!("test_rs_{JOB} /system/{STAGED} {archive}"), Duration::from_secs(900)); if let Some(why) = &result.error { return Err(format!("{why}\nthe job said:\n{}", result.stdout)); } @@ -83,7 +81,7 @@ pub fn hosted_clang_hello() -> Result<(), String> { return Err(format!("the job never said `{said}`:\n{}", result.stdout)); } } - for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: "))) { + for line in result.stdout.lines().filter(|l| l.contains(&format!("{JOB}: ")) || l.contains("pkg: ")) { eprintln!(" [{JOB}] {}", line.trim()); } Ok(()) @@ -102,21 +100,3 @@ fn host_run(mut command: Command) -> Result<(), String> { } Ok(()) } - -/// Every file under `dir`, by its path relative to `dir`. -fn tree(dir: &Path) -> Vec<(String, std::path::PathBuf)> { - let mut files = Vec::new(); - let mut pending = vec![dir.to_path_buf()]; - while let Some(at) = pending.pop() { - for entry in fs::read_dir(&at).unwrap_or_else(|e| panic!("read {}: {e}", at.display())) { - let path = entry.unwrap_or_else(|e| panic!("read {}: {e}", at.display())).path(); - if path.is_dir() { - pending.push(path); - } else { - let rel = path.strip_prefix(dir).expect("under the walked directory"); - files.push((rel.to_string_lossy().into_owned(), path)); - } - } - } - files -} diff --git a/tests/common/qemu.rs b/tests/common/qemu.rs index 5552b48ff60..ba2940e518b 100644 --- a/tests/common/qemu.rs +++ b/tests/common/qemu.rs @@ -1175,6 +1175,9 @@ pub struct BootOptions { /// the image is memoized on their names and bytes, so two boots staging /// different fixtures do not share one. pub extra_root_files: Vec<(String, Vec)>, + /// DATA's size in place of the profile's, for a boot that writes more + /// than [`NVME_SMALL`] holds, on a profile whose disk carries the image. + pub data_bytes: Option, /// Have QEMU record every PSCI call a vCPU makes into this file, with the /// calling CPU's affinity: the firmware side's own account of what the /// kernel asked of it. @@ -1219,6 +1222,7 @@ impl Default for BootOptions { mute: false, ready_marker: DEFAULT_READY, extra_root_files: Vec::new(), + data_bytes: None, psci_trace: None, rtc_base: None, } @@ -1452,7 +1456,11 @@ impl QemuInstance { ¶ms, options.debug_wait, ); - let storage = options.profile.shape().storage; + let storage = match (options.profile.shape().storage, options.data_bytes) { + (storage, None) => storage, + (Storage::Disk { .. }, Some(data_bytes)) => Storage::Disk { data_bytes }, + (Storage::Stick { .. }, Some(_)) => panic!("[qemu] a DATA size is a profile's whose disk carries its image"), + }; match storage { Storage::Stick { .. } => fs::write(&boot_image, image).expect("Failed to write test boot image"), Storage::Disk { data_bytes } => toyos_build::image::install(&image, &boot_image, data_bytes), diff --git a/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs b/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs index 10a9aebb3ed..cb79eb15d23 100644 --- a/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs +++ b/tests/guest/rust-tests/src/bin/hosted_clang_hello.rs @@ -1,41 +1,46 @@ -//! clang built for ToyOS compiles a C program here, and its driver links one, -//! starting `ld.lld` as a child; the program runs, and what each wrote is the -//! host's clang's bytes. +//! clang built for ToyOS, installed by `pkg`, compiles a C program here, and +//! its driver links one, starting `ld.lld` as a child; the program runs, and +//! what each wrote is the host's clang's bytes. //! -//! Its one argument is the directory the harness staged on ROOT: `bin/clang` -//! and `bin/ld.lld` with clang's resource headers at `lib/clang`, the C -//! sysroot at `sysroot`, the program at `src/hello.c`, and what the host's -//! clang made of it at `host/hello.o` and `host/a.out`. +//! Its arguments are the directory the harness staged on ROOT and the package +//! file in it: the package beside its `SHA256SUMS`, the program at +//! `src/hello.c`, and what the host's clang made of it at `host/hello.o` and +//! `host/a.out`. use std::path::{Path, PathBuf}; use std::process::Command; -/// Where this job writes: `/tmp` is the guest's one writable tree. +/// Where this job writes: `/tmp` is the guest's one writable tree outside the +/// package. const OUT: &str = "/tmp/hosted_clang_hello"; +/// The installed toolchain's `bin`. +const BIN: &str = "/apps/clang/bin"; + fn main() { - let staged = PathBuf::from(std::env::args().nth(1).expect("the staged directory, as the one argument")); - let sysroot = staged.join("sysroot"); + let mut args = std::env::args().skip(1); + let staged = PathBuf::from(args.next().expect("the staged directory, as the first argument")); + let package = staged.join(args.next().expect("the package's file name, as the second argument")); + let mut install = Command::new("/system/bin/pkg"); + install.arg("install").arg(&package).arg("--yes"); + run(install, "pkg install"); + let out = Path::new(OUT); std::fs::create_dir_all(out).unwrap_or_else(|e| panic!("create {OUT}: {e}")); let (object, linked) = (out.join("hello.o"), out.join("a.out")); - // The arguments are the host's, so the two products can be compared. - let clang = |args: &[&str], to: &Path| { - let mut command = Command::new(staged.join("bin/clang")); - command - .arg("--target=x86_64-unknown-toyos") - .arg(format!("--sysroot={}", sysroot.display())) - .args(args) - .arg("hello.c") - .arg("-o") - .arg(to) - .current_dir(staged.join("src")); + // The arguments are the host's but its target and sysroot, which are the + // installed clang's own: its default target, and the sysroot its + // configuration file names. + let compile = |driver: &str, args: &[&str], to: &Path| { + let mut command = Command::new(Path::new(BIN).join(driver)); + command.args(args).arg("hello.c").arg("-o").arg(to).current_dir(staged.join("src")); command }; - run(clang(&["-c"], &object), "clang -c"); - // `clang hello.c`: the driver compiles, and links by starting `ld.lld`. - run(clang(&[], &linked), "clang"); + run(compile("clang", &["-c"], &object), "clang -c"); + // `cc hello.c`: the driver by the link the package carries for it, + // compiling, and linking by starting `ld.lld`. + run(compile("cc", &[], &linked), "cc"); let ran = Command::new(&linked).output().unwrap_or_else(|e| panic!("run {}: {e}", linked.display())); assert!(ran.status.success(), "{} ended {:?}: {}", linked.display(), ran.status, String::from_utf8_lossy(&ran.stderr)); @@ -44,7 +49,7 @@ fn main() { } let mut differ = false; - for (made, by) in [(&object, "clang -c"), (&linked, "clang")] { + for (made, by) in [(&object, "clang -c"), (&linked, "cc")] { let name = made.file_name().expect("a file name"); differ |= !same(made, &staged.join("host").join(name), by); } @@ -52,16 +57,13 @@ fn main() { println!("hosted_clang_hello: ok"); } -/// Run `command`, and refuse its failure with what it said. +/// Run `command`, say what it printed, and refuse its failure. fn run(mut command: Command, what: &str) { let done = command.output().unwrap_or_else(|e| panic!("{what} did not start: {e}")); - assert!( - done.status.success(), - "{what} ended {:?}:\n{}{}", - done.status, - String::from_utf8_lossy(&done.stdout), - String::from_utf8_lossy(&done.stderr) - ); + for line in String::from_utf8_lossy(&done.stdout).lines() { + println!("hosted_clang_hello: {what}: {line}"); + } + assert!(done.status.success(), "{what} ended {:?}:\n{}", done.status, String::from_utf8_lossy(&done.stderr)); println!("hosted_clang_hello: {what} ended 0"); } diff --git a/tests/images/hostedclangcase.toml b/tests/images/hostedclangcase.toml index 95de68ebdc8..9447402b5d2 100644 --- a/tests/images/hostedclangcase.toml +++ b/tests/images/hostedclangcase.toml @@ -1,11 +1,30 @@ -# The boot `hosted_clang_hello` stages ToyOS's own clang and LLD on, with the C -# sysroot they read: a ROOT of its own, since they are a quarter of a gigabyte -# and no other test reads them. +# The boot `hosted_clang_hello` installs ToyOS's own clang and LLD in, by `pkg` +# from the package staged on its ROOT: a ROOT of its own, since the package is +# a hundred megabytes and no other test reads it, and DATA, where `/apps` is. [boot] -start = ["logkeeper", "test-runner"] +start = ["logkeeper", "diskserver", "fileserver", "test-runner"] [programs.logkeeper] service = true syscap = ["logread"] [programs.test-runner] + +# The installer the job runs, as `images/system.toml` declares it. +[programs.pkg] + +# The block service, the NVMe driver for the disk DATA is on, and the file +# servers. +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +[programs.nvme] +drives = ["pci:01:08:02/1b36:0010"] +receives = ["disk"] + +[programs.fileserver] +restart = true +roles = ["data", "log", "boot"] +receives = ["block"] diff --git a/tests/toyos.rs b/tests/toyos.rs index 2998c713f94..156991dfe91 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -212,8 +212,8 @@ const RUST_SKIP: &[&str] = &[ // must not take from its members: `bar_map_again` and the // `msix_table_read_in_place` metal row run it. "msix_in_place", - // It needs ToyOS's own clang and LLD on ROOT, which only its boot stages: - // `hosted_clang_hello` runs it on tests/images/hostedclangcase.toml. + // It needs ToyOS's own clang and LLD packaged on ROOT, which only its boot + // stages: `hosted_clang_hello` runs it on tests/images/hostedclangcase.toml. "hosted_clang_hello", // A std child's stdio, beside libc's: `libc_children` runs it on // tests/images/libccase.toml. @@ -529,9 +529,11 @@ const MACHINE_TESTS: &[&str] = &[ // block service's sessions through have no host build, and the T14 boots // from a stick beside an NVMe disk that is another system's. "nvme_disk_boots", - // clang and LLD built for ToyOS, compiling and linking inside it: the - // claim is that they run on ToyOS, which no host test can make, and the - // T14 would read nothing a guest does not. On demand: [`ON_DEMAND`]. + // clang and LLD built for ToyOS, installed by `pkg` and compiling and + // linking inside it: the claim is that they install and run on ToyOS, + // which no host test can make, and the T14 would read nothing a guest does + // not. On demand: [`ON_DEMAND`]. Red at the link until item 5 of + // `issues/toyos-builds-itself.md` lands. "hosted_clang_hello", // libc's children: each program starts and waits for its own, which is // order, completion, contents and counts, and no hardware is in it.