diff --git a/Cargo.lock b/Cargo.lock index ef2c75d4c16..379a11d64b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1379,6 +1379,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "inputsink" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", + "toyos-input", +] + [[package]] name = "inspect" version = "0.1.0" @@ -3158,6 +3167,14 @@ dependencies = [ "toyos-tmpdir", ] +[[package]] +name = "starveling" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 9627dc1ca01..b2705f11f7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,12 +143,14 @@ members = [ "system/vdagent", "system/vdagent/bridge", "tests/guest/hangupholder", + "tests/guest/inputsink", "tests/guest/kernelprobe", "tests/guest/metalprobe", "tests/guest/osbench", "tests/guest/osbench/lan", "tests/guest/osbench/rules", "tests/guest/pciholder", + "tests/guest/starveling", "tests/guest/test-runner", "tests/metal", ] diff --git a/build/build.rs b/build/build.rs index 60d4775a1a2..2a8d7a7ed1f 100644 --- a/build/build.rs +++ b/build/build.rs @@ -119,6 +119,17 @@ struct SystemConfig { /// them, because a package directory is writable. #[serde(default)] apps: AppsConfig, + /// The supervisor's own row, which no `[programs]` key is. + #[serde(default)] + supervisor: SupervisorConfig, +} + +/// **Unknown fields refused**, as a row's are. +#[derive(Deserialize, Default)] +#[serde(default, rename_all = "kebab-case", deny_unknown_fields)] +struct SupervisorConfig { + /// Its job's floor, in MiB (`toyos_manifest::Manifest::supervisor_floor`). + memory_floor_mib: u64, } #[derive(Deserialize, Default)] @@ -199,6 +210,10 @@ struct ProgramConfig { /// A login authority, for which the supervisor opens login sessions /// (`toyos_manifest::Program::login`). login: bool, + /// Its job's floor, in MiB: memory held back for it however short every + /// other job runs (`toyos_manifest::Program::memory_floor`). A control + /// path's alone, and reviewed as the privilege it is. + memory_floor_mib: u64, } /// The directories a program's crate sits directly under, by who uses it: @@ -836,6 +851,7 @@ fn render_manifest(config: &SystemConfig) -> Vec { starts: cfg.starts.clone(), login: cfg.login, folder: None, + memory_floor: mib(cfg.memory_floor_mib), } }) .collect(), @@ -843,11 +859,17 @@ fn render_manifest(config: &SystemConfig) -> Vec { apps: config.apps.receives.clone(), folder: apps_folder(config).expect("the gate above read it"), start: config.boot.start.clone(), + supervisor_floor: mib(config.supervisor.memory_floor_mib), }; toyos_manifest::render(&manifest) .unwrap_or_else(|e| panic!("the image config cannot be rendered as a manifest: {e:?}")) } +/// `mib` MiB in bytes; a floor past every machine is a broken config. +fn mib(mib: u64) -> u64 { + mib.checked_mul(1 << 20).unwrap_or_else(|| panic!("a memory floor of {mib} MiB is past every machine")) +} + /// Which build `root`'s tree makes for `arch` against the sysroot whose key /// for it is `toolchain`, as `/system/etc/os-release` records it, read with /// `git`. Untracked files are dirty whatever `status.showUntrackedFiles` @@ -3183,6 +3205,7 @@ mod tests { "tests/images/jobcase.toml", "tests/images/latencycase.toml", "tests/images/launchcase.toml", + "tests/images/floorcase.toml", "tests/images/libccase.toml", "tests/images/sessioncase.toml", "tests/images/metalcase.toml", diff --git a/images/console.toml b/images/console.toml index fdb26833b7c..7cd18c75631 100644 --- a/images/console.toml +++ b/images/console.toml @@ -16,6 +16,10 @@ assets = ["assets"] +# The controls' floors, as `images/system.toml` gives them. +[supervisor] +memory-floor-mib = 96 + [boot] start = ["logkeeper", "diskserver", "fileserver", "console", "acpiserver", "ps2server"] @@ -64,6 +68,7 @@ starts = ["toybox", "shell"] restart = true receives = ["input"] devices = ["isa:0060,0064:1,12"] +memory-floor-mib = 24 # `power` because `bin/shutdown` below is this binary, and on the machine this diff --git a/images/system.toml b/images/system.toml index f2f8018bda5..b9c2aeca941 100644 --- a/images/system.toml +++ b/images/system.toml @@ -11,6 +11,16 @@ # day delete it. assets = ["assets"] +# **Floors are the machine's protected capacity**: memory held back for a +# control path's job however short every other job runs. Only a job that does +# no ordinary client work gets one: the supervisor and ps2server. A floor +# covers what the job holds as it runs, its 8 MiB main stack's commitment +# among it, and the headroom it needs to keep answering. usbd has none, since +# it drives disks too, so a USB keyboard is not protected +# (`issues/no-physical-memory-fairness.md`). +[supervisor] +memory-floor-mib = 96 + # What the supervisor starts at boot. Program keys, not paths, and the order means # nothing: every port exists before any server runs. [boot] @@ -103,6 +113,7 @@ login = true restart = true receives = ["input"] devices = ["isa:0060,0064:1,12"] +memory-floor-mib = 24 [programs.soundserver] service = true diff --git a/issues/a-command-typed-through-qmp-arrives-truncated-under-host-load.md b/issues/a-command-typed-through-qmp-arrives-truncated-under-host-load.md new file mode 100644 index 00000000000..8564d2ca117 --- /dev/null +++ b/issues/a-command-typed-through-qmp-arrives-truncated-under-host-load.md @@ -0,0 +1,34 @@ +--- +status: open +kind: tooling +opened: 2026-10-24 +--- + +# A command typed through QMP arrives truncated under host load + +`session_lifecycle` types `detach /system/bin/hangupholder` into the +desktop's terminal, one QMP key press and release per character +(`tests/common/sessionlifecycle.rs`). Under heavy parallel host load the +guest can receive the command with keys missing, and the row then stalls +waiting for the holder's spawn. + +Sightings: +- #1057, at `4bfe7e489`, in the 22-row QEMU filter with every image building + cold (2471 s of build work across the workers). The shell received + `detach /pholder`, and the supervisor's launcher said "no row declares + /pholder". The row passed alone three times, and the same filter passed on + its next run. `sessioncase.toml` declares no memory floors, and memory was + not short. Logs: https://github.com/ToyOSOrg/evidence/tree/81614cc6bcad9dcd7bbcd47d8f09462d1da7c957/r8 + (`rows.log.gz`). +- #1060, at `04ec3f6f4`, with the host at 5.82×: `session_lifecycle` stalled + once and was not reproduced in nine later runs. The cause was not found, and + dropped keystrokes are a suspect. + +Where the keys drop is not established. The candidates are QEMU's QMP key +injection and the i8042 model's queue, which the harness fills one keystroke +at a time with no wait for the guest to drain, or the guest's path from +ps2server through the compositor to the terminal. + +Owner: the test harness. Exit: find where the keys drop, QMP or the guest's +input path, and fix it there. A guest that loses input is an input defect, +and pacing the harness to hide it is not an exit. diff --git a/issues/a-service-refused-its-restart-for-memory-stays-down.md b/issues/a-service-refused-its-restart-for-memory-stays-down.md deleted file mode 100644 index 01f9ce01fd5..00000000000 --- a/issues/a-service-refused-its-restart-for-memory-stays-down.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-23 ---- - -# A service refused its restart for want of memory stays down - -When a service ends, the supervisor starts it again once -(`system/supervisor/src/main.rs`, `AfterEnd::Restart`). If that spawn is -refused, it closes the service's ports and does not try again. Since memory is -refused at the call rather than panicking the kernel, out of memory is such a -refusal, and it is temporary. - -**Evidence.** In #1045 round 1's `dirs-uncharged` control, `kernel_objects_refused` -held every frame a process may in a shared boot, and the file server's data -instance died of an allocation it was refused. Its restart was refused too, and -it never came back after the test gave its memory back: - -`supervisor: fileserver data ended and would not start again (out of memory); its ports are closed` - -Every later member that needed `/data` would have failed for a cause outside it. - -**Exit:** a restart refused `ResourceExhausted` is retried once memory comes back, -on an event rather than a timer, such as a memory-returned notice the kernel -posts. The retry is reported, and its stall is visible. The guest test is -`kernel_objects_refused`'s own boot: a service it starves while it holds every -frame is serving again once it gives the memory back. - -Part of `issues/no-physical-memory-fairness.md`. diff --git a/issues/four-shares-take-a-file-server.md b/issues/four-shares-take-a-file-server.md index 088a8d0b895..768955ce31d 100644 --- a/issues/four-shares-take-a-file-server.md +++ b/issues/four-shares-take-a-file-server.md @@ -32,11 +32,11 @@ so a `make -j16` whose jobs each stream to a log of their own holds all 16, and separate stdout and stderr files per job need 32. Inferred, not measured: the share's 32 served connections, spent by every process of the session at one per directory, are probably the first bound such a build -reaches. Raising `STREAM_SHARE` doubles the uncharged rings a share can pin, -2 MiB each (`issues/nothing-charges-kernel-memory-to-a-process.md`), so it -waits for one of two exits: a measured parallel build on ToyOS shows which -bound it reaches and by how much, or kernel memory is charged and a share's -streams are bounded by bytes rather than by count. +reaches. Raising `STREAM_SHARE` doubles the rings a share can pin, 2 MiB +each, counted to the job whose write first needs each (`kernel/src/pipe.rs`), +so it waits for one of two exits: a measured parallel build on ToyOS shows +which bound it reaches and by how much, or a share's streams are bounded by +the bytes they cost its client's job rather than by count. Owner: the file server. diff --git a/issues/no-physical-memory-fairness.md b/issues/no-physical-memory-fairness.md index db1b88e4645..2ce97d215ce 100644 --- a/issues/no-physical-memory-fairness.md +++ b/issues/no-physical-memory-fairness.md @@ -6,53 +6,102 @@ opened: 2026-07-30 # No physical memory fairness -Any process can allocate physical memory until the system runs out. No -per-process limits, no memory pressure signals, no OOM killer. A single -misbehaving process starves everything. - -Running out is a refusal at the call, not a halted machine: a commitment is -promised its frames when it is made, and every kernel object a process causes -is charged before it is allocated (`issues/nothing-charges-kernel-memory-to-a-process.md`), -a thread's kernel stack among them. But the refusal falls on whichever process -asks next, its maker or any other. - -**System controls must stay usable under exhaustion, and today they do not** -(owner, item 71: "Preserving usable system controls under exhaustion remains -required; allocation errors alone do not establish that protection"). While -`kernel_objects_refused` holds every frame a process may, every other process -is refused too, system services included. In #1045 round 1's `dirs-uncharged` -control, the file server's data instance died of a refused allocation, and the -supervisor's one refused restart was final -(`issues/a-service-refused-its-restart-for-memory-stays-down.md`). - -**Exit:** per-job accounting, landing after the machine-level charges (item 71), -with a floor reserved for the control-path system jobs, the supervisor and the -input servers (the memory design's protected capacity), which no other job's -allocation can take. The supervisor retries a refused restart once memory -returns. Attribution does not impose fixed per-app limits (item 71). The guest -test: with one ordinary job holding every frame it may, the supervisor still -answers a stop and a restart, the input path still delivers, and a starved -service is serving again once the memory is given back. +Any ordinary job can take memory until the machine runs out, and every other +ordinary job is refused at its next call meanwhile. Nothing signals pressure, +and nothing a person can reach stops the job that took it. + +**What holds now.** Everything a process causes is counted to its job's +account (`kernel::frames::Ledger`, read through `account_info`), and the +control paths' jobs keep a floor no other job's allocation can take +(`memory-floor-mib` in `images/system.toml`: the supervisor and ps2server). +A floor is a protected total allowance: the first `floor` of the job's use +is guaranteed, cgroup v2 `memory.min`'s contract, not an emergency +allowance on top of it (owner, 2026-10-24). Attribution imposes no per-job +limit. A service whose +restart is refused for memory keeps its ports and starts again once memory +comes back, which its job's readiness says. `memory_floor_holds_the_controls` +(QEMU) and the `memory_floor` metal row hold, with one ordinary job holding +every frame: ps2server's floor carrying a key to the owner of input, a +system holder killing the greedy job, and a starved service serving again. +**No test establishes the supervisor's floor**: with only it removed the +test passes. Without it, a service that dies under exhaustion, such as +fileserver refused its writeback, can be refused its restart for the +supervisor's own memory, which closes it (below). + +**What is still missing, and is this entry's exit:** + +- **The stop UI.** Under exhaustion nothing a person holds can find and stop + the greedy job, and no exhaustion recovery is claimed for visible controls + or for input: the guest test stands a system holder in for them. The + compositor has no floor (owner, 2026-10-24). Owner: the compositor and + memory tracks. Next: a bounded emergency path in the existing compositor, + in which ordinary rendering cannot spend protected capacity, and a review + of the whole path from input to the visible controls to the stop, USB + input included, before any recovery is claimed. +- **What the supervisor holds grows with the sessions and programs it + runs**, and on its floor. A login authority connects its own control + connection and an owner its own link, and a launched program's image is + not the supervisor's (its server's object, or a copy its own job holds), + but each live program still holds a few KiB of the supervisor's + (its handle, notice and the connectors minted for it), bounded only by the + process table. Owner: the supervisor track. Exit: what a launch makes for + the child counted to the child's job, or a bound on live launches the + floor is sized for. +- **A restart refused for the supervisor's own memory closes the service + for good.** The supervisor waits for memory only where the service's own + job recorded the refusal (`start_again`); where the supervisor's job was + refused, such as mapping a boot image for the child, the service is closed + instead. Owner: the supervisor track. Exit: a start refused for + the supervisor's own memory waits on the supervisor's own job's readiness, + as one refused for the service's waits on the service's. +- **Connect-time capacity.** Both rings of a connection are its connecting + job's, allocated at each one's first write, so a server's reply spends + nothing of its own; but a client's sends are not yet capacity its receiver + chose and was charged for at accept. +- **Floor deficits are the machine's, not each floor job's.** A floor's frames + and whole superframes are held again only as they come back free and whole + (`kernel/pure/frames.rs`), and what is short is pooled: one floor job's + pinned or scattered give-back can refuse another floor job inside its own + floor. Owner: the memory track. Exit: a law where one floor job's deficit + never refuses another's use inside its floor. +- **A USB keyboard is not protected.** usbd drives disks as well as keyboards, + so it is a mixed service and has no floor; the i8042's ps2server has one. + Owner: the USB track. Exit: USB input protected as the review of the input + path above decides; a driver split only to assign a floor is not + presumed. +- **A `drives` process refused its start for memory is not retried on the + memory event**, only at the next inventory read; a service is. Owner: the + supervisor track. Exit: a `drives` start refused for memory waits on its + job's readiness, as a service's restart does. +- **Kill draws on the kernel's kept superframes.** A kill and the teardown it + starts allocate transient heap, which the kept superframes cover rather + than an allocation-free path. Owner: the memory track. Exit: kill and + teardown allocate nothing. +- **`account_info` reports no resident pages, retained objects or live + processes**, which the memory design lists. Owner: the memory track. Exit: + the three in `AccountInfo`, read without the process lock. + +**Exit:** a person can see which job holds the memory and stop it while every +frame is held, through a control drawn and served from capacity committed +before the exhaustion; and a connection's receive capacity is charged to its +receiver at accept. The guest test is `memory_floor`'s, its system holder +replaced by the control. **The class statement, which is what predicts the next instance.** *A client's request is an allocation request, and every one of them needs an owner who can -say no.* Three instances were filed under it — the compositor's windows, netd's -piped connections, and `SYS_CONNECT` pinning 4 MiB into an unbounded pending -queue — and all three now have a bound *and* a caller that hears the refusal, -which is the pair the class asks for: `toyos_desktop::max_windows` and netstack's -`places_for` each divide an eighth of physical memory by what one unit -costs and refuse past it, and a pipe allocates its ring page on first use rather -than at `create`. **What a bound alone still does not answer is whose window to -refuse.** The memory is charged to nobody, so a cap is the only thing between one -client and the machine — which is this entry, and is why both of those functions' -doc comments name a kernel memory limit as what deletes them. The kernel's -directory numbering and `mkdir`'s directories had such caps too, and charging -each to the machine deleted them. +say no.* Three instances were filed under it — the compositor's windows, +netstack's piped connections, and `SYS_CONNECT` pinning 4 MiB into an +unbounded pending queue — and all three have a bound *and* a caller that hears +the refusal: `toyos_desktop::max_windows` and netstack's `places_for` each +divide an eighth of physical memory by what one unit costs and refuse past it, +and a pipe allocates its ring page on first use rather than at `create`. Their +memory is counted to whichever job made it: a client's own connection is +its job's, but a pipe a server makes and hands out (fileserver's stream +pipes, netstack's bells) is the server's, which the server bounds. With no +per-job limit a cap is still the only thing between one client and the +machine. **Promoted to `defect` 2026-08-25** (finding-lifecycle ruling: isolation -findings are security-adjacent and are promoted, never folded). One process -starving every other of physical memory is a denial of service across the -isolation boundary that needs no privilege and nothing crafted — it is real -today, not a possibility. Owed by whoever builds per-process physical -accounting; nothing in the tree attributes memory to a process or refuses one -for its share now. +findings are security-adjacent and are promoted, never folded). One job +starving every other of memory is a denial of service across the isolation +boundary that needs no privilege and nothing crafted. diff --git a/issues/nothing-charges-kernel-memory-to-a-process.md b/issues/nothing-charges-kernel-memory-to-a-process.md deleted file mode 100644 index a9f396c0561..00000000000 --- a/issues/nothing-charges-kernel-memory-to-a-process.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -status: open -kind: track -opened: 2026-08-04 ---- - -# Nothing charges a kernel allocation to the process that caused it - -**Kernel memory a process causes is charged, but to the machine, not to -it.** Every persistent kernel object a process can reach holds a -`mm::charge::Charge` for the heap it keeps, taken before the allocation and -refused `ResourceExhausted` at the call where covering it would take what the -frame allocator holds back for the kernel; the heap grows into what charges -reserved (`kernel/pure/frames.rs`). So the terminal state below is gone for -those objects, but for the heap's fragmentation past the charges' estimates -(`issues/the-kernel-heaps-fragmentation-is-charged-to-nobody.md`). What is still missing is the attribution: no charge names the -process or job that caused it, so there is no per-process memory in `ps`, no -`meminfo`, no reap-time assertion that a dead process's charges went to zero, -and no way for the suite to assert the absence of a leak. The memory design -puts this on the scheduler's job as its `Account`, with `account_info`. - -**Exit:** each charge held against its job's account, released with the -object, and `account_info` reporting it, with a reap-time check that an ended -job's charges are what its retained objects hold. - -**The security case that once justified this is mostly gone, and that is the -news.** It was staged as an ownership discipline — a purpose-carrying allocator -type, aliases per subsystem, a charge on the four physical-memory chokepoints — -against a table of bugs that a different mechanism has since closed. Handle and -object lifetimes are refcount-driven with real `Drop` impls, so "a handle that -is not a value with a destructor" is unrepresentable; the descriptor type and -its un-refcounted clone are gone; the file cache installs a real budget and a -budget that was never installed is now a loud kernel bug; and the unbounded -user-string copy has a ceiling. What survives is the *accounting*, which nothing -has touched, plus one item: peak memory is written by two paths that overwrite -each other. - -Accounting is a pillar of `issues/toyos-explains-itself.md`. - -Blocked on nothing. Two things worth knowing before it is restarted, because -both cost a day to discover: - -- **Omitting `#[global_allocator]` enforces nothing** — rustc errors whenever - `alloc` is in the graph at all. The enforcement is a type alias with no - default parameter, about 50 lines, not the Rust-for-Linux approach of dropping - `alloc` entirely. -- **`#[must_use]` is inadequate for an obligation like this**: binding, `let _ - =`, `drop()` and burial in a collection all pass silently. A drop bomb is the - state of the art, and `Unmapped` is already exactly such an obligation. - -**The terminal state of an uncharged grower is a halted machine.** A failed -kernel allocation panics with the size, the layer and the call site named, and -every kernel panic halts. What a call allocates and frees before it returns is -not charged: it draws on the superframes the kernel keeps, 1/256 of memory and -16 MiB at least, which nothing a process causes takes. diff --git a/issues/only-one-allocation-can-be-made-to-fail-and-only-at-one-site.md b/issues/only-one-allocation-can-be-made-to-fail-and-only-at-one-site.md index d5891eff7de..c76132423d8 100644 --- a/issues/only-one-allocation-can-be-made-to-fail-and-only-at-one-site.md +++ b/issues/only-one-allocation-can-be-made-to-fail-and-only-at-one-site.md @@ -44,7 +44,7 @@ not perturb allocator order — proved by showing that with the countdown disabled the allocator's behaviour is byte-for-byte what it was. The result channel is preallocated at arm time, before the failure it exists to report. -`issues/nothing-charges-kernel-memory-to-a-process.md` holds the terminal -state every unbounded grower closes into — who dies when the heap gives out. +`issues/no-physical-memory-fairness.md` holds the terminal state every +unbounded grower closes into: which job is refused when memory runs out. Whichever of the two lands first, the other's closure runs through this actuator; they are not two answers to one question. diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index b713908b20d..6587e29ea02 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -84,9 +84,8 @@ live split of a user leaf. frames when it is made, and every other allocation is refused rather than take a promised frame (`kernel::frames`), so a commitment past what memory holds is refused at the call, not at a fault; its page tables are taken - then too. Kernel objects a process makes are still drawn from the frames - no commitment may promise, uncharged - (`issues/nothing-charges-kernel-memory-to-a-process.md`). + then too. Kernel objects a process makes are charged to its job + (`kernel/src/mm/charge.rs`). 5. **Shared text.** A spawn copies each page of its program's segments into the process at the page's first touch, out of the image object or out of a snapshot of its `/tmp` file; no fault reads a file. A ROOT program's diff --git a/issues/toyos-builds-itself.md b/issues/toyos-builds-itself.md index 999a747c446..1abc3a07f19 100644 --- a/issues/toyos-builds-itself.md +++ b/issues/toyos-builds-itself.md @@ -82,7 +82,7 @@ An owner is an issue under `issues/`, or this track. | # | Item | Status | Owner | Blocked on | |---|---|---|---|---| -| 13 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-in-4-kib-pages-still-copies-each-programs-text`, `nothing-charges-kernel-memory-to-a-process` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | +| 13 | Process memory: 4 KiB pages, a ledger, demand paging, one mapping inventory, then private file mappings and coherent shared ones (owner direction, 2026-10-21; the design is out of the tree, under review) | direction set | `process-memory-in-4-kib-pages-still-copies-each-programs-text`, `no-physical-memory-fairness` | engineering: the design, with a wait and progress design for mapped pages without an automatic hang restart (owner, 2026-10-21) | | 14 | Record locks the file server serves, and std's and cargo's locks on them | direction set | `std-file-lock-answers-ok-and-locks-nothing`, `libc-refuses-what-toyos-cannot-yet-answer` | engineering: the lock design, part of 13's | | 15 | An unlinked open file lives until its last close or unmap | direction set | `an-unlinked-open-file-answers-stale` | 13 for the mapping | | 16 | An exclusive create on `/tmp` (`O_CREAT\|O_EXCL`, `create_new`): the kernel's `OpenFlags::EXCLUSIVE` | done | this track | — | diff --git a/issues/toyos-explains-itself.md b/issues/toyos-explains-itself.md index ab2bb2b0ab5..7ad4de1b21e 100644 --- a/issues/toyos-explains-itself.md +++ b/issues/toyos-explains-itself.md @@ -58,7 +58,7 @@ rest were not put to him. | Diary | `issues/the-diary-computes-no-lateness-and-records-no-slow-system-call.md` | | Counters, sizes | this file | | Profile | behind the diary's judgement; `issues/a-frozen-toyos-waits-for-a-hand-on-the-power-button.md` | -| Accounting | `issues/nothing-charges-kernel-memory-to-a-process.md`, `issues/a-processs-memory-is-a-byte-total-that-reads-zero-under-contention.md`, `issues/blocked-time-is-invisible-while-the-park-lasts.md` | +| Accounting | `issues/a-processs-memory-is-a-byte-total-that-reads-zero-under-contention.md`, `issues/blocked-time-is-invisible-while-the-park-lasts.md` | | Inspect | `issues/the-kernel-keeps-nothing-it-enumerates.md` | | Post-mortem | step 1 of `issues/logging-records-from-every-producer-and-a-kernel-that-waits-on-nobody.md` | | Symbols | `issues/the-kernel-still-parses-what-userland-writes.md` Move 3 | diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index 5d89f9b14b4..b2e8411c536 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -419,6 +419,11 @@ pub mod seqlock; #[path = "../../src/drivers/serial_lock.rs"] pub mod serial_lock; +/// What the job a scheduler task holds counts its memory in. +#[cfg(feature = "loom")] +#[path = "../../pure/frames.rs"] +pub mod frames; + /// The scheduler core, `kernel/pure/sched/`, driven by `tests/loom_*.rs` over /// [`sched_model`]'s scaffolding. /// diff --git a/kernel/pure/frames.rs b/kernel/pure/frames.rs index 10d086050d2..2c76ec0656e 100644 --- a/kernel/pure/frames.rs +++ b/kernel/pure/frames.rs @@ -33,11 +33,23 @@ //! spare superframes, then kept ones //! (`issues/the-kernel-heaps-fragmentation-is-charged-to-nobody.md`). //! +//! **Everything a process causes is counted to its account's [`Ledger`], +//! and an account's floor is held back against every other account.** What +//! a floor account does not use of its floor is held as frames, and as the +//! whole superframes it makes, so that it can still be promised, tabled, +//! lent a run and grow its own heap pool however short every other account +//! runs. What it gives back is held for its floor again as it comes back: +//! a frame a copy still pins comes back with its last unpin, and a whole +//! superframe only once one is whole. An ordinary account has no floor and +//! no limit. A refused account's [`ready`](Frames::ready) answers whether +//! what it was refused would be granted now, and +//! [`take_returned`](Frames::take_returned) once it may be. +//! //! The storage is the caller's, sized by [`supers_for`] and [`pins_for`] for //! the span of physical memory it manages, because the kernel lays it out of //! that memory before it has a heap. -use core::sync::atomic::{AtomicU32, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; /// One frame. pub const FRAME: u64 = 4096; @@ -116,11 +128,215 @@ pub enum Discarded { Kept, } +/// One account's memory: what it holds, and the floor held back for it. +/// Written only under the allocator's lock, by the [`Frames`] methods that +/// take it, and read by anyone. +#[derive(Default)] +pub struct Ledger { + /// Frames held back for it, however short every other account runs: + /// zero for an ordinary account. + floor: AtomicU64, + /// Pages committed to it, promised or drawn; frames of its page tables; + /// superframes of its runs. + committed: AtomicU64, + tables: AtomicU64, + runs: AtomicU64, + /// Heap bytes charged to it, those of them charged to its own pool, and + /// the superframes that pool is entitled to. + heap: AtomicU64, + own: AtomicU64, + entitled: AtomicU64, + /// Its footprint once its last refused request would have been granted, + /// and how many of its requests were refused. + wanted: AtomicU64, + refusals: AtomicU64, + /// That request: its frames, the whole superframes among them, and how + /// many of those it needed in a row; and the round of waits it is armed + /// in ([`Frames::ready`]). + asked: AtomicU64, + asked_supers: AtomicU64, + asked_row: AtomicU64, + armed: AtomicU64, + peak: AtomicU64, +} + +/// A [`Ledger`] read at one moment: frames, superframes and bytes as there. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Usage { + pub floor: u64, + pub committed: u64, + pub tables: u64, + pub runs: u64, + pub heap: u64, + pub entitled: u64, + pub wanted: u64, + pub refusals: u64, + pub peak: u64, +} + +/// What one charge holds of each heap pool. Each growth takes from the pool +/// its job charges now, so a charge made before its job had a floor grows +/// from the floor once it has one; what it gives back leaves the job's own +/// pool first. +#[derive(Default, Debug)] +pub struct PoolSplit { + shared: u64, + own: u64, +} + +impl PoolSplit { + pub const NONE: Self = Self { shared: 0, own: 0 }; + + pub fn bytes(&self) -> u64 { + self.shared + self.own + } +} + +/// Which heap ledger a charge was taken from, and is given back to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Pool { + /// The one every account without a floor shares. + Shared, + /// The account's own: a floor account's, entitled to its superframes. + Own, +} + +impl Ledger { + pub const fn new() -> Self { + Self { + floor: AtomicU64::new(0), + committed: AtomicU64::new(0), + tables: AtomicU64::new(0), + runs: AtomicU64::new(0), + heap: AtomicU64::new(0), + own: AtomicU64::new(0), + entitled: AtomicU64::new(0), + wanted: AtomicU64::new(0), + refusals: AtomicU64::new(0), + asked: AtomicU64::new(0), + asked_supers: AtomicU64::new(0), + asked_row: AtomicU64::new(0), + armed: AtomicU64::new(0), + peak: AtomicU64::new(0), + } + } + + pub fn usage(&self) -> Usage { + let read = |word: &AtomicU64| word.load(Ordering::Relaxed); + Usage { + floor: read(&self.floor), + committed: read(&self.committed), + tables: read(&self.tables), + runs: read(&self.runs), + heap: read(&self.heap), + entitled: read(&self.entitled), + wanted: read(&self.wanted), + refusals: read(&self.refusals), + peak: read(&self.peak), + } + } + + /// Whether it holds nothing at all. + pub fn is_empty(&self) -> bool { + let u = self.usage(); + (u.committed, u.tables, u.runs, u.heap) == (0, 0, 0, 0) + } + + /// The heap pool a charge it takes now is taken from: its own once it + /// has a floor. + fn pool(&self) -> Pool { + if self.floor() > 0 { Pool::Own } else { Pool::Shared } + } + + fn floor(&self) -> u64 { + self.floor.load(Ordering::Relaxed) + } + + /// Frames counted against its floor: its own pool's whole entitlement, + /// since that is what is held for its heap. + fn floor_used(&self) -> u64 { + let u = self.usage(); + u.committed + u.tables + PER_SUPER as u64 * (u.runs + u.entitled) + } + + /// Frames of its floor it does not use. + fn unused(&self) -> u64 { + self.floor().saturating_sub(self.floor_used()) + } + + /// The frames it holds, its heap rounded up. + fn footprint(&self) -> u64 { + let u = self.usage(); + u.committed + u.tables + PER_SUPER as u64 * u.runs + u.heap.div_ceil(FRAME) + } + + fn add(word: &AtomicU64, n: u64) { + word.fetch_add(n, Ordering::Relaxed); + } + + fn sub(word: &AtomicU64, n: u64, what: &str) { + let was = word.fetch_sub(n, Ordering::Relaxed); + assert!(was >= n, "frames: {n} {what} given back of {was}"); + } + + /// Its last refused request: frames, whole superframes, those in a row. + fn asked(&self) -> (u64, u64, u64) { + let read = |word: &AtomicU64| word.load(Ordering::Relaxed); + (read(&self.asked), read(&self.asked_supers), read(&self.asked_row)) + } + + fn grew(&self) { + self.peak.fetch_max(self.footprint(), Ordering::Relaxed); + } +} + +/// What an allocation for a floor account leaves held back for floors. +#[derive(Clone, Copy)] +struct Floors { + unused: u64, + held: u64, + supers: u64, + whole: u64, +} + +/// The least any waiter for memory lacks, as [`Frames::met`] weighs it: +/// frames, whole superframes if it needs any, and a row if it needs one. +#[derive(Clone, Copy)] +struct Want { + frames: u64, + supers: Option, + row: u64, +} + +/// Whole superframes a waiter lacks, counted two ways against those held +/// for floors: beside every floor's, and beside as many as the floors' held +/// frames make. +#[derive(Clone, Copy)] +struct Supers { + beside_floors: u64, + beside_held: u64, +} + +impl Want { + fn least(self, other: Self) -> Self { + Self { + frames: self.frames.min(other.frames), + supers: self.supers.zip(other.supers).map(|(a, b)| Supers { + beside_floors: a.beside_floors.min(b.beside_floors), + beside_held: a.beside_held.min(b.beside_held), + }), + row: self.row.min(other.row), + } + } +} + /// The allocator. Every method but the constructors takes the caller's lock. /// -/// Invariants: `whole ≥ reserved + kept`, and `promised + PER_SUPER × -/// (reserved + kept) ≤ free`, so what is promised fits outside the -/// superframes held back and a draw never has to split one of them. +/// Invariants: `whole ≥ reserved + kept + floors.whole`, `promised + +/// PER_SUPER × (reserved + kept) + floors.held ≤ free` and `PER_SUPER × +/// floors.whole ≤ floors.held ≤ floors.unused`, so what is promised fits +/// outside what is held back and a draw never has to split a superframe held +/// whole. pub struct Frames<'m> { base: u64, supers: &'m mut [Super], @@ -137,12 +353,24 @@ pub struct Frames<'m> { /// it is to keep. kept: u64, keep_target: u64, - /// Bytes charged, the superframes covering them, those the heap holds, - /// and how many of these it grew into from what charges reserved. + /// Bytes charged to the shared pool and the superframes covering them; + /// the superframes every pool is entitled to; those the heap holds, and + /// how many of these it grew into from what charges reserved. charged: u64, + shared: u64, entitled: u64, heap: u64, drawn: u64, + /// What is held back for floors: every floor account's unused frames, + /// those of them held now (fewer while what came back is still pinned), + /// and the whole superframes they make, per account, and those held now + /// (fewer while no whole one has come back for them). + floors: Floors, + /// What the waiters of this round of waits for memory lack, the round, + /// and whether enough came back since one asked. + want: Option, + round: u64, + returned: bool, } impl<'m> Frames<'m> { @@ -165,9 +393,14 @@ impl<'m> Frames<'m> { kept: 0, keep_target: 0, charged: 0, + shared: 0, entitled: 0, heap: 0, drawn: 0, + floors: Floors { unused: 0, held: 0, supers: 0, whole: 0 }, + want: None, + round: 1, + returned: false, } } @@ -265,21 +498,36 @@ impl<'m> Frames<'m> { (self.reserved, self.kept) } - /// Bytes charged, and the superframes the heap holds. + /// Bytes charged to the shared pool, and the superframes the heap holds. pub fn charges(&self) -> (u64, u64) { (self.charged, self.heap) } - /// Free frames nothing holds back: neither promised nor in a superframe - /// reserved or kept. + /// Frames held back for floors, and the whole superframes among them. + pub fn floors_held(&self) -> (u64, u64) { + (self.floors.held, self.floors.whole) + } + + /// Free frames nothing holds back: neither promised, held for a floor, + /// nor in a superframe reserved or kept. fn room(&self) -> u64 { - let held = self.promised + PER_SUPER as u64 * (self.reserved + self.kept); + self.room_beside(self.floors.held) + } + + /// [`room`](Self::room) with `held` frames held for floors. + fn room_beside(&self, held: u64) -> u64 { + let held = self.promised + PER_SUPER as u64 * (self.reserved + self.kept) + held; self.free.checked_sub(held).unwrap_or_else(|| panic!("frames: {held} held back of {} free", self.free)) } /// Whole superframes nothing holds back. fn spare_supers(&self) -> u64 { - let held = self.reserved + self.kept; + self.spare_beside(self.floors.whole) + } + + /// [`spare_supers`](Self::spare_supers) with `whole` held for floors. + fn spare_beside(&self, whole: u64) -> u64 { + let held = self.reserved + self.kept + whole; u64::from(self.whole.len).checked_sub(held).unwrap_or_else(|| panic!("frames: {held} superframes held back of {} whole", self.whole.len)) } @@ -297,19 +545,189 @@ impl<'m> Frames<'m> { self.refill(); } - /// Keep superframes towards the target while some are spare. + /// Hold back what came back: superframes kept towards the target first, + /// then the frames of floors, then the whole superframes they make; and + /// say so, once, if a waiter's want is met. fn refill(&mut self) { while self.kept < self.keep_target && self.spare(1) { self.kept += 1; } + let back = (self.floors.unused - self.floors.held).min(self.room()); + self.floors.held += back; + let most = self.floors.supers.min(self.floors.held / PER_SUPER as u64); + while self.floors.whole < most && self.spare_supers() > 0 { + self.floors.whole += 1; + } + if self.want.is_some_and(|want| self.met(want)) { + self.want = None; + self.round += 1; + self.returned = true; + } } - /// Promise `frames` if nothing held back is needed for them. - pub fn promise(&mut self, frames: u64) -> bool { - if frames > self.room() { + /// What is held back for floors once `l`, `more` frames past what it now + /// counts against its floor, has taken them. + fn floors_after(&self, l: &Ledger, more: u64) -> Floors { + let (before, after) = (l.unused(), l.floor().saturating_sub(l.floor_used().saturating_add(more))); + let per = PER_SUPER as u64; + let unused = self.floors.unused - before + after; + let supers = self.floors.supers - before / per + after / per; + let held = self.floors.held.min(unused); + Floors { unused, held, supers, whole: self.floors.whole.min(supers).min(held / per) } + } + + /// Whether `l` may take `frames` frames and `supers` whole superframes, + /// `used` of them counted against its floor: what is held back for every + /// other floor stays, and what its own floor holds may go. The caller + /// takes them and holds back the floors this answers. + fn admit(&self, l: &Ledger, used: u64, frames: u64, supers: u64) -> Option { + let floors = self.floors_after(l, used); + let fits = self.room_beside(floors.held) >= frames + supers * PER_SUPER as u64 && self.spare_beside(floors.whole) >= supers; + fits.then_some(floors) + } + + /// `l` was refused `frames` more, `supers` of them whole superframes and + /// `row` of those in a row: what a wait for memory to come back waits for, + /// and what `l` lacks now if it waits. + fn refused(&mut self, l: &Ledger, frames: u64, supers: u64, row: u64) { + l.wanted.store(l.footprint().saturating_add(frames), Ordering::Relaxed); + l.asked.store(frames, Ordering::Relaxed); + l.asked_supers.store(supers, Ordering::Relaxed); + l.asked_row.store(row, Ordering::Relaxed); + l.refusals.fetch_add(1, Ordering::Relaxed); + self.rearm(l); + } + + /// `change` gives back some of what `l` holds: its floor holds what it no + /// longer uses, as frames come back. + fn give_back_for(&mut self, l: &Ledger, change: impl FnOnce(&Ledger)) { + let per = PER_SUPER as u64; + let before = l.unused(); + change(l); + let after = l.unused(); + self.floors.unused = self.floors.unused - before + after; + self.floors.supers = self.floors.supers - before / per + after / per; + assert!(after >= before, "frames: a give-back took {} of a floor", before - after); + self.rearm(l); + self.refill(); + } + + /// Hold `frames` back for `l` against every other account: refused, + /// changing nothing, where what is free beside everything held back + /// cannot hold the more it adds now. A floor set lower lets go of + /// what it no longer holds. + pub fn set_floor(&mut self, l: &Ledger, frames: u64) -> bool { + let per = PER_SUPER as u64; + let (before, after) = (l.unused(), frames.saturating_sub(l.floor_used())); + let (more, supers) = (after.saturating_sub(before), (after / per).saturating_sub(before / per)); + if after > before && (self.room() < more || self.spare_supers() < supers) { return false; } + l.floor.store(frames, Ordering::Relaxed); + self.floors.unused = self.floors.unused - before + after; + self.floors.supers = self.floors.supers - before / per + after / per; + self.floors.held = (self.floors.held + more).min(self.floors.unused); + self.floors.whole = (self.floors.whole + supers).min(self.floors.supers).min(self.floors.held / per); + // What its own pool kept entitled for the floor it had goes as its + // charges no longer need it. + let entitled = l.entitled.load(Ordering::Relaxed); + let keep = l.own.load(Ordering::Relaxed).div_ceil(SUPER).max(entitled.min(frames.div_ceil(per))); + if entitled > keep { + self.unentitle(entitled - keep); + l.entitled.store(keep, Ordering::Relaxed); + } + self.rearm(l); + self.refill(); + true + } + + /// Whether what `l` was last refused would be granted now; if not, the + /// next [`take_returned`](Self::take_returned) answers once it may be. + pub fn ready(&mut self, l: &Ledger) -> bool { + let (frames, supers, row) = l.asked(); + if self.admit(l, frames, frames - supers * PER_SUPER as u64, supers).is_some() && self.in_a_row(row) { + return true; + } + self.arm(l); + false + } + + /// Take the least of what `l` lacks and what this round's waiters lack. + /// What it lacks is its request less what its own floor gives it: + /// [`admit`](Self::admit) lets the request take that much of what is held + /// for floors, as frames and as the whole superframes they make. + fn arm(&mut self, l: &Ledger) { + let per = PER_SUPER as u64; + let (frames, supers, row) = l.asked(); + let unused = l.unused(); + let rest = unused.saturating_sub(frames); + let (own, own_supers) = (unused - rest, unused / per - rest / per); + let lacks = Want { + frames: frames - own, + supers: (supers > 0).then(|| Supers { + beside_floors: supers - own_supers.min(supers), + beside_held: supers - own.div_ceil(per).min(supers), + }), + row: if row > 1 { row } else { 0 }, + }; + self.want = Some(self.want.map_or(lacks, |want| want.least(lacks))); + l.armed.store(self.round, Ordering::Relaxed); + } + + /// Arm `l` again if it waits this round: what it gave back of its floor + /// may be what it lacked. + fn rearm(&mut self, l: &Ledger) { + if l.armed.load(Ordering::Relaxed) == self.round { + self.arm(l); + } + } + + /// Whether some waiter may be admitted what it lacked, `want` being the + /// least they lack. Asked at the end of [`refill`](Self::refill), where + /// floors are short of frames only once no room is left, and of whole + /// superframes only once none is spare: then each test is necessary for + /// some waiter's grant, and for one waiter without a floor sufficient. + fn met(&self, want: Want) -> bool { + let short = self.floors.unused - self.floors.held; + let whole = self.spare_supers() + self.floors.whole; + let made = self.floors.held / PER_SUPER as u64; + self.room() >= want.frames.saturating_add(short) + && want.supers.is_none_or(|lacks| { + whole >= self.floors.supers + lacks.beside_floors || whole >= made + lacks.beside_held + }) + && self.in_a_row(want.row) + } + + /// Whether `count` whole superframes lie in a row, as a run of more than + /// one takes them. + fn in_a_row(&self, count: u64) -> bool { + if count <= 1 { + return true; + } + let mut run = 0; + self.supers.iter().any(|s| { + run = if s.state == State::Whole { run + 1 } else { 0 }; + run >= count + }) + } + + /// Whether memory came back that a waiter wanted, since this last + /// answered `true`. + pub fn take_returned(&mut self) -> bool { + core::mem::take(&mut self.returned) + } + + /// Promise `frames` to `l`'s commitment if nothing held back for the + /// kernel or another account is needed for them. + pub fn promise(&mut self, l: &Ledger, frames: u64) -> bool { + let Some(floors) = self.admit(l, frames, frames, 0) else { + self.refused(l, frames, 0, 0); + return false; + }; + self.floors = floors; self.promised += frames; + Ledger::add(&l.committed, frames); + l.grew(); true } @@ -326,8 +744,9 @@ impl<'m> Frames<'m> { if pins.pinned(phys) { return Discarded::Kept; } - self.free_frame(phys, pins); + self.give_back(phys, pins); self.repromise(1); + self.refill(); Discarded::Freed } @@ -338,11 +757,23 @@ impl<'m> Frames<'m> { let _ = self.room(); } - /// Take back promises of `frames` no commitment holds any more. - pub fn unpromise(&mut self, frames: u64) { + /// Take back promises of `frames` none of `l`'s commitments holds any + /// more, which were never drawn. + pub fn unpromise(&mut self, l: &Ledger, frames: u64) { assert!(frames <= self.promised, "frames: {frames} unpromised of {} promised", self.promised); self.promised -= frames; - self.refill(); + self.give_back_for(l, |l| Ledger::sub(&l.committed, frames, "committed pages")); + } + + /// Give back `frames`, drawn for pages `l` no longer commits; a pinned + /// one is taken back by its last unpin instead. + pub fn free_drawn(&mut self, l: &Ledger, frames: impl IntoIterator, pins: &Pins) { + let mut count = 0; + for phys in frames { + self.give_back(phys, pins); + count += 1; + } + self.give_back_for(l, |l| Ledger::sub(&l.committed, count, "committed pages")); } /// One frame a promise held, which a promise guarantees is free. @@ -358,17 +789,33 @@ impl<'m> Frames<'m> { pub fn alloc_frame(&mut self) -> Option { if self.room() == 0 { self.kept = self.kept.checked_sub(1)?; + // The rest of the superframe kept no longer is room a waiter may lack. + let frame = self.take_frame(); + self.refill(); + return frame; } self.take_frame() } - /// One frame a process causes without a promise, such as a page table: - /// refused rather than take one held back. - pub fn alloc_frame_for_process(&mut self) -> Option { - if self.room() == 0 { + /// One frame for a page table of `l`'s, which is not promised: refused + /// rather than take one held back. + pub fn alloc_table(&mut self, l: &Ledger) -> Option { + let Some(floors) = self.admit(l, 1, 1, 0) else { + self.refused(l, 1, 0, 0); return None; - } - self.take_frame() + }; + self.floors = floors; + let phys = self.take_frame().expect("frames: an admitted frame is free"); + Ledger::add(&l.tables, 1); + l.grew(); + Some(phys) + } + + /// Give back a table frame [`alloc_table`](Self::alloc_table) took for + /// `l`. + pub fn free_table(&mut self, l: &Ledger, phys: u64, pins: &Pins) { + self.give_back(phys, pins); + self.give_back_for(l, |l| Ledger::sub(&l.tables, 1, "table frames")); } // A whole superframe is split only when no split one has a frame free, so @@ -462,14 +909,32 @@ impl<'m> Frames<'m> { Some(run) } - /// [`alloc_run`](Self::alloc_run) of a run a process causes, such as a - /// pipe's ring or memory a device is lent: refused rather than take a - /// superframe held back. - pub fn alloc_run_for_process(&mut self, count: usize) -> Option { - if !self.spare(count as u64) { + /// `count` superframes in a row for `l`, such as a pipe's ring or memory + /// a device is lent: refused rather than take a superframe held back. + pub fn alloc_run_for(&mut self, l: &Ledger, count: usize) -> Option { + let supers = count as u64; + let frames = supers * PER_SUPER as u64; + let Some(floors) = self.admit(l, frames, 0, supers) else { + self.refused(l, frames, supers, supers); + return None; + }; + let Some(run) = self.take_run(count) else { + self.refused(l, frames, supers, supers); return None; + }; + self.floors = floors; + Ledger::add(&l.runs, supers); + l.grew(); + Some(run) + } + + /// Give back a run [`alloc_run_for`](Self::alloc_run_for) took for `l`; + /// any of its frames still pinned is taken back by its last unpin. + pub fn free_run_for(&mut self, l: &Ledger, phys: u64, count: usize, pins: &Pins) { + for s in 0..count as u64 { + self.give_back_super(phys + s * SUPER, pins); } - self.take_run(count) + self.give_back_for(l, |l| Ledger::sub(&l.runs, count as u64, "run superframes")); } fn take_run(&mut self, count: usize) -> Option { @@ -528,40 +993,114 @@ impl<'m> Frames<'m> { self.refill(); } - /// Charge `bytes` of the kernel heap to a process's cause, refused where + /// Charge `bytes` of the kernel heap to `l`, from `pool`, refused where /// covering them would take a superframe held back. What charges cover is /// whole superframes: those the heap grew into from what charges /// reserved, and reserved ones it grows into first. None the heap took /// otherwise covers a charge, since what it holds there is not charged. - pub fn charge(&mut self, bytes: u64) -> bool { - let Some(charged) = self.charged.checked_add(bytes) else { return false }; - let need = charged.div_ceil(SUPER); - if need > self.entitled { - let more = need - self.entitled; - if !self.spare(more) { + pub fn charge(&mut self, l: &Ledger, pool: Pool, bytes: u64) -> bool { + let (charged, entitled) = match pool { + Pool::Shared => (self.charged, self.shared), + Pool::Own => (l.own.load(Ordering::Relaxed), l.entitled.load(Ordering::Relaxed)), + }; + let Some(charged) = charged.checked_add(bytes) else { + self.refused(l, u64::MAX, 0, 0); + return false; + }; + let more = charged.div_ceil(SUPER).saturating_sub(entitled); + if more > 0 { + let used = match pool { + Pool::Shared => 0, + Pool::Own => more * PER_SUPER as u64, + }; + let Some(floors) = self.admit(l, used, 0, more) else { + self.refused(l, more * PER_SUPER as u64, more, 0); return false; - } - self.entitled = need; + }; + self.floors = floors; + self.entitled += more; self.reserved += more; + match pool { + Pool::Shared => self.shared += more, + Pool::Own => Ledger::add(&l.entitled, more), + } } - self.charged = charged; + match pool { + Pool::Shared => self.charged = charged, + Pool::Own => l.own.store(charged, Ordering::Relaxed), + } + Ledger::add(&l.heap, bytes); + l.grew(); true } - /// Give back a charge of `bytes`. Superframes stay reserved for one more - /// than the charges need, so a charge and its uncharge at a boundary do - /// not take and give back one each time. - pub fn uncharge(&mut self, bytes: u64) { - self.charged = self.charged.checked_sub(bytes).unwrap_or_else(|| panic!("frames: {bytes} uncharged of {}", self.charged)); - let need = self.charged.div_ceil(SUPER) + 1; - if self.entitled > need { - self.entitled = need; - self.drawn = self.drawn.min(need); - self.reserved = need - self.drawn; - self.refill(); + /// Give back `bytes` of a charge to `l` from `pool`. The shared pool + /// stays entitled to one superframe more than its charges need, so a + /// charge and its uncharge at a boundary do not take and give back one + /// each time. A floor account's own pool stays entitled to as much of + /// its floor as it ever grew into: given back, a superframe the heap + /// still held would be counted free for the floor and be in use. + pub fn uncharge(&mut self, l: &Ledger, pool: Pool, bytes: u64) { + Ledger::sub(&l.heap, bytes, "heap bytes"); + match pool { + Pool::Shared => { + self.charged = self.charged.checked_sub(bytes).unwrap_or_else(|| panic!("frames: {bytes} uncharged of {}", self.charged)); + let need = self.charged.div_ceil(SUPER) + 1; + if self.shared > need { + self.unentitle(self.shared - need); + self.shared = need; + self.refill(); + } + } + Pool::Own => { + Ledger::sub(&l.own, bytes, "heap bytes of its own pool"); + let entitled = l.entitled.load(Ordering::Relaxed); + let floor = entitled.min(l.floor().div_ceil(PER_SUPER as u64)); + let keep = l.own.load(Ordering::Relaxed).div_ceil(SUPER).max(floor); + if entitled > keep { + self.unentitle(entitled - keep); + self.give_back_for(l, |l| Ledger::sub(&l.entitled, entitled - keep, "entitled superframes")); + } + } } } + /// `bytes` more of `held`, a charge to `l`, from the pool `l` charges now; + /// refused whole. + pub fn grow(&mut self, l: &Ledger, held: &mut PoolSplit, bytes: u64) -> bool { + let pool = l.pool(); + if !self.charge(l, pool, bytes) { + return false; + } + match pool { + Pool::Shared => held.shared += bytes, + Pool::Own => held.own += bytes, + } + true + } + + /// `bytes` fewer of `held`, from the job's own pool first. + pub fn shrink(&mut self, l: &Ledger, held: &mut PoolSplit, bytes: u64) { + assert!(bytes <= held.bytes(), "frames: {bytes} given back of a charge of {}", held.bytes()); + let own = bytes.min(held.own); + if own > 0 { + self.uncharge(l, Pool::Own, own); + held.own -= own; + } + if bytes > own { + self.uncharge(l, Pool::Shared, bytes - own); + held.shared -= bytes - own; + } + } + + /// `supers` fewer superframes entitled to any pool: reserved ones go + /// first, and those the heap grew into stay its own. + fn unentitle(&mut self, supers: u64) { + self.entitled -= supers; + self.drawn = self.drawn.min(self.entitled); + self.reserved = self.entitled - self.drawn; + } + /// A superframe for the kernel heap to grow by: a reserved one, or else /// one for the kernel's own use. pub fn alloc_heap(&mut self) -> Option { @@ -739,6 +1278,7 @@ mod tests { struct Machine { frames: Frames<'static>, pins: Pins<'static>, + l: Ledger, } /// `supers` superframes from `BASE`, `usable` of memory described, then sealed. @@ -753,7 +1293,7 @@ mod tests { frames.withhold(start, end); } frames.seal(); - Machine { frames, pins: Pins::new(BASE, words) } + Machine { frames, pins: Pins::new(BASE, words), l: Ledger::new() } } fn whole(supers: usize) -> Machine { @@ -793,13 +1333,13 @@ mod tests { let mut m = whole(3); m.frames.keep(1); let all = 2 * PER_SUPER as u64; - assert!(!m.frames.promise(all + 1), "a promise into the kept superframe"); - assert!(m.frames.promise(all - 10)); + assert!(!m.frames.promise(&m.l, all + 1), "a promise into the kept superframe"); + assert!(m.frames.promise(&m.l, all - 10)); assert_eq!(m.frames.promised(), all - 10); - assert_eq!(m.frames.alloc_run_for_process(1), None, "a superframe out of promised frames"); - let free: Vec = core::iter::from_fn(|| m.frames.alloc_frame_for_process()).collect(); + assert_eq!(m.frames.alloc_run_for(&m.l, 1), None, "a superframe out of promised frames"); + let free: Vec = core::iter::from_fn(|| m.frames.alloc_table(&m.l)).collect(); assert_eq!(free.len(), 10, "only the unpromised frames go to anyone else"); - assert!(!m.frames.promise(1), "nothing is left to promise"); + assert!(!m.frames.promise(&m.l, 1), "nothing is left to promise"); let drawn: Vec = (0..all - 10).map(|_| m.frames.alloc_promised()).collect(); assert_eq!((m.frames.promised(), m.frames.free_frames()), (0, PER_SUPER as u64)); @@ -808,12 +1348,12 @@ mod tests { // again; one given back by none is anyone's. assert_eq!(m.frames.discard(drawn[0], &m.pins), Discarded::Freed); assert_eq!(m.frames.promised(), 1); - assert_eq!(m.frames.alloc_frame_for_process(), None, "a frame promised again went to another"); - m.frames.free_frame(drawn[1], &m.pins); - assert!(m.frames.alloc_frame_for_process().is_some(), "a frame no promise holds was refused"); - assert_eq!(m.frames.alloc_frame_for_process(), None, "the promised one went too"); - m.frames.unpromise(1); - assert!(m.frames.alloc_frame_for_process().is_some(), "an unpromised frame stayed held"); + assert_eq!(m.frames.alloc_table(&m.l), None, "a frame promised again went to another"); + m.frames.free_drawn(&m.l, [drawn[1]], &m.pins); + assert!(m.frames.alloc_table(&m.l).is_some(), "a frame no promise holds was refused"); + assert_eq!(m.frames.alloc_table(&m.l), None, "the promised one went too"); + m.frames.unpromise(&m.l, 1); + assert!(m.frames.alloc_table(&m.l).is_some(), "an unpromised frame stayed held"); } /// A kept superframe is the kernel's alone: refused to whatever a process @@ -824,14 +1364,14 @@ mod tests { let mut m = whole(2); m.frames.keep(1); let all = PER_SUPER as u64; - let tables: Vec = core::iter::from_fn(|| m.frames.alloc_frame_for_process()).collect(); + let tables: Vec = core::iter::from_fn(|| m.frames.alloc_table(&m.l)).collect(); assert_eq!(tables.len() as u64, all, "a table took a frame of the kept superframe"); - assert_eq!(m.frames.alloc_run_for_process(1), None, "a process's run took the kept superframe"); + assert_eq!(m.frames.alloc_run_for(&m.l, 1), None, "a process's run took the kept superframe"); assert_eq!(m.frames.held_back(), (0, 1)); let kernel = m.frames.alloc_frame().expect("the kernel was refused a kept frame"); assert_eq!(m.frames.held_back(), (0, 0), "a kept superframe split for the kernel stayed kept"); for &f in &tables { - m.frames.free_frame(f, &m.pins); + m.frames.free_table(&m.l, f, &m.pins); } assert_eq!(m.frames.held_back(), (0, 1), "a superframe back whole was not kept again"); m.frames.free_frame(kernel, &m.pins); @@ -846,12 +1386,12 @@ mod tests { fn a_charge_reserves_what_the_heap_grows_into() { let mut m = whole(4); m.frames.keep(1); - assert!(m.frames.charge(SUPER + 1)); + assert!(m.frames.charge(&m.l, Pool::Shared, SUPER + 1)); assert_eq!(m.frames.held_back(), (2, 1)); - assert!(!m.frames.charge(2 * SUPER), "a charge into the kept superframe"); + assert!(!m.frames.charge(&m.l, Pool::Shared, 2 * SUPER), "a charge into the kept superframe"); assert_eq!((m.frames.charges(), m.frames.held_back()), ((SUPER + 1, 0), (2, 1)), "a refused charge left something behind"); - assert!(!m.frames.promise(PER_SUPER as u64 + 1), "a promise into the reserved superframes"); - assert!(m.frames.promise(PER_SUPER as u64)); + assert!(!m.frames.promise(&m.l, PER_SUPER as u64 + 1), "a promise into the reserved superframes"); + assert!(m.frames.promise(&m.l, PER_SUPER as u64)); let first = m.frames.alloc_heap().unwrap(); let second = m.frames.alloc_heap().unwrap(); @@ -865,7 +1405,7 @@ mod tests { assert_eq!(m.frames.held_back(), (0, 1), "the superframe past the charges was not kept again"); m.frames.free_heap(second, &m.pins); assert_eq!(m.frames.held_back(), (1, 1), "a superframe the charges cover was not reserved again"); - m.frames.uncharge(SUPER + 1); + m.frames.uncharge(&m.l, Pool::Shared, SUPER + 1); assert_eq!(m.frames.held_back(), (0, 1), "a superframe past one of slack stayed reserved"); } @@ -875,7 +1415,7 @@ mod tests { fn the_heaps_own_superframes_cover_no_charge() { let mut m = whole(4); let own = [m.frames.alloc_heap().unwrap(), m.frames.alloc_heap().unwrap()]; - assert!(m.frames.charge(1)); + assert!(m.frames.charge(&m.l, Pool::Shared, 1)); assert_eq!(m.frames.held_back(), (1, 0), "a charge leaned on the heap's own superframes"); m.frames.free_heap(own[0], &m.pins); assert_eq!(m.frames.held_back(), (1, 0)); @@ -893,16 +1433,16 @@ mod tests { #[test] fn an_uncharge_keeps_one_superframe_of_slack() { let mut m = whole(4); - assert!(m.frames.charge(1)); - m.frames.uncharge(1); + assert!(m.frames.charge(&m.l, Pool::Shared, 1)); + m.frames.uncharge(&m.l, Pool::Shared, 1); assert_eq!(m.frames.held_back(), (1, 0)); - assert!(m.frames.charge(SUPER)); + assert!(m.frames.charge(&m.l, Pool::Shared, SUPER)); assert_eq!(m.frames.held_back(), (1, 0)); - assert!(m.frames.charge(1)); + assert!(m.frames.charge(&m.l, Pool::Shared, 1)); assert_eq!(m.frames.held_back(), (2, 0)); - m.frames.uncharge(SUPER); + m.frames.uncharge(&m.l, Pool::Shared, SUPER); assert_eq!(m.frames.held_back(), (2, 0)); - m.frames.uncharge(1); + m.frames.uncharge(&m.l, Pool::Shared, 1); assert_eq!((m.frames.charges(), m.frames.held_back()), ((0, 0), (1, 0))); } @@ -914,7 +1454,7 @@ mod tests { fn a_discard_keeps_a_pinned_frame_and_promises_only_what_came_back() { let mut m = whole(1); let all = PER_SUPER as u64; - assert!(m.frames.promise(all)); + assert!(m.frames.promise(&m.l, all)); let drawn: Vec = (0..all).map(|_| m.frames.alloc_promised()).collect(); assert!(m.pins.pin(drawn[0])); assert_eq!(m.frames.discard(drawn[0], &m.pins), Discarded::Kept, "a pinned frame was freed"); @@ -1071,6 +1611,459 @@ mod tests { let _ = m.pins.unpin(BASE); } + const PER: u64 = PER_SUPER as u64; + + /// An account with `frames` of floor, set on `m`. + fn floored(m: &mut Machine, frames: u64) -> Ledger { + let l = Ledger::new(); + assert!(m.frames.set_floor(&l, frames), "a floor of {frames} frames on an idle machine"); + l + } + + /// Promise `l` every frame it may still be promised. + fn exhaust(frames: &mut Frames, l: &Ledger) -> u64 { + let room = frames.room(); + assert!(frames.promise(l, room)); + assert!(!frames.promise(l, 1), "a frame past the room was promised"); + room + } + + /// With every frame an ordinary account may take promised to it, the + /// floor account is promised, tabled and lent a run up to its floor, and + /// refused past it; the ordinary account is refused each meanwhile. + #[test] + fn a_floor_is_held_against_every_other_account() { + let mut m = whole(8); + m.frames.keep(1); + let floor = 2 * PER + 100; + let f = floored(&mut m, floor); + exhaust(&mut m.frames, &m.l); + assert_eq!(m.frames.alloc_table(&m.l), None, "an ordinary table took a floor's frame"); + assert_eq!(m.frames.alloc_run_for(&m.l, 1), None, "an ordinary run took a floor's superframe"); + assert!(!m.frames.charge(&m.l, Pool::Shared, SUPER), "an ordinary charge took a floor's superframe"); + + let run = m.frames.alloc_run_for(&f, 1).expect("the floor account was refused a run inside its floor"); + assert!(m.frames.alloc_table(&f).is_some(), "the floor account was refused a table inside its floor"); + assert!(m.frames.promise(&f, floor - PER - 1), "the floor account was refused the rest of its floor"); + assert!(!m.frames.promise(&f, 1), "the floor account took a frame past its floor"); + assert_eq!(m.frames.floors_held(), (0, 0)); + m.frames.free_run_for(&f, run, 1, &m.pins); + assert!(!m.frames.promise(&m.l, 1), "a floor's run, given back, went to an ordinary account"); + assert_eq!(m.frames.floors_held(), (PER, 1), "a run given back was not held whole for its floor"); + } + + /// A floor's superframes stay whole: a run another account takes comes + /// out of whole superframes nothing holds, even where frames scattered + /// through split ones would cover it. + #[test] + fn a_floors_superframes_stay_whole_beside_scattered_room() { + let mut m = whole(4); + let f = floored(&mut m, PER); + // One split superframe: half its frames taken, half free and scattered. + let tables: Vec = (0..2 * PER).map(|_| m.frames.alloc_table(&m.l).unwrap()).collect(); + for &t in tables.iter().step_by(2) { + m.frames.free_table(&m.l, t, &m.pins); + } + assert!(m.frames.room() >= PER, "the scattered frames cover a run"); + assert_eq!(m.frames.alloc_run_for(&m.l, 1), Some(BASE + 2 * SUPER), "the one spare whole superframe"); + assert_eq!(m.frames.alloc_run_for(&m.l, 1), None, "a run took the floor's whole superframe"); + assert!(m.frames.alloc_run_for(&f, 1).is_some(), "the floor account found no whole superframe"); + } + + /// A floor account's own heap pool is entitled from its floor, so a + /// charge needing a new superframe is covered however short the rest of + /// the machine runs; given back, it stays entitled as far as the floor. + #[test] + fn a_floor_accounts_heap_draws_on_its_floor_and_keeps_it() { + let mut m = whole(8); + let f = floored(&mut m, 2 * PER); + exhaust(&mut m.frames, &m.l); + assert_eq!(f.pool(), Pool::Own); + assert!(m.frames.charge(&f, Pool::Own, 1)); + assert_eq!(m.frames.held_back().0, 1, "its own pool reserved no superframe"); + assert!(m.frames.charge(&f, Pool::Own, SUPER), "a second superframe inside its floor was refused"); + assert!(!m.frames.charge(&f, Pool::Own, SUPER), "a third superframe past its floor was granted"); + m.frames.uncharge(&f, Pool::Own, SUPER + 1); + assert_eq!(f.usage().entitled, 2, "its own pool gave back what its floor holds for it"); + assert!(!m.frames.promise(&m.l, 1), "a floor account's uncharge freed a frame for another account"); + assert!(m.frames.charge(&f, Pool::Own, SUPER), "its pool, still entitled, refused a charge"); + } + + /// Past its floor, a floor account competes for the rest as any other + /// does, and what it gives back is held for its floor first. + #[test] + fn past_its_floor_an_account_is_ordinary_and_gives_back_to_its_floor_first() { + let mut m = whole(4); + let f = floored(&mut m, PER); + let room = m.frames.room(); + assert!(m.frames.promise(&f, PER + room), "its floor and the rest"); + assert!(!m.frames.promise(&m.l, 1)); + m.frames.unpromise(&f, 10); + assert!(m.frames.promise(&m.l, 10), "what it gave back past its floor stayed held"); + m.frames.unpromise(&f, room - 10 + 5); + assert!(!m.frames.promise(&m.l, room - 5 + 1), "the 5 frames under its floor went to another account"); + assert!(m.frames.promise(&m.l, room - 10)); + } + + /// A floor is set only where it can be held whole now, and set to + /// nothing lets all of it go. + #[test] + fn a_floor_is_set_only_where_it_is_free_and_unset_lets_it_go() { + let mut m = whole(4); + let room = exhaust(&mut m.frames, &m.l); + let f = Ledger::new(); + assert!(!m.frames.set_floor(&f, 1), "a floor set out of frames promised to another account"); + m.frames.unpromise(&m.l, room); + assert!(m.frames.set_floor(&f, 3 * PER)); + assert_eq!(m.frames.floors_held(), (3 * PER, 3)); + assert!(!m.frames.promise(&m.l, m.frames.free_frames() - 3 * PER + 1)); + assert!(m.frames.set_floor(&f, 0)); + assert_eq!(m.frames.floors_held(), (0, 0)); + assert!(m.frames.promise(&m.l, m.frames.room())); + assert!(m.frames.room() == 0 && m.frames.free_frames() == 4 * PER); + } + + /// A charge made before its job had a floor grows from the floor once it + /// has one, however short the rest of the machine runs, and gives back + /// the floor's part first. + #[test] + fn a_charge_made_before_its_floor_grows_from_it() { + let mut m = whole(8); + let f = Ledger::new(); + let mut held = PoolSplit::NONE; + assert!(m.frames.grow(&f, &mut held, 100)); + assert!(m.frames.set_floor(&f, 2 * PER)); + exhaust(&mut m.frames, &m.l); + assert!(m.frames.grow(&f, &mut held, SUPER), "a charge made before the floor was refused the floor"); + assert_eq!((m.frames.charges().0, f.usage().entitled), (100, 1), "its growth went to the shared pool"); + m.frames.shrink(&f, &mut held, SUPER + 50); + assert_eq!((held.bytes(), m.frames.charges().0, f.usage().heap), (50, 50, 50), "the floor's part was not given back first"); + m.frames.shrink(&f, &mut held, 50); + assert!(f.usage().heap == 0 && held.bytes() == 0); + } + + /// A floor let go of takes with it what its own heap pool kept entitled + /// for it, once the pool's charges no longer need it: a job that ends + /// holds no superframe reserved for nothing. + #[test] + fn a_floor_let_go_releases_what_its_pool_kept() { + let mut m = whole(8); + let f = floored(&mut m, 2 * PER); + assert!(m.frames.charge(&f, Pool::Own, SUPER + 1)); + m.frames.uncharge(&f, Pool::Own, SUPER + 1); + assert_eq!(f.usage().entitled, 2, "its pool kept what its floor holds"); + assert!(m.frames.set_floor(&f, 0)); + assert_eq!((f.usage().entitled, m.frames.held_back().0), (0, 0), "a reserved superframe outlived the floor"); + assert!(f.is_empty()); + } + + /// A refused account's wait for memory is answered once what it lacked + /// is free, and not before; the answer is given once. + #[test] + fn a_wait_for_memory_is_answered_once_what_it_lacked_comes_back() { + let mut m = whole(4); + let greedy = Ledger::new(); + exhaust(&mut m.frames, &greedy); + let starved = Ledger::new(); + assert!(!m.frames.promise(&starved, 300)); + assert_eq!(starved.usage().refusals, 1, "a refusal went uncounted"); + assert!(!m.frames.ready(&starved)); + m.frames.unpromise(&greedy, 299); + assert!(!m.frames.take_returned(), "answered with 299 of 300 frames back"); + m.frames.unpromise(&greedy, 1); + assert!(m.frames.take_returned(), "300 frames back went unanswered"); + assert!(!m.frames.take_returned(), "answered twice"); + assert!(m.frames.ready(&starved)); + assert!(m.frames.promise(&starved, 300)); + } + + /// A waiter refused a smaller request while it waits waits for that one. + #[test] + fn a_smaller_refusal_while_waiting_is_what_is_waited_for() { + let mut m = whole(4); + exhaust(&mut m.frames, &m.l); + let starved = Ledger::new(); + assert!(!m.frames.promise(&starved, 300)); + assert!(!m.frames.ready(&starved)); + assert!(!m.frames.promise(&starved, 10)); + m.frames.unpromise(&m.l, 10); + assert!(m.frames.take_returned(), "memory for the smaller request went unanswered"); + assert!(m.frames.ready(&starved)); + } + + /// The kernel taking a frame of a kept superframe frees the rest of it + /// to the room, which answers a waiter. + #[test] + fn a_kept_superframe_split_answers_a_wait() { + let mut m = whole(2); + m.frames.keep(1); + exhaust(&mut m.frames, &m.l); + let starved = Ledger::new(); + assert!(!m.frames.promise(&starved, 10)); + assert!(!m.frames.ready(&starved)); + assert!(m.frames.alloc_frame().is_some(), "the kernel's frame from its kept superframe"); + assert!(m.frames.take_returned(), "the rest of the superframe went unanswered"); + assert!(m.frames.ready(&starved)); + } + + /// A job refused a whole superframe is not ready while only scattered + /// frames come back, however many: what it waits for is answered by a + /// whole superframe, and then once. + #[test] + fn a_wait_for_a_superframe_is_answered_by_one_and_not_by_frames() { + let mut m = whole(4); + let tables: Vec = core::iter::from_fn(|| m.frames.alloc_table(&m.l)).collect(); + assert_eq!(tables.len(), 4 * PER_SUPER); + for &t in tables.iter().step_by(2) { + m.frames.free_table(&m.l, t, &m.pins); + } + assert!(m.frames.room() >= PER && m.frames.spare_supers() == 0, "the frames free are scattered"); + let starved = Ledger::new(); + assert_eq!(m.frames.alloc_run_for(&starved, 1), None); + assert!(!m.frames.ready(&starved), "ready for a superframe with only scattered frames free"); + // All of the first superframe's frames but one come back: still no whole one. + let first: Vec = tables.iter().copied().skip(1).step_by(2).filter(|&t| t < BASE + SUPER).collect(); + for &t in &first[1..] { + m.frames.free_table(&m.l, t, &m.pins); + } + assert!(!m.frames.take_returned(), "answered with frames back and no whole superframe"); + assert!(!m.frames.ready(&starved)); + m.frames.free_table(&m.l, first[0], &m.pins); + assert!(m.frames.take_returned(), "a whole superframe back went unanswered"); + assert!(!m.frames.take_returned(), "answered twice"); + assert!(m.frames.ready(&starved)); + assert!(m.frames.alloc_run_for(&starved, 1).is_some()); + } + + /// A floor account refused a superframe while its floor's own still comes + /// back pinned is answered when the pin goes, though nothing else comes + /// back: its wait counts what its floor holds for it, as its request does. + #[test] + fn a_floor_accounts_wait_is_answered_by_its_own_floor() { + let mut m = whole(4); + let f = floored(&mut m, PER); + let table = m.frames.alloc_table(&m.l).unwrap(); + assert!(m.pins.pin(table)); + // Every other superframe and frame is taken, not promised: none is spare. + while m.frames.alloc_run_for(&m.l, 1).is_some() {} + while m.frames.alloc_table(&m.l).is_some() {} + let run = m.frames.alloc_run_for(&f, 1).expect("a run inside its floor"); + let pinned = run + 9 * FRAME; + assert!(m.pins.pin(pinned)); + m.frames.free_run_for(&f, run, 1, &m.pins); + assert_eq!(m.frames.alloc_run_for(&f, 1), None, "a run while its floor's superframe is pinned"); + assert!(!m.frames.ready(&f)); + // Something else is given back, and comes back pinned too: the floor + // is still short of the frame its superframe needs. + m.frames.free_table(&m.l, table, &m.pins); + assert!(!m.frames.take_returned(), "answered while its floor is short"); + assert_eq!(m.pins.unpin(pinned), Unpinned::Release); + m.frames.release(pinned); + assert!(m.frames.take_returned(), "its floor's superframe back went unanswered"); + assert!(!m.frames.take_returned(), "answered twice"); + assert!(m.frames.ready(&f)); + assert!(m.frames.alloc_run_for(&f, 1).is_some()); + } + + /// A wait for a run of superframes is answered by whole ones in a row, + /// not by as many apart. + #[test] + fn a_wait_for_a_run_is_answered_by_a_row() { + let mut m = whole(6); + let supers: Vec = (0..6).map(|_| m.frames.alloc_super().unwrap()).collect(); + m.frames.free_super(supers[0], &m.pins); + m.frames.free_super(supers[2], &m.pins); + let starved = Ledger::new(); + assert_eq!(m.frames.alloc_run_for(&starved, 2), None); + assert!(!m.frames.ready(&starved)); + m.frames.free_super(supers[4], &m.pins); + assert!(!m.frames.take_returned(), "answered with three whole superframes apart"); + m.frames.free_super(supers[3], &m.pins); + assert!(m.frames.take_returned(), "a row back went unanswered"); + assert!(m.frames.ready(&starved)); + assert!(m.frames.alloc_run_for(&starved, 2).is_some()); + } + + /// However floors, pins and fragmentation stand, a waiter for memory is + /// told once what it was refused would be granted: after every step, no + /// account still waits whose last refused request `admit` would grant. + #[test] + fn a_wait_for_memory_is_never_missed() { + let mut m = whole(12); + m.frames.keep(1); + let accounts = [Ledger::new(), floored(&mut m, 2 * PER + 50), floored(&mut m, PER), Ledger::new()]; + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut drawn: [Vec; 4] = Default::default(); + let mut tables: [Vec; 4] = Default::default(); + let mut runs: [Vec<(u64, usize)>; 4] = Default::default(); + let mut pinned: Vec = Vec::new(); + let mut waiting = [false; 4]; + let (mut posts, mut floor_waits) = (0, 0); + for step in 0..40_000 { + // The last account only ever asks, and so only ever waits. + let a = if next() % 8 == 0 { 3 } else { next() as usize % 3 }; + let l = &accounts[a]; + let refused = match next() % 8 { + 0 if a < 3 => { + let n = next() % 700; + let ok = m.frames.promise(l, n); + if ok { + drawn[a].extend((0..n).map(|_| m.frames.alloc_promised())); + } + !ok + } + 1 if !drawn[a].is_empty() => { + let n = 1 + next() as usize % drawn[a].len().min(400); + let gone: Vec = (0..n).map(|_| drawn[a].swap_remove(next() as usize % drawn[a].len())).collect(); + for &phys in &gone { + if next() % 64 == 0 && m.pins.pin(phys) { + pinned.push(phys); + } + } + m.frames.free_drawn(l, gone, &m.pins); + false + } + 2 if !pinned.is_empty() => { + let phys = pinned.swap_remove(next() as usize % pinned.len()); + if m.pins.unpin(phys) == Unpinned::Release { + m.frames.release(phys); + } + false + } + 3 => { + let count = 1 + next() as usize % 2; + match m.frames.alloc_run_for(l, count) { + Some(run) if a < 3 => { + runs[a].push((run, count)); + false + } + Some(run) => { + m.frames.free_run_for(l, run, count, &m.pins); + false + } + None => true, + } + } + 4 if !runs[a].is_empty() => { + let (run, count) = runs[a].swap_remove(next() as usize % runs[a].len()); + m.frames.free_run_for(l, run, count, &m.pins); + false + } + 5 if a < 3 => match m.frames.alloc_table(l) { + Some(t) => { + tables[a].push(t); + false + } + None => true, + }, + 6 if !tables[a].is_empty() => { + let t = tables[a].swap_remove(next() as usize % tables[a].len()); + m.frames.free_table(l, t, &m.pins); + false + } + 7 if a == 3 => !m.frames.promise(l, PER + next() % (4 * PER)) || { + m.frames.unpromise(l, l.usage().committed); + false + }, + _ => false, + }; + if refused && !m.frames.ready(l) { + waiting[a] = true; + floor_waits += usize::from(l.usage().floor > 0); + } + if m.frames.take_returned() { + posts += 1; + for (w, l) in waiting.iter_mut().zip(&accounts) { + *w = *w && !m.frames.ready(l); + } + } + for (i, l) in accounts.iter().enumerate().filter(|&(i, _)| waiting[i]) { + let (frames, supers, row) = l.asked(); + let grants = m.frames.admit(l, frames, frames - supers * PER, supers).is_some() && m.frames.in_a_row(row); + assert!(!grants, "step {step}: account {i} would be granted what it waits for and was not told"); + } + } + assert!(posts > 100 && floor_waits > 100, "the model waited too little: {posts} answers, {floor_waits} floor waits"); + } + + /// However ordinary accounts fragment memory around it, an account's + /// floor is held: at every step, what it does not use is promisable to it + /// and to no other account, and a promise it is owed always draws. + #[test] + fn fragmentation_never_takes_a_floor() { + let mut m = whole(16); + m.frames.keep(2); + let f = floored(&mut m, 3 * PER + 77); + let mut seed = 0x51f1_5eed_c0ff_ee11u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let (mut drawn, mut tables, mut runs, mut floor_drawn) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let (mut promised, mut floor_promised) = (0u64, 0u64); + for step in 0..30_000 { + match next() % 9 { + 0 => { + let n = next() % 600; + if m.frames.promise(&m.l, n) { + promised += n; + } + } + 1 if promised > 0 => { + drawn.push(m.frames.alloc_promised()); + promised -= 1; + } + 2 if !drawn.is_empty() => { + let i = next() as usize % drawn.len(); + m.frames.free_drawn(&m.l, [drawn.swap_remove(i)], &m.pins); + } + 3 => tables.extend(m.frames.alloc_table(&m.l)), + 4 if !tables.is_empty() => m.frames.free_table(&m.l, tables.swap_remove(next() as usize % tables.len()), &m.pins), + 5 => runs.extend(m.frames.alloc_run_for(&m.l, 1)), + 6 if !runs.is_empty() => m.frames.free_run_for(&m.l, runs.swap_remove(next() as usize % runs.len()), 1, &m.pins), + 7 => { + let n = next() % 200; + if m.frames.promise(&f, n) { + floor_promised += n; + for _ in 0..n / 2 { + floor_drawn.push(m.frames.alloc_promised()); + floor_promised -= 1; + } + } + } + 8 if !floor_drawn.is_empty() => { + let n = (next() as usize % floor_drawn.len()).max(1); + let gone: Vec = floor_drawn.drain(..n).collect(); + m.frames.free_drawn(&f, gone, &m.pins); + } + _ => {} + } + let unused = f.unused(); + assert_eq!(m.frames.floors_held().0, unused, "step {step}: the floor's frames are not all held"); + if step % 97 == 0 && unused > 0 { + assert!(!m.frames.promise(&m.l, m.frames.room() + 1), "step {step}: an ordinary promise took a floor's frame"); + assert!(m.frames.promise(&f, unused), "step {step}: the floor account was refused {unused} of its floor"); + m.frames.unpromise(&f, unused); + } + } + while promised + floor_promised > 0 { + let _ = m.frames.alloc_promised(); + if promised > 0 { + promised -= 1; + } else { + floor_promised -= 1; + } + } + } + /// However a process's 4 KiB traffic fragments memory, the heap always /// finds a whole superframe for what its charges cover, and a promised /// frame is always there to draw: a deterministic mix of promises, @@ -1095,7 +2088,7 @@ mod tests { match next() % 12 { 0 => { let n = next() % 300; - if m.frames.promise(n) { + if m.frames.promise(&m.l, n) { promised += n; } } @@ -1113,20 +2106,20 @@ mod tests { // Every 7th drawn frame goes back unpromised, scattering holes. let i = next() as usize % drawn.len(); if i.is_multiple_of(7) { - m.frames.free_frame(drawn.swap_remove(i), &m.pins); + m.frames.free_drawn(&m.l, [drawn.swap_remove(i)], &m.pins); } } - 5 => tables.extend(m.frames.alloc_frame_for_process()), - 6 if !tables.is_empty() => m.frames.free_frame(tables.swap_remove(next() as usize % tables.len()), &m.pins), - 7 => runs.extend(m.frames.alloc_run_for_process(1)), - 8 if !runs.is_empty() => m.frames.free_super(runs.swap_remove(next() as usize % runs.len()), &m.pins), + 5 => tables.extend(m.frames.alloc_table(&m.l)), + 6 if !tables.is_empty() => m.frames.free_table(&m.l, tables.swap_remove(next() as usize % tables.len()), &m.pins), + 7 => runs.extend(m.frames.alloc_run_for(&m.l, 1)), + 8 if !runs.is_empty() => m.frames.free_run_for(&m.l, runs.swap_remove(next() as usize % runs.len()), 1, &m.pins), 9 => { let bytes = next() % (SUPER / 2); - if m.frames.charge(bytes) { + if m.frames.charge(&m.l, Pool::Shared, bytes) { charges.push(bytes); } } - 10 if !charges.is_empty() => m.frames.uncharge(charges.swap_remove(next() as usize % charges.len())), + 10 if !charges.is_empty() => m.frames.uncharge(&m.l, Pool::Shared, charges.swap_remove(next() as usize % charges.len())), 11 => match next() % 2 { 0 => kernel.extend(m.frames.alloc_frame()), _ if !kernel.is_empty() => m.frames.free_frame(kernel.swap_remove(next() as usize % kernel.len()), &m.pins), diff --git a/kernel/pure/sched/job.rs b/kernel/pure/sched/job.rs index c8fb823e267..86320dab3c0 100644 --- a/kernel/pure/sched/job.rs +++ b/kernel/pure/sched/job.rs @@ -6,7 +6,11 @@ //! The job counts its runnable ordinary threads on every CPU; each CPU reads //! the count when it decides, so a change on one CPU reaches the job's groups //! elsewhere at their own CPUs' next decisions. +//! +//! **A job is also the account its memory is counted to** ([`Ledger`]): +//! every page its threads commit, and the kernel memory its objects keep. +use crate::frames::Ledger; use crate::sched::sync::{AtomicU32, Ordering}; /// Monotonic and never reused: a run queue keys the job's group by it. @@ -20,11 +24,29 @@ pub struct Job { key: JobKey, /// Runnable ordinary threads of this job, on every CPU. runnable: AtomicU32, + memory: Ledger, +} + +/// Every charge, page and run of a job holds it, so the last of it going +/// is its account at nothing: anything left is a give-back that missed it. +impl Drop for Job { + fn drop(&mut self) { + let usage = self.memory.usage(); + assert!( + self.memory.is_empty() && usage.floor == 0 && usage.entitled == 0, + "job {:?} ended holding {usage:?}", + self.key + ); + } } impl Job { pub fn new(key: JobKey) -> Self { - Self { key, runnable: AtomicU32::new(0) } + Self { key, runnable: AtomicU32::new(0), memory: Ledger::new() } + } + + pub fn memory(&self) -> &Ledger { + &self.memory } pub fn key(&self) -> JobKey { diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index ecf97503599..8201e749860 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -28,7 +28,9 @@ use toyos_bootmap::aarch64::{coverage, direct_map_end, Coverage, ATTR_DEVICE, ATTR_NORMAL, ATTR_NORMAL_NC}; +use alloc::sync::Arc; use kernel::pcid::{Alloc, Pcid, PcidPool}; +use kernel::sched::job::Job; use super::tlb; use crate::mm::policy::{CachePolicy, MmioPolicy, Prot}; @@ -142,13 +144,13 @@ impl Table { unsafe { Table::at_mut(frame.into_raw()) } } - /// A table of a user space, or `None` where taking a frame would cut into - /// what the kernel keeps (`pmm::alloc_table_frame`). - fn try_new() -> Option<&'static mut Self> { - let frame = crate::mm::pmm::alloc_table_frame()?; + /// A table of a user space, counted to `job`, or `None` where taking a + /// frame would cut into what is held back (`pmm::alloc_table_frame`). + fn try_new(job: &Job) -> Option<&'static mut Self> { + let phys = crate::mm::pmm::alloc_table_frame(job)?; // SAFETY: a zeroed frame just allocated: a table of invalid // descriptors nothing else names. - Some(unsafe { Table::at_mut(frame.into_raw()) }) + Some(unsafe { Table::at_mut(phys) }) } fn phys(&self) -> u64 { @@ -190,11 +192,12 @@ fn published() { unsafe { core::arch::asm!("dsb ishst", "isb", options(nostack, preserves_flags)) }; } -/// Give back a table [`Table::new`] made, which nothing links any more. -fn free_table(phys: u64) { - // SAFETY: the caller unlinked the table, and every table is a frame - // `Table::new` gave up. - drop(unsafe { crate::mm::pmm::Frame::from_raw(phys) }); +/// Give back a table of `job`'s user space [`Table::try_new`] made, which +/// nothing links any more. +fn free_table(job: &Job, phys: u64) { + // SAFETY: the caller unlinked the table, after the shootdown that let + // every CPU stop walking it. + unsafe { crate::mm::pmm::free_table_frame(job, phys) }; } /// A root and every table under it, each a frame the descriptor linking it @@ -557,13 +560,14 @@ impl Unlinked { self.head = phys; } - /// Free the tables, only after the shootdown that followed the unlink. - pub fn free(self) { + /// Free the tables of `job`'s space, only after the shootdown that + /// followed the unlink. + fn free(self, job: &Job) { let mut at = self.head; while at != 0 { // SAFETY: a table of the chain, which only this frees. let next = unsafe { Table::at_mut(at) }.0[0]; - free_table(at); + free_table(job, at); at = next; } } @@ -576,18 +580,20 @@ impl Unlinked { pub struct UserTables { tables: Tables, asid: Asid, + /// The job its tables are counted to. + job: Arc, } impl UserTables { /// An empty user half, or `None` when every user ASID is held or no - /// frame is free for its root. - pub fn new() -> Option { - let root = Table::try_new()?.phys(); + /// frame is free for its root. Its tables are counted to `job`. + pub fn new(job: &Arc) -> Option { + let root = Table::try_new(job)?.phys(); let Some(asid) = alloc_asid() else { - free_table(root); + free_table(job, root); return None; }; - Some(Self { tables: Tables { root }, asid: Asid::User(asid) }) + Some(Self { tables: Tables { root }, asid: Asid::User(asid), job: Arc::clone(job) }) } /// The kernel's own space, whose user half is empty: what a kernel thread @@ -595,7 +601,7 @@ impl UserTables { pub fn kernel() -> Self { let root = KERNEL_TTBR0.load(core::sync::atomic::Ordering::Acquire) & ADDR; assert!(root != 0, "paging: the kernel's space before `init`"); - Self { tables: Tables { root }, asid: Asid::Kernel } + Self { tables: Tables { root }, asid: Asid::Kernel, job: Arc::clone(crate::mm::charge::boot()) } } pub fn root(&self) -> Root { @@ -612,6 +618,7 @@ impl UserTables { /// built linked for an [`unlink`](Self::unlink). Only an invalid entry is /// written, which owes nothing. pub fn build(&mut self, start: u64, end: u64) -> Option { + let job = Arc::clone(&self.job); let mut built = 0; let mut va = start & !(PAGE_2M - 1); while va < end { @@ -621,7 +628,7 @@ impl UserTables { let table = unsafe { Table::at_mut(level) }; let i = index(va, depth); if table.0[i] & VALID == 0 { - let Some(child) = Table::try_new() else { + let Some(child) = Table::try_new(&job) else { published(); return None; }; @@ -754,7 +761,13 @@ impl UserTables { /// Unlink every table over `[start, end)` whose span `needed` answers /// `false` for, and every table above one that is then empty. A table /// unlinked holds no leaf: a span no range needs maps nothing. They are - /// freed by [`Unlinked::free`], after the caller's shootdown. + /// freed by [`free`](Self::free), after the caller's shootdown. + /// Free tables [`unlink`](Self::unlink) took out, only after the + /// shootdown that followed it. + pub fn free(&self, unlinked: Unlinked) { + unlinked.free(&self.job); + } + pub fn unlink(&mut self, start: u64, end: u64, needed: impl Fn(u64, u64) -> bool) -> Unlinked { let mut unlinked = Unlinked::none(); // SAFETY: as `each_table`'s. @@ -819,8 +832,8 @@ impl Drop for UserTables { fn drop(&mut self) { assert!(matches!(self.asid, Asid::User(_)), "paging: the kernel's address space dropped"); let all = self.unlink(0, toyos_userbound::USER_TOP, |_, _| false); - all.free(); - free_table(self.tables.root); + all.free(&self.job); + free_table(&self.job, self.tables.root); } } diff --git a/kernel/src/arch/x86_64/paging.rs b/kernel/src/arch/x86_64/paging.rs index 69677fac85d..65707e17d9a 100644 --- a/kernel/src/arch/x86_64/paging.rs +++ b/kernel/src/arch/x86_64/paging.rs @@ -10,7 +10,9 @@ // executable (`mm::audit_wx`). use crate::log; +use alloc::sync::Arc; use alloc::vec::Vec; +use kernel::sched::job::Job; use kernel::pcid::{Alloc, Pcid, PcidPool}; @@ -139,8 +141,8 @@ const _: () = assert!( struct PageTablePage([u64; 512]); impl PageTablePage { - /// An empty table in a frame of its own, owned from here by the entry - /// that links it: [`free_table`] gives it back. + /// An empty table of the kernel's in a frame of its own, owned from here + /// by the entry that links it. fn new() -> &'static mut PageTablePage { let frame = crate::mm::pmm::alloc_frame().expect("paging: no frame for a page table"); // SAFETY: a zeroed frame just allocated, so a table of empty entries @@ -148,13 +150,13 @@ impl PageTablePage { unsafe { Self::from_phys_mut(frame.into_raw()) } } - /// A table of a user space, or `None` where taking a frame would cut into - /// what the kernel keeps (`pmm::alloc_table_frame`). - fn try_new() -> Option<&'static mut PageTablePage> { - let frame = crate::mm::pmm::alloc_table_frame()?; + /// A table of a user space, counted to `job`, or `None` where taking a + /// frame would cut into what is held back (`pmm::alloc_table_frame`). + fn try_new(job: &Job) -> Option<&'static mut PageTablePage> { + let phys = crate::mm::pmm::alloc_table_frame(job)?; // SAFETY: a zeroed frame just allocated, so a table of empty entries // nothing else names. - Some(unsafe { Self::from_phys_mut(frame.into_raw()) }) + Some(unsafe { Self::from_phys_mut(phys) }) } fn phys(&self) -> u64 { @@ -457,11 +459,12 @@ fn alloc_pcid() -> Option { } } -/// Give back a table [`PageTablePage::new`] made, which nothing links any more. -fn free_table(phys: u64) { - // SAFETY: the caller unlinked the table, and every table is a frame - // `PageTablePage::new` gave up. - drop(unsafe { crate::mm::pmm::Frame::from_raw(phys) }); +/// Give back a table of `job`'s user space [`PageTablePage::try_new`] made, +/// which nothing links any more. +fn free_table(job: &Job, phys: u64) { + // SAFETY: the caller unlinked the table, after the shootdown that let + // every CPU stop walking it. + unsafe { crate::mm::pmm::free_table_frame(job, phys) }; } /// The kernel half every space shares: PML4[256..511] and what they name. @@ -783,13 +786,14 @@ impl Unlinked { self.head = phys; } - /// Free the tables, only after the shootdown that followed the unlink. - pub fn free(self) { + /// Free the tables of `job`'s space, only after the shootdown that + /// followed the unlink. + fn free(self, job: &Job) { let mut at = self.head; while at != 0 { // SAFETY: a table of the chain, which only this frees. let next = unsafe { PageTablePage::from_phys_mut(at) }.0[0]; - free_table(at); + free_table(job, at); at = next; } } @@ -805,15 +809,18 @@ pub struct UserTables { /// Owned for this space's life: dropping it returns a user tag, so two live /// spaces can never share one. pcid: PcidHandle, + /// The job its tables are counted to. + job: Arc, } impl UserTables { /// A new space with the kernel half copied in, or `None` when every user - /// PCID is held by a live space or no frame is free for its root. - pub fn new() -> Option { - let pml4 = PageTablePage::try_new()?; + /// PCID is held by a live space or no frame is free for its root. Its + /// tables are counted to `job`. + pub fn new(job: &Arc) -> Option { + let pml4 = PageTablePage::try_new(job)?; let Some(pcid) = alloc_pcid() else { - free_table(pml4.phys()); + free_table(job, pml4.phys()); return None; }; let kernel = KERNEL.lock(); @@ -826,7 +833,7 @@ impl UserTables { ); pml4.init_entry(slot, entry); } - Some(Self { root: pml4.phys(), pcid: PcidHandle::User(pcid) }) + Some(Self { root: pml4.phys(), pcid: PcidHandle::User(pcid), job: Arc::clone(job) }) } /// The kernel's own space, whose user half is empty: what a kernel thread @@ -834,7 +841,7 @@ impl UserTables { pub fn kernel() -> Self { let root = KERNEL.lock().root; assert!(root != 0, "paging: the kernel's space before `init`"); - Self { root, pcid: PcidHandle::Kernel } + Self { root, pcid: PcidHandle::Kernel, job: Arc::clone(crate::mm::charge::boot()) } } pub fn root(&self) -> Cr3 { @@ -863,6 +870,7 @@ impl UserTables { /// built linked for an [`unlink`](Self::unlink). Only an empty entry is /// written, which owes nothing. pub fn build(&mut self, start: u64, end: u64) -> Option { + let job = Arc::clone(&self.job); let mut built = 0; let mut va = start & !(PAGE_2M - 1); while va < end { @@ -870,7 +878,7 @@ impl UserTables { let mut level = self.root_table_mut(); for index in [a, b, c] { if level[index] & PAGE_PRESENT == 0 { - let child = PageTablePage::try_new()?; + let child = PageTablePage::try_new(&job)?; level.write(index, va, child.phys() | TABLE_FLAGS).expect_install("build"); built += 1; } @@ -991,8 +999,14 @@ impl UserTables { /// Unlink every table over `[start, end)` whose span `needed` answers /// `false` for, and every table above one that is then empty; a table /// that still links anything stays. A table unlinked holds no leaf: a span - /// no range needs maps nothing. They are freed by [`Unlinked::free`], + /// no range needs maps nothing. They are freed by [`free`](Self::free), /// after the caller's shootdown. + /// Free tables [`unlink`](Self::unlink) took out, only after the + /// shootdown that followed it. + pub fn free(&self, unlinked: Unlinked) { + unlinked.free(&self.job); + } + pub fn unlink(&mut self, start: u64, end: u64, needed: impl Fn(u64, u64) -> bool) -> Unlinked { let mut unlinked = Unlinked::none(); let root = self.root_table_mut(); @@ -1046,8 +1060,8 @@ impl Drop for UserTables { fn drop(&mut self) { assert!(matches!(self.pcid, PcidHandle::User(_)), "paging: the kernel's address space dropped"); let all = self.unlink(0, (ROOT_HIGH_HALF as u64) << 39, |_, _| false); - all.free(); - free_table(self.root); + all.free(&self.job); + free_table(&self.job, self.root); } } diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 7e817fd9c31..409c11d8012 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -69,7 +69,7 @@ static PAGE: core::sync::atomic::AtomicPtr(), core::mem::size_of::()) }; diff --git a/kernel/src/elf/mod.rs b/kernel/src/elf/mod.rs index bec4012b400..aa5bbfff15a 100644 --- a/kernel/src/elf/mod.rs +++ b/kernel/src/elf/mod.rs @@ -50,7 +50,7 @@ pub struct Run(alloc::sync::Arc); impl Run { /// `size` bytes of zeroed memory in one run. pub fn new(size: usize) -> Option { - crate::object::shm::SharedMemObject::create_run(size as u64).ok().map(Self) + crate::object::shm::SharedMemObject::create_run(&crate::mm::charge::current(), size as u64).ok().map(Self) } pub fn ptr(&self) -> *mut u8 { diff --git a/kernel/src/file_cache.rs b/kernel/src/file_cache.rs index 2c71d542c01..1242219c64e 100644 --- a/kernel/src/file_cache.rs +++ b/kernel/src/file_cache.rs @@ -45,7 +45,7 @@ impl Page { /// A page of zeros, or `Exhausted`, charged before it is allocated with /// its place in its file. fn blank() -> Result { - let charge = Charge::new(charge::heap(core::alloc::Layout::new::<[u8; PAGE_SIZE]>()) + charge::arc::() + charge::map_entry::())?; + let charge = Charge::current(charge::heap(core::alloc::Layout::new::<[u8; PAGE_SIZE]>()) + charge::arc::() + charge::map_entry::())?; Ok(Self { bytes: blank_page(), _charge: charge }) } @@ -119,7 +119,7 @@ pub fn init() { /// Allocate a new FileId. The file cache is the sole allocator. pub fn create_file(evictable: bool, mtime: u64) -> Result { - let charge = Charge::new(charge::map_entry::())?; + let charge = Charge::current(charge::map_entry::())?; let mut cache = FILE_CACHE.lock(); let id = cache.next_id; cache.next_id += 1; @@ -384,7 +384,7 @@ pub fn snapshot(file_id: FileId) -> Result { if file.pages.len() > crate::mm::MAX_HEAP_ALLOC / core::mem::size_of::<(u32, PageData)>() { return Err(SyscallError::ResourceExhausted); } - let charge = Charge::new(charge::arc::() + charge::array::<(u32, PageData)>(file.pages.len()))?; + let charge = Charge::current(charge::arc::() + charge::array::<(u32, PageData)>(file.pages.len()))?; let pages = file.pages.iter().map(|(&idx, page)| (idx, Arc::clone(&page.data))).collect(); Ok(Snapshot { size: file.size, pages, _charge: charge }) } diff --git a/kernel/src/inbox/mod.rs b/kernel/src/inbox/mod.rs index 0dc6a4e9543..6b7e1474b52 100644 --- a/kernel/src/inbox/mod.rs +++ b/kernel/src/inbox/mod.rs @@ -358,10 +358,10 @@ pub fn create(depth: u32) -> Result<(InboxRef, u64), SyscallError> { let submission_size = depth; let completion_size = depth * 2; - let charge = Charge::new(heap())?; + let charge = Charge::current(heap())?; let space = process::current_address_space(); let bytes = SUBMISSIONS_OFF + u64::from(MAX_SUBMISSION_DEPTH) * core::mem::size_of::() as u64; - let ring = SharedMemObject::create(bytes)?; + let ring = SharedMemObject::create(&crate::mm::charge::current(), bytes)?; // Laid out before mapped: mapping first lets a sibling thread write the // rings while the kernel initializes them. ring.write(0, &ring_layout(submission_size, completion_size)); diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index be39f249f18..cf1e05b6d78 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -20,7 +20,7 @@ mod start; mod tls; -pub use start::{build_child_handles, PendingHandles}; +pub use start::{build_child_handles, Commit, PendingHandles}; pub(crate) use start::{alloc_kernel_stack, KernelStack, Start}; pub use tls::{TlsBlock, DTV_INITIAL_CAPACITY, VARIANT as TLS_VARIANT}; @@ -166,9 +166,10 @@ fn segment_prot(seg: &toyos_elf::Segment) -> kernel::vm::Prot { /// Load a program and place its main thread under `parent`, answering its pid -/// and what `commit` left its caller holding of it. `commit` builds the -/// child's handle table around the child's handle to itself, once nothing is -/// left to refuse, and names the job it runs in. +/// and what `commit` left its caller holding of it. Everything built for the +/// child is counted to [`Commit::job`], which it runs in, and `commit` builds +/// the child's handle table around its handle to itself once nothing is left +/// to refuse. /// /// `path` is the program, opened here — or, with `image`, the bytes the caller /// read from it itself, and then nothing opens it. `argv[0]` is only the name the child goes by. @@ -176,24 +177,23 @@ fn segment_prot(seg: &toyos_elf::Segment) -> kernel::vm::Prot { /// `Refusal`, not `-> !`, is the error type: every failure below owns a /// partly built process (address space, stack, kernel stack), and nothing /// unwinds, so the error must travel out as a value rather than strand it. -pub fn spawn( +pub fn spawn( path: &str, argv: &[&str], - commit: impl FnOnce( - start::Own, - ) -> Result<(HandleTable, Endowments, Arc, H), crate::object::Refusal>, + commit: C, cwd: String, env: Vec, image: Option>, parent: Parent, -) -> Result<(Pid, H), crate::object::Refusal> { +) -> Result<(Pid, C::Held), crate::object::Refusal> { + let job = Arc::clone(commit.job()); // An argv of only separators survives sys_spawn's split as an empty slice. let Some(&name) = argv.first() else { return Err(SyscallError::InvalidArgument.into()); }; // Before anything is built; dropped on every way out below but the insert. let admission = Admission::ask(parent)?; - let mut charge = ProcessData::charge(path, &env).map_err(SyscallError::from)?; + let mut charge = ProcessData::charge(&job, path, &env).map_err(SyscallError::from)?; let t0 = crate::clock::nanos_since_boot(); let backing: Arc = match image { @@ -240,8 +240,8 @@ pub fn spawn( let t1 = crate::clock::nanos_since_boot(); - let Some(child_pt) = crate::mm::space::Space::new_user() else { - log!("spawn: {}: no user PCID free — too many live address spaces", path); + let Some(child_pt) = crate::mm::space::Space::new_user(&job) else { + log!("spawn: {}: no address space: every user PCID is held, or no frame is left for its root", path); return Err(SyscallError::ResourceExhausted.into()); }; let (image_start, base, stack_base, sp) = { @@ -343,7 +343,7 @@ pub fn spawn( bias: base, }); - let (ks_alloc, ks_sp) = match alloc_kernel_stack(Start::Process { entry, sp }) { + let (ks_alloc, ks_sp) = match alloc_kernel_stack(&job, Start::Process { entry, sp }) { Some(ks) => ks, None => { log!("spawn: {}: failed to allocate kernel stack", path); @@ -356,7 +356,7 @@ pub fn spawn( let notice = crate::object::notice::NoticeObject::new().map_err(SyscallError::from)?; // The point of no return: every failure above answers the caller with its // table untouched. - let (handles, endowments, job, held) = commit(start::Own::new(&object, ¬ice))?; + let (handles, endowments, held) = commit.commit(start::Own::new(&object, ¬ice))?; let proc_data = Arc::new(Lock::new(ProcessData { handles, cwd, @@ -448,11 +448,12 @@ const SUPERVISOR: &str = "supervisor"; /// processes that can ever claim a device, enter the RT band, or power off. /// Panics on failure: a boot that cannot start the supervisor has nowhere to report to. pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { - let mut handles = HandleTable::new(); + let job = crate::object::job::new_job(); + let mut handles = HandleTable::new(&job); let mut entries = Vec::new(); let mut labels = Vec::new(); // One console, the supervisor's descriptors 0 to 2. - let console = KObjectRef::Console(crate::object::device::ConsoleObject::new().expect("spawn_supervisor: no heap for its console")); + let console = KObjectRef::Console(crate::object::device::ConsoleObject::new(&job).expect("spawn_supervisor: no heap for its console")); for fd in ["0", "1", "2"] { let entry = crate::object::HandleEntry::new(console.clone(), ops::initial_rights(&console)); let handle = handles.install(entry).expect("spawn_supervisor: three handles cannot exhaust an empty table"); @@ -471,6 +472,7 @@ pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { .union(Rights::TRANSFER) .union(Rights::DEVICE) .union(Rights::JOBS) + .union(Rights::FLOOR) .union(Rights::RT) .union(Rights::LOG) .union(Rights::WAIT) @@ -513,13 +515,32 @@ pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { .expect("spawn_supervisor: a boot image is ordinary memory, its length inside it"), ); // Built by the kernel and owing nobody anything: no table but the supervisor's own holds it. - let commit = |own: start::Own| { - own.endow(&mut handles, &mut entries, &mut labels); - Ok((handles, Endowments::new(entries, labels), crate::object::job::new_job(), ())) - }; - match spawn(SUPERVISOR, &[SUPERVISOR], commit, String::from("/"), Vec::new(), Some(image), Parent::Root) { + let table = SupervisorTable { job, handles, entries, labels }; + match spawn(SUPERVISOR, &[SUPERVISOR], table, String::from("/"), Vec::new(), Some(image), Parent::Root) { Ok((pid, ())) => pid, Err(crate::object::Refusal::Error(e)) => panic!("spawn_supervisor: failed to spawn: {e:?}"), Err(crate::object::Refusal::Handle(e)) => panic!("spawn_supervisor: {e}"), } } + +/// The supervisor's table as the kernel lays it out, in a job of its own. +struct SupervisorTable { + job: Arc, + handles: HandleTable, + entries: Vec, + labels: Vec, +} + +impl Commit for SupervisorTable { + type Held = (); + + fn job(&self) -> &Arc { + &self.job + } + + fn commit(self, own: start::Own) -> Result<(HandleTable, Endowments, ()), crate::object::Refusal> { + let Self { mut handles, mut entries, mut labels, .. } = self; + own.endow(&mut handles, &mut entries, &mut labels); + Ok((handles, Endowments::new(entries, labels), ())) + } +} diff --git a/kernel/src/loader/start.rs b/kernel/src/loader/start.rs index 2663cfe8aad..74919ddbdca 100644 --- a/kernel/src/loader/start.rs +++ b/kernel/src/loader/start.rs @@ -93,15 +93,16 @@ impl KernelStack { } } -/// Allocate a kernel stack and lay out the frame `context_switch` will restore. -pub(crate) fn alloc_kernel_stack(start: Start) -> Option<(KernelStack, u64)> { +/// Allocate a kernel stack, charged to the job its task runs in, and lay out +/// the frame `context_switch` will restore. +pub(crate) fn alloc_kernel_stack(job: &Arc, start: Start) -> Option<(KernelStack, u64)> { use crate::arch::entry::{kernel_start, process_start, thread_start}; let (trampoline, user_entry, user_sp, arg, user): (unsafe extern "C" fn(), _, _, _, _) = match start { Start::Process { entry, sp } => (process_start, entry.addr(), sp, 0, true), Start::Thread { entry, sp, arg } => (thread_start, entry.addr(), sp, arg, true), Start::Kernel { body, arg } => (kernel_start, body as usize as u64, 0, arg, false), }; - let charge = Charge::new(KernelStack::heap(user)).ok()?; + let charge = Charge::new(job, KernelStack::heap(user)).ok()?; let layout = KernelStack::layout_for(user); let alloc = OwnedAlloc::new(layout.size(), layout.align())?; scheduler::write_stack_canary(&alloc); @@ -126,14 +127,36 @@ pub struct PendingHandles { table: HandleTable, endow: Vec, labels: Vec, + /// The job the child runs in, which everything built for it is counted to. + job: Arc, } -impl PendingHandles { +/// What a spawn's caller gives the child: the job it runs in, and its handle +/// table, made once nothing is left to refuse. +pub trait Commit { + /// What the caller is left holding of the child. + type Held; + + /// The job the child runs in, which everything built for it is counted to. + fn job(&self) -> &Arc; + + /// The child's table around `own`, what the kernel puts beside its + /// endowments. + fn commit(self, own: Own) -> Result<(HandleTable, Endowments, Self::Held), Refusal>; +} + +impl Commit for PendingHandles { + type Held = RawHandle; + + fn job(&self) -> &Arc { + &self.job + } + /// Take the endowed handles out of the caller's table and put its handle to the child in it, all under one lock hold: a refusal leaves the table unchanged. - /// `own` is what the kernel puts in the child's table beside the endowments. The child's job is the one endowed under - /// [`JOB_LABEL`], or else the caller's; the caller's handle to the child is the last answer. - pub fn commit(self, own: Own) -> Result<(HandleTable, Endowments, Arc, RawHandle), Refusal> { - let Self { mut table, endow, mut labels } = self; + /// `own` is what the kernel puts in the child's table beside the endowments; the caller's handle to the child is the + /// last answer. The handle under [`JOB_LABEL`] still names [`job`](Self::job), or the spawn is refused. + fn commit(self, own: Own) -> Result<(HandleTable, Endowments, RawHandle), Refusal> { + let Self { mut table, endow, mut labels, job } = self; let data_arc = process_data(); let mut data = data_arc.lock(); @@ -174,6 +197,11 @@ impl PendingHandles { } moving.push((EndowEntry { label_off, label_len, handle, _pad: 0 }, handle)); } + // Before any removal: the table changed under the spawn if the job + // everything was built for is no longer the one it names. + if !Arc::ptr_eq(&job, endowed_job.as_ref().unwrap_or(&data.job)) { + return Err(SyscallError::InvalidArgument.into()); + } // Checked before any removal, so a failed install can't strand a handle out of a table that never spawned. if !table.make_room(moving.len() + Own::LABELS.len()) || !data.handles.make_room(1) { return Err(SyscallError::ResourceExhausted.into()); @@ -189,7 +217,7 @@ impl PendingHandles { if let KObjectRef::Console(parent_console) = data.handles.get_ref(*handle, Rights::NONE)? { let key = Arc::as_ptr(parent_console) as usize; if !minted.iter().any(|(seen, _)| *seen == key) { - let console = crate::object::device::ConsoleObject::new().map_err(SyscallError::from)?; + let console = crate::object::device::ConsoleObject::new(&job).map_err(SyscallError::from)?; minted.push((key, KObjectRef::Console(console))); } } @@ -222,11 +250,10 @@ impl PendingHandles { // Minted while `own` is held, and `own` goes into a table no thread reaches until the child lands: another thread of the caller closing this handle never closes the object's last. let held = ops::install(&mut data.handles, own.process.object().clone()) .expect("a caller's table with verified room refused its child"); - let job = endowed_job.unwrap_or_else(|| data.job.clone()); drop(data); drop(displaced); own.endow(&mut table, &mut entries, &mut labels); - Ok((table, Endowments::new(entries, labels), job, held)) + Ok((table, Endowments::new(entries, labels), held)) } } @@ -272,5 +299,21 @@ pub fn build_child_handles(endow: &UserBytes, labels: &[u8]) -> Result Result, Refusal> { + let data_arc = process_data(); + let data = data_arc.lock(); + for entry in raw.as_chunks::().0 { + let word = |at: usize| u32::from_ne_bytes([entry[at], entry[at + 1], entry[at + 2], entry[at + 3]]) as usize; + let named = word(0).checked_add(word(4)).and_then(|end| labels.get(word(0)..end)); + if named == Some(JOB_LABEL.as_bytes()) { + return Ok(data.handles.get_sent::(RawHandle(word(8) as u32), Rights::WRITE)?.job().clone()); + } + } + Ok(data.job.clone()) } diff --git a/kernel/src/loader/tls.rs b/kernel/src/loader/tls.rs index cc9c4b0b59f..8a8620c6b98 100644 --- a/kernel/src/loader/tls.rs +++ b/kernel/src/loader/tls.rs @@ -50,11 +50,12 @@ impl TlsBlock { build_combined(modules, tls) } - /// Map the block into `space`, answering its mapping, the thread pointer, + /// Map the block into `space`, counted to its job, answering its mapping, the thread pointer, /// and where that pointer lies in the block. `None` when `space` has no /// room or no memory is left for it. pub fn publish(mut self, pt: &PageTables) -> Option<(KernelMapping, u64, usize)> { - let object = SharedMemObject::create(self.bytes.len() as u64).ok()?; + let job = alloc::sync::Arc::clone(pt.lock().job()); + let object = SharedMemObject::create(&job, self.bytes.len() as u64).ok()?; let mut space = pt.lock(); let at = space.reserve(object.size(), At::Anywhere(self.align), kernel::vm::Prot::None).ok()?; rebase(&mut self.bytes, self.tp_offset, at.raw()); diff --git a/kernel/src/mm/charge.rs b/kernel/src/mm/charge.rs index f2958f0b0c6..bed543c2464 100644 --- a/kernel/src/mm/charge.rs +++ b/kernel/src/mm/charge.rs @@ -1,24 +1,29 @@ -//! The kernel heap a process causes, charged before it is allocated and given -//! back when what it covers goes. +//! The kernel heap a process causes, charged to its job before it is +//! allocated and given back when what it covers goes. //! //! **An object that outlives the call that made it holds a [`Charge`] for //! every heap allocation it keeps**, taken in its constructor before the -//! allocations and dropped with it. A charge is refused, `ResourceExhausted` -//! at the call, where covering it would take what the frame allocator holds -//! back for the kernel (`pmm::charge`); an allocation it covers then finds -//! room, since the heap grows into what charges reserved first, as long as -//! the heap's fragmentation stays within what the charges' estimates -//! overcount (`issues/the-kernel-heaps-fragmentation-is-charged-to-nobody.md`). -//! What a call allocates and frees before it returns, past what its thread is +//! allocations and dropped with it, against the job that made it: the +//! caller's ([`current`]), or the one a spawn makes it for. A charge is +//! refused, `ResourceExhausted` at the call, where covering it would take +//! what the frame allocator holds back for the kernel or another job's floor +//! (`pmm::charge`); an allocation it covers then finds room, since the heap +//! grows into what charges reserved first, as long as the heap's +//! fragmentation stays within what the charges' estimates overcount +//! (`issues/the-kernel-heaps-fragmentation-is-charged-to-nobody.md`). What a +//! call allocates and frees before it returns, past what its thread is //! charged for, is not charged: the kernel's kept superframes cover it. +use alloc::sync::Arc; use core::alloc::Layout; +use kernel::frames::PoolSplit; +use kernel::sched::job::Job; use toyos_abi::syscall::SyscallError; use super::pmm; -/// Covering a charge would take what the kernel holds back. +/// Covering a charge would take what is held back. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Exhausted; @@ -28,63 +33,103 @@ impl From for SyscallError { } } -/// Bytes of the kernel heap charged, given back when it drops. -#[derive(Debug)] -pub struct Charge(u64); +/// The job the running thread is in, whose account what its call causes is +/// charged to; before any thread runs, the boot's own. +pub fn current() -> Arc { + crate::sched::driver::current_job().unwrap_or_else(|| Arc::clone(boot())) +} + +/// The account of what the kernel makes before any thread runs, made by +/// [`init`]. +static BOOT: core::sync::atomic::AtomicPtr> = core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()); + +/// Make the boot's account; `mm::init` calls it once the heap is up. +pub(super) fn init() { + let job: &'static Arc = alloc::boxed::Box::leak(alloc::boxed::Box::new(crate::object::job::new_job())); + BOOT.store(core::ptr::from_ref(job).cast_mut(), core::sync::atomic::Ordering::Release); +} + +pub fn boot() -> &'static Arc { + let job = BOOT.load(core::sync::atomic::Ordering::Acquire); + assert!(!job.is_null(), "charge: a charge before `mm::init`"); + // SAFETY: `init` stored a leaked box's pointer, never cleared or freed. + unsafe { &*job } +} + +/// Bytes of the kernel heap charged to a job, given back when it drops. Each +/// growth comes from the pool the job charges at the time (`frames::PoolSplit`). +pub struct Charge { + job: Arc, + held: PoolSplit, +} + +impl core::fmt::Debug for Charge { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?} of job {}", self.held, self.job.key().0) + } +} impl Charge { - /// No bytes: a charge to grow later. - pub const NONE: Self = Self(0); + /// No bytes yet, charged to `job` as it grows. + pub fn none(job: &Arc) -> Self { + Self { job: Arc::clone(job), held: PoolSplit::NONE } + } - /// `bytes` of heap. - pub fn new(bytes: usize) -> Result { - let mut charge = Self::NONE; + /// `bytes` of heap, charged to `job`. + pub fn new(job: &Arc, bytes: usize) -> Result { + let mut charge = Self::none(job); charge.grow(bytes)?; Ok(charge) } + /// `bytes` of heap, charged to the caller's job. + pub fn current(bytes: usize) -> Result { + Self::new(¤t(), bytes) + } + /// `more` bytes besides, refused whole. pub fn grow(&mut self, more: usize) -> Result<(), Exhausted> { - if more == 0 { - return Ok(()); - } - if !pmm::charge(more as u64) { - return Err(Exhausted); + if more == 0 || pmm::charge(&self.job, &mut self.held, more as u64) { + Ok(()) + } else { + Err(Exhausted) } - self.0 += more as u64; - Ok(()) } /// `less` bytes fewer. pub fn shrink(&mut self, less: usize) { - let less = less as u64; - assert!(less <= self.0, "charge: {less} of {} bytes given back", self.0); if less > 0 { - pmm::uncharge(less); - self.0 -= less; + pmm::uncharge(&self.job, &mut self.held, less as u64); } } /// Hold exactly `bytes`: grown, refused whole, or shrunk. pub fn resize(&mut self, bytes: usize) -> Result<(), Exhausted> { - match (bytes as u64).checked_sub(self.0) { + let held = self.held.bytes(); + match (bytes as u64).checked_sub(held) { Some(more) => self.grow(more as usize), None => { - self.shrink((self.0 - bytes as u64) as usize); + self.shrink((held - bytes as u64) as usize); Ok(()) } } } pub fn bytes(&self) -> usize { - self.0 as usize + self.held.bytes() as usize + } + + /// The job it is charged to. + pub fn job(&self) -> &Arc { + &self.job } } impl Drop for Charge { fn drop(&mut self) { - if self.0 > 0 { - pmm::uncharge(self.0); + let bytes = self.held.bytes(); + if bytes > 0 { + pmm::uncharge(&self.job, &mut self.held, bytes); } } } diff --git a/kernel/src/mm/mod.rs b/kernel/src/mm/mod.rs index bb5b4330bcb..384cbdc2ea5 100644 --- a/kernel/src/mm/mod.rs +++ b/kernel/src/mm/mod.rs @@ -182,6 +182,7 @@ pub fn init(memory_map: &'static [MemoryMapEntry], reserved: &[Region], image: & pmm::init(memory_map, reserved); DIRECT_MAP_END.set(paging::init(memory_map, crate::drivers::panic_console::scanout(), image)); alloc::init(); + charge::init(); paging::seal_kernel_half(); space::init_kernel(); } diff --git a/kernel/src/mm/pmm.rs b/kernel/src/mm/pmm.rs index d893612aa8d..dd474d594f5 100644 --- a/kernel/src/mm/pmm.rs +++ b/kernel/src/mm/pmm.rs @@ -1,22 +1,28 @@ //! Physical memory: `kernel::frames`' allocator over every usable frame //! firmware's map names, its records laid out at boot in memory it manages. //! -//! [`Frame`] owns one 4 KiB frame and [`PhysPage`] one 2 MiB superframe; each -//! returns its memory when it drops. Pins ([`pin_range`]) are taken without -//! the allocator's lock. +//! [`Frame`] owns one 4 KiB frame, [`PhysPage`] one 2 MiB superframe of the +//! kernel's and [`Run`] a run of an account's; each returns its memory when it +//! drops. Pins ([`pin_range`]) are taken without the allocator's lock. //! -//! What a process causes — a commitment, a page table, a pipe's ring, a run a -//! device is lent, a [`charge`] of the kernel heap — is refused rather than -//! take what the allocator holds back for the kernel; only the kernel's own -//! allocations may take it. - +//! **What a process causes is counted to the job that caused it**, its +//! account — a commitment, a page table, a pipe's ring, a run a device is +//! lent, a [`charge`] of the kernel heap — and is refused rather than take +//! what the allocator holds back for the kernel or for another account's +//! floor ([`set_floor`]); only the kernel's own allocations may take what it +//! keeps. A job refused memory can wait for it to come back: [`ready`] +//! answers whether it has, and [`returned`] is posted once it has. + +use alloc::sync::Arc; use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering}; -use kernel::frames::{self, Frames, Pins, Super, Unpinned, FRAME, SUPER}; +use kernel::frames::{self, Frames, Pins, PoolSplit, Super, Unpinned, FRAME, SUPER}; +use kernel::sched::job::Job; pub use kernel::frames::Discarded; use super::{DirectMap, PAGE_2M}; use crate::sync::Lock; +use crate::watch::IrqWatch; use crate::MemoryMapEntry; /// Region of physical memory to exclude from the free list. @@ -40,7 +46,7 @@ impl PhysPage { impl Drop for PhysPage { fn drop(&mut self) { - FRAMES.lock().as_mut().expect("pmm: a page outlived no allocator").free_super(self.phys, &pins()); + with(|f| f.free_super(self.phys, &pins())); } } @@ -56,57 +62,97 @@ impl Frame { DirectMap::from_phys(self.phys) } - /// Give the frame's ownership to whatever records `phys`, which must - /// return it through [`Frame::from_raw`]. + /// Give the frame's ownership to whatever records `phys`: a page + /// committed to a job, which returns it through [`free_drawn`]. pub fn into_raw(self) -> u64 { core::mem::ManuallyDrop::new(self).phys } - - /// # Safety - /// `phys` is a frame [`Frame::into_raw`] gave up, taken back once. - pub unsafe fn from_raw(phys: u64) -> Self { - Self { phys } - } } impl Drop for Frame { fn drop(&mut self) { - FRAMES.lock().as_mut().expect("pmm: a frame outlived no allocator").free_frame(self.phys, &pins()); + with(|f| f.free_frame(self.phys, &pins())); } } static FRAMES: Lock>> = Lock::new(None); -/// Give back `frames`, which leaves the space owned, under one hold of the -/// allocator's lock. +/// Posted once memory a refused job waits for has come back. +static RETURNED: IrqWatch = IrqWatch::new(); + +/// Run `f` on the allocator, and post [`RETURNED`] once its lock is let go +/// if what `f` gave back met a waiter's want. The post frees nothing, so it +/// may be made wherever memory is given back. +fn with(f: impl FnOnce(&mut Frames<'static>) -> R) -> R { + try_with(|frames| Some(f(frames))).expect("pmm: memory reached before the allocator") +} + +/// [`with`], answering `None` before the allocator exists. +fn try_with(f: impl FnOnce(&mut Frames<'static>) -> Option) -> Option { + let (answer, returned) = { + let mut guard = FRAMES.lock(); + let frames = guard.as_mut()?; + let answer = f(frames); + (answer, frames.take_returned()) + }; + if returned { + RETURNED.post_in_place(); + } + answer +} + +/// What a job waiting for memory to come back registers on. +pub fn returned() -> &'static IrqWatch { + &RETURNED +} + +/// Whether what `job` lacked when it was last refused is free for it now; +/// if not, [`returned`] is posted once it is. +pub fn ready(job: &Job) -> bool { + with(|f| f.ready(job.memory())) +} + +/// Hold `bytes` back for `job`, which has none, however short every other +/// job runs: `AlreadyExists` where it has one, and `ResourceExhausted`, +/// nothing changed, where what is free now cannot hold it. +pub fn hold_floor(job: &Job, bytes: u64) -> Result<(), toyos_abi::syscall::SyscallError> { + use toyos_abi::syscall::SyscallError; + with(|f| match job.memory().usage().floor { + 0 if f.set_floor(job.memory(), bytes.div_ceil(FRAME)) => Ok(()), + 0 => Err(SyscallError::ResourceExhausted), + _ => Err(SyscallError::AlreadyExists), + }) +} + +/// Let go of what [`hold_floor`] held back for `job`. +pub fn release_floor(job: &Job) { + let set = with(|f| f.set_floor(job.memory(), 0)); + assert!(set, "pmm: a floor let go of was refused"); +} + +/// Give back `frames` drawn for pages `job` committed and no longer does, +/// under one hold of the allocator's lock. /// /// # Safety /// Each is a frame [`Frame::into_raw`] gave up, that no leaf on any CPU maps /// any more and nothing else frees. -pub unsafe fn free_raw(frames: &[u64]) { +pub unsafe fn free_drawn(job: &Job, frames: &[u64]) { let pins = pins(); - let mut guard = FRAMES.lock(); - let allocator = guard.as_mut().expect("pmm: a frame outlived no allocator"); - for &phys in frames { - allocator.free_frame(phys, &pins); - } + with(|f| f.free_drawn(job.memory(), frames.iter().copied(), &pins)); } -/// Give back every frame of `frames` under one hold of the allocator's lock. -pub fn free_all(frames: alloc::boxed::Box<[Frame]>) { - let mut frames = frames.into_vec(); - { - let pins = pins(); - let mut guard = FRAMES.lock(); - let allocator = guard.as_mut().expect("pmm: a frame outlived no allocator"); - for frame in frames.drain(..) { - allocator.free_frame(frame.into_raw(), &pins); - } +/// Give back every frame of `frames`, drawn for pages `job` committed, under +/// one hold of the allocator's lock. +pub fn free_all(job: &Job, frames: alloc::boxed::Box<[Frame]>) { + let pins = pins(); + with(|f| f.free_drawn(job.memory(), frames.iter().map(|frame| frame.phys), &pins)); + // Given up only once the allocator's lock is let go: the list goes back + // to the heap, which may give a page back through it. + for frame in frames.into_vec() { + let _ = frame.into_raw(); } - // The list goes back to the heap only once the allocator's lock is let - // go, since the heap may give a page back through it. - drop(frames); } + /// The pin counters, published once by [`init`]: where they are, how many, /// and the physical address the first one counts for. static PINS: (AtomicPtr, AtomicUsize, AtomicU64) = @@ -215,30 +261,30 @@ fn kept_supers(frames: &Frames) -> u64 { (frames.usable_frames() / 256).max((16 << 20) / FRAME).div_ceil(SUPER / FRAME) } -/// Promise `frames` to a commitment, which draws each with +/// Promise `frames` to a commitment of `job`'s, which draws each with /// [`alloc_promised`] when it is first touched; `false`, and nothing -/// promised, where that would take what the kernel holds back. -pub fn promise(frames: u64) -> bool { - FRAMES.lock().as_mut().is_some_and(|f| f.promise(frames)) +/// promised, where that would take what is held back. +pub fn promise(job: &Job, frames: u64) -> bool { + frames == 0 || with(|f| f.promise(job.memory(), frames)) } -/// Charge `bytes` of the kernel heap to what a process caused, before it is -/// allocated; `false`, and nothing charged, where covering them would take -/// what the kernel holds back. The heap grows into what charges reserve -/// first, so an allocation a charge covers finds its room. -pub fn charge(bytes: u64) -> bool { - FRAMES.lock().as_mut().is_some_and(|f| f.charge(bytes)) +/// Charge `bytes` more of the kernel heap to `job`, into `held`, before they +/// are allocated; `false`, and nothing charged, where covering them would +/// take what is held back. The heap grows into what charges reserve first, so +/// an allocation a charge covers finds its room. +pub fn charge(job: &Job, held: &mut PoolSplit, bytes: u64) -> bool { + with(|f| f.grow(job.memory(), held, bytes)) } -/// Give back a [`charge`]. -pub fn uncharge(bytes: u64) { - FRAMES.lock().as_mut().expect("pmm: a charge outlived no allocator").uncharge(bytes); +/// Give back `bytes` of a [`charge`]. +pub fn uncharge(job: &Job, held: &mut PoolSplit, bytes: u64) { + with(|f| f.shrink(job.memory(), held, bytes)); } -/// Take back promises no commitment holds any more. -pub fn unpromise(frames: u64) { +/// Take back promises none of `job`'s commitments holds any more. +pub fn unpromise(job: &Job, frames: u64) { if frames > 0 { - FRAMES.lock().as_mut().expect("pmm: a promise outlived no allocator").unpromise(frames); + with(|f| f.unpromise(job.memory(), frames)); } } @@ -249,38 +295,40 @@ pub fn unpromise(frames: u64) { /// `phys` is an owned leaf's frame, which no leaf on any CPU maps any more, /// unless this answers [`Discarded::Kept`]. pub unsafe fn discard(phys: u64) -> Discarded { - FRAMES.lock().as_mut().expect("pmm: a frame outlived no allocator").discard(phys, &pins()) + let pins = pins(); + with(|f| f.discard(phys, &pins)) } /// A zeroed frame a promise held, which the promise guarantees is free. pub fn alloc_promised() -> Frame { - let phys = FRAMES.lock().as_mut().expect("pmm: a promise outlived no allocator").alloc_promised(); - let frame = Frame { phys }; + let frame = Frame { phys: with(Frames::alloc_promised) }; // SAFETY: just taken off the allocator, so unaliased, and inside the direct map. unsafe { core::ptr::write_bytes(frame.direct_map().as_mut_ptr::(), 0, FRAME as usize) }; frame } -/// A zeroed frame for a page table a process causes, refused rather than -/// take what the kernel holds back: a process's tables are not promised. -pub fn alloc_table_frame() -> Option { - let phys = FRAMES.lock().as_mut()?.alloc_frame_for_process()?; - let frame = Frame { phys }; +/// A zeroed frame for a page table of `job`'s, refused rather than take what +/// is held back: a process's tables are not promised. It goes back through +/// [`free_table_frame`]. +pub fn alloc_table_frame(job: &Job) -> Option { + let phys = with(|f| f.alloc_table(job.memory()))?; // SAFETY: just taken off the allocator, so unaliased, and inside the direct map. - unsafe { core::ptr::write_bytes(frame.direct_map().as_mut_ptr::(), 0, FRAME as usize) }; - Some(frame) + unsafe { core::ptr::write_bytes(DirectMap::from_phys(phys).as_mut_ptr::(), 0, FRAME as usize) }; + Some(phys) } -/// One zeroed 2 MiB page for the kernel's own use. -pub fn alloc_page() -> Option { - let phys = FRAMES.lock().as_mut()?.alloc_super()?; - Some(zeroed(PhysPage { phys })) +/// Give back a frame [`alloc_table_frame`] took for `job`. +/// +/// # Safety +/// No table links it any more, and no CPU can still walk into it. +pub unsafe fn free_table_frame(job: &Job, phys: u64) { + let pins = pins(); + with(|f| f.free_table(job.memory(), phys, &pins)); } -/// One zeroed 2 MiB page a process causes, such as a pipe's ring: refused -/// rather than take what the kernel holds back. -pub fn alloc_page_for_process() -> Option { - let phys = FRAMES.lock().as_mut()?.alloc_run_for_process(1)?; +/// One zeroed 2 MiB page for the kernel's own use. +pub fn alloc_page() -> Option { + let phys = with(Frames::alloc_super)?; Some(zeroed(PhysPage { phys })) } @@ -290,6 +338,46 @@ fn zeroed(page: PhysPage) -> PhysPage { page } +/// Physically contiguous 2 MiB pages a job holds, such as a pipe's ring or +/// memory a device is lent; they go back, counted to it, when this drops. +pub struct Run { + base: u64, + count: usize, + job: Arc, +} + +impl Run { + /// `count` zeroed pages in a row for `job`, refused rather than take what + /// is held back; `None` for none, which a caller's bogus size and a + /// legitimate zero would both ask. + pub fn new(job: &Arc, count: usize) -> Option { + if count == 0 { + return None; + } + let base = with(|f| f.alloc_run_for(job.memory(), count))?; + // SAFETY: every superframe of the run was just taken off the allocator, + // so it is unaliased and within the direct map. + unsafe { core::ptr::write_bytes(DirectMap::from_phys(base).as_mut_ptr::(), 0, count * PAGE_2M as usize) }; + Some(Self { base, count, job: Arc::clone(job) }) + } + + pub fn direct_map(&self) -> DirectMap { + DirectMap::from_phys(self.base) + } + + /// Its pages. + pub fn count(&self) -> usize { + self.count + } +} + +impl Drop for Run { + fn drop(&mut self) { + let pins = pins(); + with(|f| f.free_run_for(self.job.memory(), self.base, self.count, &pins)); + } +} + /// A 2 MiB superframe the kernel heap holds, holding what its last owner left /// in it: the heap answers uninitialized memory, and `alloc_zeroed` writes its /// own zeros. It goes back when it drops. @@ -319,41 +407,26 @@ impl HeapPage { impl Drop for HeapPage { fn drop(&mut self) { - FRAMES.lock().as_mut().expect("pmm: a heap page outlived no allocator").free_heap(self.phys, &pins()); + with(|f| f.free_heap(self.phys, &pins())); } } /// One zeroed 4 KiB frame. pub fn alloc_frame() -> Option { - let phys = FRAMES.lock().as_mut()?.alloc_frame()?; + // Through `try_with`: a frame of a kept superframe frees the rest of it + // to the room, which may be what a waiter lacks. + let phys = try_with(Frames::alloc_frame)?; let frame = Frame { phys }; // SAFETY: just taken off the allocator, so unaliased, and inside the direct map. unsafe { core::ptr::write_bytes(frame.direct_map().as_mut_ptr::(), 0, FRAME as usize) }; Some(frame) } -/// `count` physically contiguous zeroed 2 MiB pages a process causes, such as -/// memory a device is lent: refused rather than take what the kernel holds -/// back. -pub fn alloc_contiguous(count: usize) -> Option> { - // `count` comes from userland, so a bogus 0 and a legitimate `mmap(0)` can't be told apart here — refuse, don't assert. - if count == 0 { - return None; - } - let base = FRAMES.lock().as_mut()?.alloc_run_for_process(count)?; - Some(run(base, count)) -} - /// `count` physically contiguous zeroed 2 MiB pages for the kernel's own use, /// such as a kernel driver's DMA pool. pub fn alloc_contiguous_for_kernel(count: usize) -> Option> { assert!(count > 0, "pmm: a kernel run of no pages"); let base = FRAMES.lock().as_mut()?.alloc_run(count)?; - Some(run(base, count)) -} - -/// The pages of a run just taken at `base`, zeroed. -fn run(base: u64, count: usize) -> alloc::vec::Vec { let mut pages = alloc::vec::Vec::with_capacity(count); for i in 0..count as u64 { let page = PhysPage { phys: base + i * PAGE_2M }; @@ -361,7 +434,7 @@ fn run(base: u64, count: usize) -> alloc::vec::Vec { unsafe { core::ptr::write_bytes(page.direct_map().as_mut_ptr::(), 0, PAGE_2M as usize) }; pages.push(page); } - pages + Some(pages) } /// Pin every managed frame `[phys, phys + len)` touches so none is reissued while a user-copy window covers it; `false`, and nothing pinned, on a pin-count overflow. @@ -385,7 +458,7 @@ pub fn unpin_range(phys: u64, len: usize) { fn unpin(pins: &Pins, frame: u64) { if pins.unpin(frame) == Unpinned::Release { - FRAMES.lock().as_mut().expect("pmm: a pinned frame outlived no allocator").release(frame); + with(|f| f.release(frame)); } } diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 2d6c069e69e..3cc57b27c45 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -26,6 +26,8 @@ use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; use kernel::vm::{self, Access, Entry, Kind, Prot}; use toyos_abi::syscall::SyscallError; +use kernel::sched::job::Job; + use super::charge::{self, Charge}; use super::paging::{Leaf, Mapping, Root, Unlinked, UserTables}; use super::policy::{self, CachePolicy}; @@ -155,16 +157,18 @@ pub struct Space { pub type PageTables = Arc; impl Space { - /// A new user space holding nothing, or `None` when every tag the - /// architecture has for one is held by a live space. - pub fn new_user() -> Option { + /// A new user space holding nothing, whose memory is counted to `job`, + /// or `None` when every tag the architecture has for one is held by a + /// live space or no frame is left for its root. + pub fn new_user(job: &Arc) -> Option { let usage = Arc::new(Usage::default()); let space = AddressSpace { inventory: Inventory::new(FLOOR, TOP), - tables: UserTables::new()?, + tables: UserTables::new(job)?, usage: Arc::clone(&usage), kernel: false, - ranges: Charge::NONE, + ranges: Charge::none(job), + job: Arc::clone(job), }; Some(Arc::new(Self { inner: Lock::new(space), usage })) } @@ -201,7 +205,8 @@ pub(super) fn init_kernel() { tables: UserTables::kernel(), usage: Arc::clone(&usage), kernel: true, - ranges: Charge::NONE, + ranges: Charge::none(charge::boot()), + job: Arc::clone(charge::boot()), }; // Leaked: it outlives every task, and its tables are the kernel's. let space: &'static PageTables = alloc::boxed::Box::leak(alloc::boxed::Box::new(Arc::new(Space { inner: Lock::new(space), usage }))); @@ -223,6 +228,8 @@ pub struct AddressSpace { kernel: bool, /// What its ranges take of the heap, in grains of [`RANGE_GRAIN`]. ranges: Charge, + /// The job its memory is counted to: its process's. + job: Arc, } /// Ranges a space is charged for at a time, so that a change adding one @@ -260,6 +267,11 @@ impl AddressSpace { self.tables.root() } + /// The job its memory is counted to. + pub fn job(&self) -> &Arc { + &self.job + } + /// Where a range of `len` bytes goes, changing nothing: a `Fixed` one is /// put in place of what is there by [`put`](Self::put). `len` is whole /// pages. @@ -300,7 +312,7 @@ impl AddressSpace { .sum(); self.inventory.insert(start, entry); let freed = self.clear(start, end); - pmm::unpromise(committed - freed); + pmm::unpromise(&self.job, committed - freed); self.usage.taken.fetch_add(1, Relaxed); // The objects go here, after every CPU has stopped reaching them. drop(removed); @@ -373,11 +385,14 @@ impl AddressSpace { let start = self.place(len, at)?; let entry = Entry { end: start + len, prot, kind }; let promised = if entry.committed() { len / PAGE_SIZE } else { 0 }; - if !pmm::promise(promised) { + if !pmm::promise(&self.job, promised) { return Err(SyscallError::ResourceExhausted); } if entry.backed() { - self.build(start, start + len).inspect_err(|_| pmm::unpromise(promised))?; + if let Err(e) = self.build(start, start + len) { + pmm::unpromise(&self.job, promised); + return Err(e); + } } self.put(start, entry); Ok(UserAddr::new(start)) @@ -395,7 +410,7 @@ impl AddressSpace { if !unlinked.is_empty() { self.tables.shoot_down(start, end, &unlinked); } - unlinked.free(); + self.tables.free(unlinked); Err(SyscallError::ResourceExhausted) } @@ -450,7 +465,7 @@ impl AddressSpace { .map(|(s, entry)| (entry.end - s) / PAGE_SIZE) .sum(); let freed = self.clear(start, end); - pmm::unpromise(committed - freed); + pmm::unpromise(&self.job, committed - freed); self.usage.taken.fetch_add(1, Relaxed); // The objects go here, after every CPU has stopped reaching them. drop(removed); @@ -479,11 +494,14 @@ impl AddressSpace { Prot::None => 0, _ => self.pages_where(start, end, |kind| matches!(kind, Kind::Private { committed: false })), }; - if !pmm::promise(commits) { + if !pmm::promise(&self.job, commits) { return Err(SyscallError::ResourceExhausted); } if commits > 0 { - self.build(start, end).inspect_err(|_| pmm::unpromise(commits))?; + if let Err(e) = self.build(start, end) { + pmm::unpromise(&self.job, commits); + return Err(e); + } } self.inventory.update(start, end, |_, entry| { entry.prot = prot; @@ -587,7 +605,7 @@ impl AddressSpace { entry.kind = Kind::Private { committed: false }; }); let freed = self.clear(start, end); - pmm::unpromise(committed - freed); + pmm::unpromise(&self.job, committed - freed); self.settle(); Ok(()) } @@ -665,7 +683,7 @@ impl AddressSpace { crate::sched::futex::revoke_frames(frames); // SAFETY: frames owned leaves gave up, which no leaf maps on any // CPU now and no waiter is keyed on. - unsafe { pmm::free_raw(frames) }; + unsafe { pmm::free_drawn(&self.job, frames) }; freed += frames.len() as u64; }); let inventory = &self.inventory; @@ -673,7 +691,7 @@ impl AddressSpace { if !unlinked.is_empty() { self.shoot_down(start, end, &unlinked); } - unlinked.free(); + self.tables.free(unlinked); self.usage.freed(freed); freed } @@ -847,10 +865,10 @@ impl Drop for AddressSpace { let mut freed = 0; self.tables.take_owned(FLOOR, TOP, |frames| { // SAFETY: the space's own frames, which nothing maps once it drops. - unsafe { pmm::free_raw(frames) }; + unsafe { pmm::free_drawn(&self.job, frames) }; freed += frames.len() as u64; }); self.usage.freed(freed); - pmm::unpromise(committed - freed); + pmm::unpromise(&self.job, committed - freed); } } diff --git a/kernel/src/object/device.rs b/kernel/src/object/device.rs index b0e5df3789f..1804df94258 100644 --- a/kernel/src/object/device.rs +++ b/kernel/src/object/device.rs @@ -160,9 +160,10 @@ pub struct ConsoleObject { } impl ConsoleObject { - pub fn new() -> Result, Exhausted> { + /// A console of its own for a holder in `job`, charged to it. + pub fn new(job: &Arc) -> Result, Exhausted> { Ok(Arc::new(Self { - core: Self::new_core(0)?, + core: Self::new_core_for(job, 0)?, line: crate::sync::Lock::new(crate::drivers::serial::ConsoleLine::new()), })) } diff --git a/kernel/src/object/handle.rs b/kernel/src/object/handle.rs index 5c3dc2535a2..fb76692f0f5 100644 --- a/kernel/src/object/handle.rs +++ b/kernel/src/object/handle.rs @@ -181,8 +181,14 @@ pub struct HandleTable { } impl HandleTable { - pub const fn new() -> Self { - Self { slots: Vec::new(), free: Vec::new(), charge: Charge::NONE } + /// An empty table, its storage charged to `job` as it grows. + pub fn new(job: &Arc) -> Self { + Self { slots: Vec::new(), free: Vec::new(), charge: Charge::none(job) } + } + + /// The job its storage is charged to. + pub fn job(&self) -> &Arc { + self.charge.job() } /// Make sure `n` more `install`s all succeed: `false`, with nothing diff --git a/kernel/src/object/job.rs b/kernel/src/object/job.rs index 6478cb3f047..0f7128236ae 100644 --- a/kernel/src/object/job.rs +++ b/kernel/src/object/job.rs @@ -1,17 +1,26 @@ //! A handle to a job: the owner a process's threads, and its children's unless -//! a spawn names another, share CPU time under (`kernel::sched::job`). A spawn +//! a spawn names another, share CPU time under (`kernel::sched::job`), and the +//! account their memory is counted to (`kernel::frames::Ledger`). A spawn //! that endows one under `JOB_LABEL` with `WRITE` puts its child in it. //! -//! The job lives while a thread of it runs or a handle to it is held, and -//! not past both: its launcher ending takes nothing from it. +//! The job lives while a thread of it runs, a handle to it is held or an +//! object charged to it lives, and not past all three: its launcher ending +//! takes nothing from it. +//! +//! **A floor lasts as long as the object it was set through**: a `SysCap` +//! holding `FLOOR` sets one through this object, and its last handle's +//! going lets the floor go. A readable poll answers once memory the job was +//! refused is free for it (`pmm::ready`). use alloc::sync::Arc; -use core::sync::atomic::{AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::sched::job::{Job, JobKey}; use toyos_abi::handle::Rights; +use toyos_abi::syscall::{AccountInfo, SyscallError}; use crate::mm::charge::Exhausted; +use crate::mm::{pmm, PAGE_2M, PAGE_SIZE}; use super::{KObjectVariant, ObjectCore}; @@ -19,21 +28,64 @@ pub struct JobObject { /// Visible to `object`, where `kobject!` generates this type's `core()`. pub(super) core: ObjectCore, job: Arc, + /// Whether the job's floor was set through this object, which then holds it. + floor: AtomicBool, } impl JobObject { /// What a handle the kernel mints to a job carries: `WRITE` to place a - /// child in it, `DUP` and `TRANSFER` to hand that on. - pub const RIGHTS: Rights = Rights::WRITE.union(Rights::DUP).union(Rights::TRANSFER); + /// child in it or set its floor, `READ` to read its account, `WAIT` to + /// wait for memory it was refused, `DUP` and `TRANSFER` to hand that on. + pub const RIGHTS: Rights = + Rights::WRITE.union(Rights::READ).union(Rights::WAIT).union(Rights::DUP).union(Rights::TRANSFER); /// Charged the job besides, which a job a thread runs in can outlive. pub fn new(job: Arc) -> Result, Exhausted> { - Ok(Arc::new(Self { core: Self::new_core(crate::mm::charge::arc::())?, job })) + Ok(Arc::new(Self { core: Self::new_core(crate::mm::charge::arc::())?, job, floor: AtomicBool::new(false) })) } pub fn job(&self) -> &Arc { &self.job } + + /// Hold `bytes` back for the job for as long as this object lives: + /// `InvalidArgument` for none, `AlreadyExists` where the job has a + /// floor, and `ResourceExhausted` where what is free cannot hold it. + pub fn set_floor(&self, bytes: u64) -> Result<(), SyscallError> { + if bytes == 0 { + return Err(SyscallError::InvalidArgument); + } + pmm::hold_floor(&self.job, bytes)?; + self.floor.store(true, Ordering::Release); + Ok(()) + } + + /// What the job's account holds. + pub fn account(&self) -> AccountInfo { + let u = self.job.memory().usage(); + let pages = |n: u64| n.saturating_mul(PAGE_SIZE); + AccountInfo { + floor: pages(u.floor), + committed: pages(u.committed), + kernel: u.heap.saturating_add(pages(u.tables)).saturating_add(u.runs.saturating_mul(PAGE_2M)), + peak: pages(u.peak), + wanted: pages(u.wanted), + refusals: u.refusals, + } + } + + /// Whether memory the job was refused is free for it now. + pub fn ready(&self) -> bool { + pmm::ready(&self.job) + } +} + +impl Drop for JobObject { + fn drop(&mut self) { + if *self.floor.get_mut() { + pmm::release_floor(&self.job); + } + } } /// A job no thread has joined yet. diff --git a/kernel/src/object/mod.rs b/kernel/src/object/mod.rs index 47c664fdb5b..80cf0e199c0 100644 --- a/kernel/src/object/mod.rs +++ b/kernel/src/object/mod.rs @@ -106,6 +106,12 @@ impl ObjectCore { self.retired.load(Ordering::Acquire) } + /// The job the object is charged to: the caller's that made it, or the + /// one it was made for ([`KObjectVariant::new_core_for`]). + pub fn job(&self) -> &Arc { + self._charge.job() + } + fn enrol(live: &'static AtomicU64, charge: Charge) -> Self { live.fetch_add(1, Ordering::Relaxed); Self { @@ -139,9 +145,15 @@ pub trait KObjectVariant: ZeroHandles + Send + Sync + Sized + 'static { fn from_ref(r: &KObjectRef) -> Option<&Arc>; fn core(&self) -> &ObjectCore; /// A fresh core, enrolled in this type's census — the only way to build - /// one — charged the object's own `Arc` and the `extra` heap bytes its - /// constructor keeps besides; refused before anything is allocated. - fn new_core(extra: usize) -> Result; + /// one — charged to the caller's job the object's own `Arc` and the + /// `extra` heap bytes its constructor keeps besides; refused before + /// anything is allocated. + fn new_core(extra: usize) -> Result { + Self::new_core_for(&charge::current(), extra) + } + + /// [`new_core`](Self::new_core), charged to `job`. + fn new_core_for(job: &Arc, extra: usize) -> Result; } /// Declares the closed set of object types; each row says whether its last @@ -202,8 +214,8 @@ macro_rules! kobject { &self.core } - fn new_core(extra: usize) -> Result { - let charge = Charge::new(charge::arc::<$ty>().saturating_add(extra))?; + fn new_core_for(job: &Arc, extra: usize) -> Result { + let charge = Charge::new(job, charge::arc::<$ty>().saturating_add(extra))?; Ok(ObjectCore::enrol(&census::$variant, charge)) } } diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 457b85b390c..80cbb05c9de 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -192,7 +192,8 @@ pub fn close(table: &mut HandleTable, h: RawHandle) -> Result<(), super::HandleE /// Release every handle a process holds; runs on the kill path too, which /// does not unwind, and allocates nothing. pub fn close_all(table: &mut HandleTable) { - drop(core::mem::replace(table, HandleTable::new())); + let job = Arc::clone(table.job()); + drop(core::mem::replace(table, HandleTable::new(&job))); } /// The pipe a read of `object` took from, held past the handle table's lock @@ -240,6 +241,8 @@ pub fn pipe_write(object: &KObjectRef) -> Option<(&PipeRef, WaitClass, Landing)> /// share of one that goes with its object. pub enum WatchRef { Static(&'static Watch), + /// A static watch posted where nothing may be freed, as `pmm::returned` is. + InPlace(&'static crate::watch::IrqWatch), Shared(Arc), /// The watch of the function or row a claim holds: named by the claim, /// and reached only with what the claim lends. @@ -250,6 +253,7 @@ impl WatchRef { pub(crate) fn add_poll(&self, entry: PollEntry) { match self { Self::Static(watch) => watch.add_poll(entry), + Self::InPlace(watch) => watch.add_poll(entry), Self::Shared(watch) => watch.add_poll(entry), Self::Claim(claim) => claim.add_poll(entry), } @@ -258,6 +262,7 @@ impl WatchRef { pub fn cancel_polls(&self) { match self { Self::Static(watch) => watch.cancel_polls(), + Self::InPlace(watch) => watch.cancel_polls(), Self::Shared(watch) => watch.cancel_polls(), // Its release answers them, and its close ends none (`close_ends_polls`). Self::Claim(_) => unreachable!("a claim's polls are cancelled by its release alone"), @@ -274,6 +279,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { KObjectRef::Acceptor(a) => Some(WatchRef::Shared(a.watch().clone())), KObjectRef::Process(p) => Some(WatchRef::Shared(p.watch().clone())), KObjectRef::Notice(n) => Some(WatchRef::Shared(n.watch().clone())), + KObjectRef::Job(_) => Some(WatchRef::InPlace(crate::mm::pmm::returned())), // Nothing posts for a serial byte, so a console's readable poll has // no source (`issues/a-console-watch-waits-on-the-keyboard-not-the-serial-line.md`). KObjectRef::Console(_) => None, @@ -287,7 +293,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::Namespace(_) - | KObjectRef::SharedMem(_) | KObjectRef::Null(_) | KObjectRef::Job(_) => None, + | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, } } @@ -724,7 +730,7 @@ pub fn has_data(object: &KObjectRef) -> bool { KObjectRef::Console(_) => serial::has_data(), KObjectRef::Acceptor(a) => a.has_pending(), KObjectRef::Process(p) => p.finished(), - KObjectRef::Job(_) => false, + KObjectRef::Job(j) => j.ready(), KObjectRef::Notice(n) => n.readable(), KObjectRef::File(_) | KObjectRef::Null(_) => true, KObjectRef::Device(d) => match d.class() { diff --git a/kernel/src/object/service.rs b/kernel/src/object/service.rs index 8dc486cd90f..2ee268e87b5 100644 --- a/kernel/src/object/service.rs +++ b/kernel/src/object/service.rs @@ -44,7 +44,7 @@ impl HandleQueue { let bound = charge::arc::() + charge::array::<(u64, Vec)>(MAX_QUEUED_BATCHES.next_power_of_two()) + MAX_QUEUED_BATCHES * charge::array::(MAX_TRANSFER_HANDLES); - let charge = Charge::new(bound)?; + let charge = Charge::current(bound)?; Ok(Arc::new(Self { batches: Lock::new(Some(Batches::new())), _charge: charge })) } diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index 49177a00e89..7d4be235696 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -17,11 +17,13 @@ use alloc::boxed::Box; use alloc::sync::{Arc, Weak}; use alloc::vec::Vec; +use kernel::sched::job::Job; + use toyos_abi::syscall::SyscallError; use crate::mm::charge::{self, Charge, Exhausted}; use crate::mm::policy::CachePolicy; -use crate::mm::pmm::{self, Frame, PhysPage}; +use crate::mm::pmm::{self, Frame, Run}; use crate::mm::space::Space; use crate::mm::{PAGE_2M, PAGE_SIZE}; use crate::{DirectMap, UserAddr}; @@ -42,7 +44,7 @@ enum Backing { Frames(Vec>), /// 2 MiB pages of its own in one physical run: what a device may be lent /// or granted, and what a kernel-built image addresses as one slice. - Run(Vec), + Run(Run), /// A payload the loader verified and handed over at `phys`, `len` bytes /// of the object's pages: kept for the machine's life and written by /// nobody, so every mapping of it is read-only (`crate::bootimage`). @@ -71,17 +73,17 @@ impl SharedMemObject { Arc::new(Self { core, backing, size, cache }) } - /// `size` bytes of zeroed memory in whole pages, frame by frame; - /// `InvalidArgument` for no bytes or a size no page count holds, - /// `ResourceExhausted` where frames run out. - pub fn create(size: u64) -> Result, SyscallError> { + /// `size` bytes of zeroed memory in whole pages, frame by frame, counted + /// to `job`; `InvalidArgument` for no bytes or a size no page count + /// holds, `ResourceExhausted` where frames run out. + pub fn create(job: &Arc, size: u64) -> Result, SyscallError> { let pages = pages(size)?; let (count, full, rest) = (pages.div_ceil(CHUNK as u64), pages / CHUNK as u64, (pages % CHUNK as u64) as usize); let lists = charge::array::(CHUNK).saturating_mul(full as usize).saturating_add(charge::array::(rest)); - let core = Self::new_core(charge::array::>(count as usize).saturating_add(lists))?; + let core = Self::new_core_for(job, charge::array::>(count as usize).saturating_add(lists))?; // Promised whole before any is taken: frames one at a time until none // is left would starve every other allocation meanwhile. - if !pmm::promise(pages) { + if !pmm::promise(job, pages) { return Err(SyscallError::ResourceExhausted); } let mut pieces = Vec::with_capacity(count as usize); @@ -101,13 +103,14 @@ impl SharedMemObject { /// `size` bytes of zeroed memory in whole pages, over one run of whole /// 2 MiB pages: what a device can be lent, since a domain maps 2 MiB at a /// time and a device with no domain reaches memory by its physical address. - pub fn create_run(size: u64) -> Result, SyscallError> { + /// Counted to `job`. + pub fn create_run(job: &Arc, size: u64) -> Result, SyscallError> { let bytes = pages(size)? * PAGE_SIZE; let run = crate::mm::align_2m_checked(bytes).ok_or(SyscallError::InvalidArgument)?; let count = (run / PAGE_2M) as usize; - let core = Self::new_core(charge::array::(count))?; - let pages = pmm::alloc_contiguous(count).ok_or(SyscallError::ResourceExhausted)?; - Ok(Self::new(core, Backing::Run(pages), bytes, CachePolicy::Normal)) + let core = Self::new_core_for(job, 0)?; + let run = Run::new(job, count).ok_or(SyscallError::ResourceExhausted)?; + Ok(Self::new(core, Backing::Run(run), bytes, CachePolicy::Normal)) } /// An aperture over `size` bytes of memory somebody else owns at `phys`, @@ -118,7 +121,7 @@ impl SharedMemObject { (phys | size) & (PAGE_SIZE - 1) == 0 && size != 0, "shm: an aperture of {size:#x} bytes at {phys:#x} is not whole 4 KiB pages" ); - let mapped = Held::new((Vec::new(), Charge::NONE)); + let mapped = Held::new((Vec::new(), Charge::none(&charge::current()))); Ok(Self::new(Self::new_core(0)?, Backing::Aperture { phys, mapped }, size, cache)) } @@ -201,7 +204,7 @@ impl SharedMemObject { /// included: one set of pages is never lent under two objects. pub fn run(&self) -> Option<(u64, u64)> { match &self.backing { - Backing::Run(pages) => Some((pages[0].direct_map().phys(), pages.len() as u64 * PAGE_2M)), + Backing::Run(run) => Some((run.direct_map().phys(), run.count() as u64 * PAGE_2M)), Backing::Frames(_) | Backing::Boot { .. } | Backing::Aperture { .. } | Backing::Sub { .. } => None, } } @@ -215,7 +218,7 @@ impl SharedMemObject { let frame = (page / PAGE_SIZE) as usize; pieces[frame / CHUNK][frame % CHUNK].direct_map().phys() } - Backing::Run(pages) => pages[0].direct_map().phys() + page, + Backing::Run(run) => run.direct_map().phys() + page, Backing::Boot { phys, .. } | Backing::Aperture { phys, .. } => phys + page, Backing::Sub { whole, offset, .. } => whole.page(offset + page), } @@ -303,11 +306,11 @@ impl Drop for SharedMemObject { for mut piece in pieces.drain(..) { piece.sort_unstable_by_key(|frame| frame.direct_map().phys()); crate::sched::futex::revoke_sorted(&piece, |frame| frame.direct_map().phys()); - pmm::free_all(piece); + pmm::free_all(self.core.job(), piece); } } - Backing::Run(pages) => { - crate::sched::futex::revoke_run(pages[0].direct_map().phys(), pages.len() as u64 * PAGE_2M); + Backing::Run(run) => { + crate::sched::futex::revoke_run(run.direct_map().phys(), run.count() as u64 * PAGE_2M); } Backing::Boot { .. } | Backing::Aperture { .. } | Backing::Sub { .. } => {} } diff --git a/kernel/src/pcidev/mod.rs b/kernel/src/pcidev/mod.rs index f03e0e41a17..31992b54a08 100644 --- a/kernel/src/pcidev/mod.rs +++ b/kernel/src/pcidev/mod.rs @@ -1384,7 +1384,7 @@ pub fn dma_alloc( if held + span > MAX_GRANT_TOTAL { return Err(SyscallError::ResourceExhausted); } - let memory = SharedMemObject::create_run(span)?; + let memory = SharedMemObject::create_run(&crate::mm::charge::current(), span)?; let (phys, _) = memory.run().expect("pcidev: a grant is one run"); // The pages are the allocator's own and 2 MiB, so a domain that cannot // take them is a kernel bug rather than the device's answer. diff --git a/kernel/src/pipe.rs b/kernel/src/pipe.rs index 3c8b9649887..548a396ec9b 100644 --- a/kernel/src/pipe.rs +++ b/kernel/src/pipe.rs @@ -3,6 +3,7 @@ use crate::mm::pmm; use kernel::message::Window; use kernel::pipe::{End, Holders, Landing, Released}; +use kernel::sched::job::Job; use toyos_abi::ring::Ring; use alloc::sync::Arc; @@ -93,8 +94,8 @@ const _: () = assert!( /// A pipe's ring page and the cursors over it, in kernel memory. struct Backing { - /// Owns the page `ring` addresses. - _page: pmm::PhysPage, + /// Owns the page `ring` addresses, counted to the job that made the pipe. + _page: pmm::Run, ring: Ring, } @@ -108,8 +109,9 @@ struct Pipe { /// end sits on it, and only the other end's last holder posts it — a byte /// moved through the pipe wakes no poll that asked about a leaving. left: [Arc; 2], - /// Its own heap and its watches. - _charge: Charge, + /// Its own heap and its watches, charged to the job that made it, which + /// its ring is counted to as well. + charge: Charge, } /// What a pipe's lock holds. @@ -127,7 +129,7 @@ struct State { read: u64, } -// SAFETY: `Backing::ring`'s raw pointer and its owning `PhysPage` move together in the same struct, and the pipe's lock is the only way to reach a `State`, serializing all access to it. +// SAFETY: `Backing::ring`'s raw pointer and its owning `pmm::Run` move together in the same struct, and the pipe's lock is the only way to reach a `State`, serializing all access to it. unsafe impl Send for State {} impl Pipe { @@ -138,11 +140,13 @@ impl Pipe { } impl State { - /// Allocate the ring page if this is the first use; `None` where the - /// kernel holds back what is left, which userland's write is refused. - fn back(&mut self) -> Option<&mut Backing> { + /// Allocate the ring page if this is the first use, counted to `job`, the + /// one that made the pipe, whoever writes first: a connection's are both + /// its connecting job's, so a server's reply spends nothing of its own. + /// `None` where what is left is held back, which the write is refused. + fn back(&mut self, job: &Arc) -> Option<&mut Backing> { if self.backing.is_none() { - let page = pmm::alloc_page_for_process()?; + let page = pmm::Run::new(job, 1)?; // SAFETY: a fresh 2 MiB page this `State` owns for as long as the `Ring` addresses it. let ring = unsafe { Ring::new(page.direct_map().as_mut_ptr(), PIPE_SIZE) }; self.backing = Some(Backing { _page: page, ring }); @@ -173,13 +177,13 @@ impl State { /// A pipe with no traffic owns no physical memory: its ring page is /// allocated at its first `write`, which is refused if none is left. pub fn create() -> Result<(PipeReader, PipeWriter), Exhausted> { - let charge = Charge::new(Pipe::heap())?; + let charge = Charge::current(Pipe::heap())?; let pipe = PipeRef(Arc::new(Pipe { state: Lock::new(State { backing: None, holders: Holders::new(), rt_boost_pending: false, written: 0, read: 0 }), read_watch: Arc::new(Watch::new()), write_watch: Arc::new(Watch::new()), left: [Arc::new(Watch::new()), Arc::new(Watch::new())], - _charge: charge, + charge, })); Ok((PipeReader::acquire(&pipe), PipeWriter::acquire(&pipe))) } @@ -259,7 +263,7 @@ impl PipeRef { if !pipe.holders.held(End::Read) { return PipeWrite::BrokenPipe; } - let Some(backing) = pipe.back() else { return PipeWrite::NoMemory }; + let Some(backing) = pipe.back(self.0.charge.job()) else { return PipeWrite::NoMemory }; let Some(n) = landing.takes(backing.ring.space(), buf.len()) else { return PipeWrite::NoRoom }; let wrote = backing.ring.write(n, |off, mut dst| buf.read_run(off, &mut dst)); assert_eq!(wrote, n, "pipe: a ring with room for a write took less of it"); diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 8f4df5fa236..99fbe533db5 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -340,7 +340,7 @@ impl ProcessData { /// charged on its own — its handles, threads, ranges and objects: its /// records, its place in the process table and among its parent's /// children, its strings at their bound, and its endowments at theirs. - pub fn charge(path: &str, env: &[u8]) -> Result { + pub fn charge(job: &Arc, path: &str, env: &[u8]) -> Result { // The process table doubles, and its control byte beside each. let entry = 2 * (core::mem::size_of::<(Pid, ProcessEntry)>() + 1); // A thread table of one thread is a whole leaf. @@ -357,7 +357,7 @@ impl ProcessData { + charge::array::(env.len()); let endowments = charge::array::(toyos_abi::syscall::MAX_ENDOWMENTS) + charge::array::(toyos_abi::syscall::MAX_LABELS_LEN); - Charge::new(records + strings + endowments) + Charge::new(job, records + strings + endowments) } } @@ -664,7 +664,7 @@ pub fn spawn_thread( // not a kernel bug. let (tls_alloc, thread_pointer, tp_offset) = block.publish(&parent_addr_space)?; - let (ks_alloc, ks_sp) = match alloc_kernel_stack(Start::Thread { entry, sp: stack_ptr, arg }) { + let (ks_alloc, ks_sp) = match alloc_kernel_stack(&job, Start::Thread { entry, sp: stack_ptr, arg }) { Some(ks) => ks, None => { tls_alloc.release(&parent_addr_space); diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index df18f57a6b2..5825912e406 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -705,6 +705,11 @@ pub fn current_shared() -> Option> { try_with_cpu(|cpu| cpu.running().map(|t| t.shared().clone())).flatten() } +/// The job the running task is in, whose account its call's memory is charged to. `None` off-task. +pub fn current_job() -> Option> { + try_with_cpu(|cpu| cpu.running().map(|t| t.job().clone())).flatten() +} + /// The running task's cross-CPU face, where its own watch lives. `None` off-task. pub fn current_handle() -> Option> { try_with_cpu(|cpu| cpu.running().map(|t| t.ext().handle.clone())).flatten() diff --git a/kernel/src/sched/kthread.rs b/kernel/src/sched/kthread.rs index be190fcbba6..4af681834a5 100644 --- a/kernel/src/sched/kthread.rs +++ b/kernel/src/sched/kthread.rs @@ -74,7 +74,9 @@ pub fn is_kernel_task(id: TaskId) -> bool { /// Start a kernel thread running `body(arg)` on its own kernel stack and return its scheduler faces. pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched { - let (stack, entry_sp) = crate::loader::alloc_kernel_stack(crate::loader::Start::Kernel { body, arg }) + // A job of its own: a kernel thread is nobody's helper. + let job = crate::object::job::new_job(); + let (stack, entry_sp) = crate::loader::alloc_kernel_stack(&job, crate::loader::Start::Kernel { body, arg }) .unwrap_or_else(|| panic!("kthread: no kernel stack for {name}")); // Before the table lock: a panic holding the process table hangs the machine. @@ -85,9 +87,6 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched let mut short = [0u8; THREAD_NAME_LEN]; let len = name.len().min(THREAD_NAME_LEN - 1); short[..len].copy_from_slice(&name.as_bytes()[..len]); - // A job of its own: a kernel thread is nobody's helper. - let job = crate::object::job::new_job(); - // One hold across insert and place: a visible pid already has its thread scheduled. let mut guard = PROCESS_TABLE.lock(); let table = guard.as_mut().expect("kthread: spawned before process::init"); @@ -127,7 +126,7 @@ pub fn spawn(name: &str, body: extern "C" fn(u64) -> !, arg: u64) -> ThreadSched /// Every field is the empty value but its job: a kernel thread has no user half. fn kernel_process_data(name: &str, job: Arc) -> ProcessData { ProcessData { - handles: crate::object::HandleTable::new(), + handles: crate::object::HandleTable::new(&job), cwd: String::from("/"), env: Vec::new(), elf: ElfInfo::none(), @@ -136,8 +135,8 @@ fn kernel_process_data(name: &str, job: Arc) -> Process spawn_ns: crate::clock::nanos_since_boot(), accounting: ProcessAccounting::default(), endowments: Endowments::empty(), + charge: crate::mm::charge::Charge::none(&job), job, - charge: crate::mm::charge::Charge::NONE, } } diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 28ac025fe00..078810ab35f 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -48,7 +48,7 @@ use super::machine::{ use super::machine::lower_sysinfo_bound; use super::proc::{ spawn_place, sys_endowments, sys_exit, sys_nanosleep, sys_process_stats, - sys_job_create, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, + sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, }; use super::vm::{ shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, @@ -188,7 +188,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> .map(|len| len.min(crate::user_ptr::MAX_USER_STR) as usize); let argv = crate::mm::charge::array::<&str>(copies[1] / 2 + 1); let past = (copies.iter().sum::() + argv).saturating_sub(2 * crate::user_ptr::MAX_USER_STR as usize); - let _copies = match crate::mm::charge::Charge::new(past) { + let _copies = match crate::mm::charge::Charge::current(past) { Ok(charge) => charge, Err(e) => return SyscallError::from(e).to_u64(), }; @@ -450,7 +450,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> }; sys_trace_read(&ctx, RawHandle(a1 as u32), UserAddr::new(a2), &mut out, a4 as usize) } - SYS_ACCEPT => sys_accept(RawHandle(a1 as u32)), + SYS_ACCEPT => sys_accept(RawHandle(a1 as u32), a2), SYS_MESSAGE_SEND => { let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; let unknown_flags = args.flags & !MESSAGE_NONBLOCK != 0; @@ -484,7 +484,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> }; sys_handle_recv(RawHandle(a1 as u32), &mut out, a3 as usize) } - SYS_SHM_CREATE => sys_shm_create(a1, a2), + SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32)), SYS_SHM_MAP => sys_shm_map(RawHandle(a1 as u32)), SYS_SHM_SUB => sys_shm_sub(RawHandle(a1 as u32), a2, a3), SYS_PORT_CREATE => sys_port_create(), @@ -622,6 +622,12 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> DA::GLOBAL_LEAF_REWRITE => crate::mm::paging::debug_global_leaf(a2), _ => SyscallError::InvalidArgument.to_u64(), }, + SYS_JOB_FLOOR => sys_job_floor(RawHandle(a1 as u32), RawHandle(a2 as u32), a3), + SYS_ACCOUNT_INFO => { + if a3 < core::mem::size_of::() as u64 { return SyscallError::InvalidArgument.to_u64(); } + let Some(addr) = UserAddr::checked(a2) else { return bad_addr }; + sys_account_info(&ctx, RawHandle(a1 as u32), addr) + }, SYS_PROCESS_STATS => { let stats_size = core::mem::size_of::() as u64; if a3 < stats_size { return SyscallError::InvalidArgument.to_u64(); } diff --git a/kernel/src/syscall/fs.rs b/kernel/src/syscall/fs.rs index a55e2c87c1e..a114e4f6dc0 100644 --- a/kernel/src/syscall/fs.rs +++ b/kernel/src/syscall/fs.rs @@ -50,7 +50,7 @@ pub(super) fn sys_stat(path: &str, follow: bool) -> Result u64 { let cwd = process::with_process_data(|d| d.cwd.clone()); // Held until the listing is let go, as this call returns. - let mut listing = crate::mm::charge::Charge::NONE; + let mut listing = crate::mm::charge::Charge::none(&crate::mm::charge::current()); let entries = match vfs::lock().list(&cwd, path, &mut listing) { Ok(e) => e, Err(e) => return e.to_u64(), diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index c510f12e0de..ffa69594c6d 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -244,7 +244,12 @@ fn connect_through(connector: &port::Connector) -> u64 { h.0 as u64 } -pub(super) fn sys_accept(h: RawHandle) -> u64 { +pub(super) fn sys_accept(h: RawHandle, flags: u64) -> u64 { + let waits = match flags { + 0 => true, + ACCEPT_NONBLOCK => false, + _ => return SyscallError::InvalidArgument.to_u64(), + }; let acceptor = match process::with_process_data(|data| { data.handles.get::(h, Rights::READ) }) { @@ -270,6 +275,9 @@ pub(super) fn sys_accept(h: RawHandle) -> u64 { if acceptor.closed() { return SyscallError::Gone.to_u64(); } + if !waits { + return SyscallError::WouldBlock.to_u64(); + } let parkable = crate::scheduler::Parkable::at_entry(); if watch::wait_until( &parkable, @@ -328,11 +336,20 @@ pub(super) fn sys_port_badge( } /// Make a shared region and hand back the one handle to it, carrying MAP, DUP and TRANSFER — naming it is what authorizes mapping it. -/// `SHM_DEVICE` asks for memory a device can be lent; any other bit is refused. -pub(super) fn sys_shm_create(size: u64, flags: u64) -> u64 { - let created = match flags { - 0 => crate::object::shm::SharedMemObject::create(size), - SHM_DEVICE => crate::object::shm::SharedMemObject::create_run(size), +/// `SHM_DEVICE` asks for memory a device can be lent, and `SHM_FOR_JOB` +/// counts it to the job `job` names rather than the caller's; any other +/// bit is refused. +pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { + let charged = match flags & SHM_FOR_JOB { + 0 => crate::mm::charge::current(), + _ => match process::with_process_data(|data| data.handles.get::(job, Rights::WRITE)) { + Ok(object) => alloc::sync::Arc::clone(object.job()), + Err(e) => return e.refuse(), + }, + }; + let created = match flags & !SHM_FOR_JOB { + 0 => crate::object::shm::SharedMemObject::create(&charged, size), + SHM_DEVICE => crate::object::shm::SharedMemObject::create_run(&charged, size), _ => return SyscallError::InvalidArgument.to_u64(), }; let object = match created { diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index 0e4674bda87..57e67efb772 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -55,7 +55,7 @@ pub(super) fn sys_spawn( parent: process::Parent, ) -> u64 { // Nothing to clean up: spawn's frame owns the child's resources on error. - match process::spawn(path, args, |own| pending.commit(own), cwd, env, image, parent) { + match process::spawn(path, args, pending, cwd, env, image, parent) { Ok((_, handle)) => u64::from(handle.0), Err(e) => e.refuse(), } @@ -259,3 +259,31 @@ pub(super) fn sys_process_stats( Err(e) => e.to_u64(), } } + +/// Hold memory back for the job a handle names, gated on a `SysCap` carrying +/// [`Rights::FLOOR`] and the job handle's `WRITE`. +pub(super) fn sys_job_floor(syscap: RawHandle, job: RawHandle, bytes: u64) -> u64 { + if let Err(e) = demand_syscap(syscap, Rights::FLOOR) { + return e.refuse(); + } + let object = match process::with_process_data(|data| data.handles.get::(job, Rights::WRITE)) { + Ok(object) => object, + Err(e) => return e.refuse(), + }; + match object.set_floor(bytes) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} + +/// Copy out what the account of the job a handle names holds, gated on its `READ`. +pub(super) fn sys_account_info(ctx: &crate::user_ptr::SyscallContext, job: RawHandle, out: UserAddr) -> u64 { + let object = match process::with_process_data(|data| data.handles.get::(job, Rights::READ)) { + Ok(object) => object, + Err(e) => return e.refuse(), + }; + match ctx.copy_out(out, &object.account()) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index 95258dfb6a4..9eaacd01ab9 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -297,8 +297,8 @@ fn tls_alloc_block(module_id: u64) -> Result { let tls_vaddr = match existing { Some(vaddr) => vaddr, None => { - let place = Charge::new(charge::map_entry::<(process::Tid, u64), (process::KernelMapping, Charge)>())?; - let block = crate::object::shm::SharedMemObject::create(tls_memsz.max(1) as u64)?; + let place = Charge::current(charge::map_entry::<(process::Tid, u64), (process::KernelMapping, Charge)>())?; + let block = crate::object::shm::SharedMemObject::create(&crate::mm::charge::current(), tls_memsz.max(1) as u64)?; if let Some(template) = &tls_template { // SAFETY: the template is a bounds-checked view of the module's // own image, which its `LoadedLib` keeps alive. diff --git a/kernel/src/tmpfs.rs b/kernel/src/tmpfs.rs index 96605090046..aded31712d5 100644 --- a/kernel/src/tmpfs.rs +++ b/kernel/src/tmpfs.rs @@ -74,7 +74,7 @@ pub struct TmpFs { /// The charge of `name` holding an entry that keeps `kept` heap bytes besides. fn name_charge(name: &str, kept: usize) -> Result { - Ok(Charge::new(charge::map_entry::() + charge::array::(name.len()) + kept)?) + Ok(Charge::current(charge::map_entry::() + charge::array::(name.len()) + kept)?) } /// The heap a file's entry keeps besides its name. diff --git a/kernel/src/vfs.rs b/kernel/src/vfs.rs index 4e342d60a31..8f98cf5c2db 100644 --- a/kernel/src/vfs.rs +++ b/kernel/src/vfs.rs @@ -181,7 +181,7 @@ fn seen_heap(name: &str) -> usize { /// The charge of a directory a map of `V` beside its path holds. fn dir_charge(path: &str) -> Result { - Ok(Charge::new(charge::map_entry::() + charge::array::(path.len()))?) + Ok(Charge::current(charge::map_entry::() + charge::array::(path.len()))?) } const _: () = assert!(core::mem::size_of::<(String, u64)>() == 32); @@ -635,7 +635,7 @@ impl Vfs { } let (fs, fs_path) = self.resolve_fs(&mount, &subdir).ok_or(SyscallError::NotFound)?; - let mut listing = Charge::NONE; + let mut listing = Charge::none(&charge::current()); let names = fs.list(&fs_path, MAX_LIST_ENTRIES, &mut listing)?; let child_prefix = format!("{fs_path}/"); let is_file = names.contains(&fs_path); diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index 07525d3838e..f0f137d34ab 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -27,6 +27,8 @@ //! state it keeps machine state of its own in `/state/` //! role a file server for ``: one process of it per role //! restart the supervisor starts it again when it ends +//! memory-floor memory held back for its job however short every other runs +//! supervisor-floor the same for the supervisor's own job //! supervisor-serve a name the supervisor serves itself //! start the supervisor starts this program at boot //! app-receive a connector every program launched from /apps holds @@ -337,6 +339,11 @@ pub struct Program { /// screen's row's at its start and any on `toyos_session::LOGIN_PORT`. /// ([`launch`]). pub login: bool, + /// Bytes held back for its job however short every other job runs: the + /// protected capacity of a control path the machine has to keep + /// answering under exhaustion, and zero for every other row. Only the + /// signed image declares one; an installed package's row never has one. + pub memory_floor: u64, } impl Program { @@ -416,6 +423,8 @@ pub struct Manifest { /// Program names, in the order `[boot] start` gave them — which orders /// nothing, because every port exists before any server runs. pub start: Vec, + /// [`Program::memory_floor`] for the supervisor's own job. + pub supervisor_floor: u64, } impl Manifest { @@ -554,6 +563,12 @@ pub fn render(manifest: &Manifest) -> Result, RenderError> { if program.login { out.push_str("login\n"); } + if program.memory_floor > 0 { + out.push_str(&format!("memory-floor {}\n", program.memory_floor)); + } + } + if manifest.supervisor_floor > 0 { + out.push_str(&format!("supervisor-floor {}\n", manifest.supervisor_floor)); } for name in &manifest.supervisor_serves { check("supervisor", "supervisor_serves", name)?; @@ -644,6 +659,11 @@ fn reject_newline(program: &str, field: &'static str, value: &str) -> Result<(), Ok(()) } +/// A record's byte count, which only [`render`] writes. +fn bytes(line: &str, rest: &str) -> u64 { + rest.parse().unwrap_or_else(|_| panic!("manifest: a byte count that is none: {line}")) +} + /// Read back what [`render`] wrote. /// /// Panics on a line neither half can produce. This file is the build system's @@ -686,6 +706,7 @@ pub fn parse(text: &str) -> Manifest { assert!(manifest.folder.replace(access).is_none(), "manifest: `app-folder` twice"); } "start" => manifest.start.push(rest.to_string()), + "supervisor-floor" => manifest.supervisor_floor = bytes(line, rest), "" => {} _ => { let program = manifest @@ -707,6 +728,7 @@ pub fn parse(text: &str) -> Manifest { "restart" if rest.is_empty() => program.restart = true, "starts" => program.starts.push(rest.to_string()), "login" if rest.is_empty() => program.login = true, + "memory-floor" => program.memory_floor = bytes(line, rest), other => panic!("manifest: unknown record `{other}`"), } } @@ -762,6 +784,7 @@ mod tests { source: Source::Path("/system/bin/vdagent".into()), receives: vec!["clipboard".into()], drives: vec!["pci:07:80:00/1af4:1043".into()], + memory_floor: 16 << 20, ..Program::default() }, Program { @@ -777,6 +800,7 @@ mod tests { apps: vec!["compositor".into(), "soundserver".into()], folder: Some(grants::Access::ReadWrite), start: vec!["compositor".into(), "soundserver".into()], + supervisor_floor: 48 << 20, } } @@ -790,6 +814,7 @@ mod tests { assert!(row.syscap.is_empty()); assert!(row.serves.is_empty()); assert!(row.provides.is_empty()); + assert_eq!(row.memory_floor, 0, "a package's row held memory back"); assert_eq!(row.source.path(), Some("/apps/gbae/gbae")); } diff --git a/lib/session/src/lib.rs b/lib/session/src/lib.rs index 7a09ffe7a12..a5ec3ebdc61 100644 --- a/lib/session/src/lib.rs +++ b/lib/session/src/lib.rs @@ -32,9 +32,19 @@ pub const ADMIN_HOLDER: &str = "logout"; /// The session owner's row. pub const OWNER: &str = "session"; -/// The label of the session owner's connection to the supervisor. +/// The label of the namespace the session owner connects its link to the +/// supervisor through, holding [`LINK`]: the owner connects, so the link is +/// counted to the owner's job and not the supervisor's. pub const LINK_LABEL: &str = "session-link"; +/// The name of the link in the owner's [`LINK_LABEL`] namespace. +pub const LINK: &str = "link"; + +/// The name of a session's controls in a namespace a login authority is +/// handed: it connects its control connection itself, which is then its +/// own, as a launcher's is. +pub const CONTROL: &str = "control"; + /// The label of the session launcher's acceptor in the session owner. pub const LAUNCHER_ACCEPTOR_LABEL: &str = "session-launcher"; @@ -43,13 +53,14 @@ pub const LAUNCHER_ACCEPTOR_LABEL: &str = "session-launcher"; pub const CONTROL_ACCEPTOR_LABEL: &str = "session-controls"; /// The label a login authority the supervisor started with its session holds -/// that session's control connection under. +/// that session's controls under, a namespace holding [`CONTROL`]. pub const CONTROL_LABEL: &str = "session-control"; /// On [`LOGIN_PORT`]: open a remote session. No payload. pub const MSG_OPEN: u32 = 1; /// The answer: the session's launcher, as a namespace holding -/// `toyos::launch::LAUNCHER`, then its control connection. +/// `toyos::launch::LAUNCHER`, then its controls, a namespace holding +/// [`CONTROL`]. pub const MSG_OPENED: u32 = 2; /// The answer to a request refused: the reason, UTF-8. pub const MSG_REFUSED: u32 = 3; @@ -273,8 +284,11 @@ pub fn open( let n = conn.recv_bytes(&header, &mut why).unwrap_or(0); return Err(format!("the supervisor refused a session: {}", String::from_utf8_lossy(&why[..n]))); } - let [launcher, control] = conn.recv_handles_exact::<2>(&header).ok_or("the session came without its handles")?; - Ok((toyos::namespace::Namespace::received(launcher), toyos::ipc::Connection::received(control))) + let [launcher, controls] = conn.recv_handles_exact::<2>(&header).ok_or("the session came without its handles")?; + let control = toyos::namespace::Namespace::received(controls) + .open(CONTROL) + .map_err(|e| format!("the session's controls refused its control connection: {e:?}"))?; + Ok((toyos::namespace::Namespace::received(launcher), control)) } #[cfg(test)] diff --git a/sdk/abi/src/handle.rs b/sdk/abi/src/handle.rs index 8877d86950a..0f96241d5dc 100644 --- a/sdk/abi/src/handle.rs +++ b/sdk/abi/src/handle.rs @@ -231,10 +231,17 @@ impl Rights { /// /// [`SYS_JOB_CREATE`]: crate::syscall::SYS_JOB_CREATE pub const JOBS: Rights = Rights(1 << 15); + /// On a `SysCap`: hold memory back for a job ([`SYS_JOB_FLOOR`]), which + /// every other job is then refused. What it holds is protected capacity + /// for the system's controls, so the supervisor alone holds it, and sets + /// what a signed `system.toml` row declares. + /// + /// [`SYS_JOB_FLOOR`]: crate::syscall::SYS_JOB_FLOOR + pub const FLOOR: Rights = Rights(1 << 16); /// Every bit that has a caller. A wider set than this is a bug in whoever /// composed it, not a right nobody uses. - pub const ALL: Rights = Rights(0xffff); + pub const ALL: Rights = Rights(0x1_ffff); pub const fn from_bits(bits: u32) -> Option { if bits & !Self::ALL.0 == 0 { Some(Rights(bits)) } else { None } @@ -265,7 +272,7 @@ impl Rights { /// refusal saying which right was missing. impl core::fmt::Debug for Rights { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - const NAMES: [(Rights, &str); 16] = [ + const NAMES: [(Rights, &str); 17] = [ (Rights::DUP, "DUP"), (Rights::TRANSFER, "TRANSFER"), (Rights::READ, "READ"), @@ -282,6 +289,7 @@ impl core::fmt::Debug for Rights { (Rights::COUNTERS, "COUNTERS"), (Rights::TRACE, "TRACE"), (Rights::JOBS, "JOBS"), + (Rights::FLOOR, "FLOOR"), ]; if self.0 == 0 { return f.write_str("NONE"); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index dcb934f5cfb..a91729ebac1 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -181,6 +181,11 @@ pub const PAGE_SIZE: usize = 4096; /// [`SHM_DEVICE_PAGE`]s of one physical run, as a device's domain maps it. pub const SHM_DEVICE: u64 = 1; +/// [`shm_create_for`]: the region is counted to the job the call's third +/// argument names, a handle carrying `WRITE`, rather than the caller's: the +/// authority to place a child in a job is the authority to make memory for it. +pub const SHM_FOR_JOB: u64 = 1 << 1; + /// What a [`SHM_DEVICE`] region is a whole number of. pub const SHM_DEVICE_PAGE: u64 = 2 * 1024 * 1024; @@ -321,6 +326,13 @@ pub const SYS_VM_DISCARD: u64 = 141; pub const SYS_VM_DECOMMIT: u64 = 142; /// A memory object over part of another's pages. See [`shm_sub`]. pub const SYS_SHM_SUB: u64 = 143; +/// Hold memory back for a job however short every other runs, gated by +/// [`Rights::FLOOR`] on a `SysCap`. See [`job_floor`]. +/// +/// [`Rights::FLOOR`]: crate::handle::Rights::FLOOR +pub const SYS_JOB_FLOOR: u64 = 144; +/// What a job's account holds. See [`account_info`]. +pub const SYS_ACCOUNT_INFO: u64 = 145; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -328,13 +340,13 @@ pub const SYS_SHM_SUB: u64 = 143; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 145; +pub const SYSCALL_PROFILE_BINS: usize = 147; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_SHM_SUB < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_ACCOUNT_INFO < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -1731,6 +1743,31 @@ pub fn job_create(syscap: RawHandle) -> Result { check(syscall(SYS_JOB_CREATE, syscap.0 as u64, 0, 0, 0)).map(|v| RawHandle(v as u32)) } +/// Hold `bytes` back for the job `job` names, which only it may then take +/// however short every other job runs, presenting a `SysCap` handle that +/// carries [`Rights::FLOOR`] and the job's handle with `WRITE`. The floor +/// lasts while that job handle's object does, and goes with its last +/// handle. `ResourceExhausted` where what is free now cannot hold it, and +/// `AlreadyExists` where another handle's object already holds one for the +/// job. +/// +/// [`Rights::FLOOR`]: crate::handle::Rights::FLOOR +pub fn job_floor(syscap: RawHandle, job: RawHandle, bytes: u64) -> Result<(), SyscallError> { + check_unit(syscall(SYS_JOB_FLOOR, syscap.0 as u64, job.0 as u64, bytes, 0)) +} + +/// What the account of the job `job` names holds, its handle carrying +/// `READ`. +pub fn account_info(job: RawHandle, info: &mut AccountInfo) -> Result<(), SyscallError> { + check_unit(syscall( + SYS_ACCOUNT_INFO, + job.0 as u64, + info as *mut AccountInfo as u64, + core::mem::size_of::() as u64, + 0, + )) +} + /// A handle to the job the caller runs in, carrying `WRITE`, `DUP` and /// `TRANSFER`: what a launch carries for its program to join the caller's job. /// Asks no right: it names only what the caller is already in. @@ -1949,6 +1986,14 @@ pub fn accept(acceptor: RawHandle) -> Result { check(syscall(SYS_ACCEPT, acceptor.0 as u64, 0, 0, 0)).map(|v| RawHandle(v as u32)) } +/// [`accept`]'s flag: answer `WouldBlock` rather than wait when none is queued. +pub const ACCEPT_NONBLOCK: u64 = 1; + +/// [`accept`] of a connection already queued; `WouldBlock` when none is. +pub fn accept_nonblock(acceptor: RawHandle) -> Result { + check(syscall(SYS_ACCEPT, acceptor.0 as u64, ACCEPT_NONBLOCK, 0, 0)).map(|v| RawHandle(v as u32)) +} + /// Handles one [`message_send`] may carry. /// /// Policy on the primitive: a caller asking for one more is refused by name @@ -2040,6 +2085,13 @@ pub fn shm_create_flags(size: usize, flags: u64) -> Result Result { + check(syscall(SYS_SHM_CREATE, size as u64, SHM_FOR_JOB, job.0 as u64, 0)).map(|v| RawHandle(v as u32)) +} + /// Map the region `shm` names into this process, readable and writable, and /// answer where. Needs [`Rights::MAP`]. /// @@ -2667,6 +2719,29 @@ pub fn modules(answer: &[u8]) -> impl Iterator { }) } +crate::user_safe! { + /// A job's account, as [`SYS_ACCOUNT_INFO`] answers it, in bytes. A + /// job's memory stays counted to it while an object it made lives, so + /// an account outlives the processes that ran in it. + #[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] + pub struct AccountInfo { + /// Held back for it however short every other job runs. + pub floor: u64, + /// Private and object memory its processes committed. + pub committed: u64, + /// Kernel memory it caused: heap its objects keep, its page tables, + /// and runs such as pipe rings and memory a device is lent. + pub kernel: u64, + /// The most of the two it held at once. + pub peak: u64, + /// What it would have held had its last refused request been + /// granted; zero if none was. + pub wanted: u64, + /// How many of its requests were refused for memory. + pub refusals: u64, + } +} + crate::user_safe! { /// Per-process accounting statistics, as [`SYS_PROCESS_STATS`] answers them. #[derive(Clone, Copy, Default)] diff --git a/sdk/std/sys/fs.rs b/sdk/std/sys/fs.rs index cd419554bd6..c51633e9fa3 100644 --- a/sdk/std/sys/fs.rs +++ b/sdk/std/sys/fs.rs @@ -593,8 +593,8 @@ pub fn set_times_nofollow(p: &Path, times: FileTimes) -> io::Result<()> { /// The whole of the file at `path` as a program image: what a spawn of a /// program the kernel cannot open itself hands the kernel. -pub fn read_image(path: &Path) -> io::Result { - files::read_image(bytes(path)).map_err(|refused| match refused { +pub fn read_image(path: &Path, job: Option) -> io::Result { + files::read_image_for(bytes(path), job).map_err(|refused| match refused { toyos::process::ImageRefused::Empty => { io::const_error!(io::ErrorKind::InvalidData, "the program is an empty file") } diff --git a/sdk/std/sys/process.rs b/sdk/std/sys/process.rs index 91207b88ceb..c25f3927ebe 100644 --- a/sdk/std/sys/process.rs +++ b/sdk/std/sys/process.rs @@ -140,7 +140,7 @@ impl Command { if !kept { self.image = None; if self.boot_image.is_none() && crate::sys::fs::is_served(Path::new(&source)) { - let image = crate::sys::fs::read_image(Path::new(&source))?; + let image = crate::sys::fs::read_image(Path::new(&source), self.endowed_job())?; self.image = Some((source, image)); } } @@ -213,6 +213,12 @@ impl Command { self.extra_fds.push((child_fd, handle, kind)); } + /// The job the child is endowed to run in, which its program image is + /// read for: the child keeps the image as long as it runs. + fn endowed_job(&self) -> Option { + self.endowments.iter().find(|(label, _)| label == toyos::job::JOB_LABEL).map(|(_, handle)| toyos::RawHandle(*handle)) + } + /// Give the child `handle` under the name `label`. /// /// The handle is **moved**: after a successful spawn this process no longer @@ -376,8 +382,9 @@ impl Command { toyos::spawn::Start::Helper(helper) => helper, }; // A program on a file server is taken here as the memory object its - // server holds it in, or read into one of this process's own, which - // the kernel pages the child from: it opens only what it serves itself. + // server holds it in, or read into one counted to the job the child is + // endowed to run in or else to this process's, which the kernel pages + // the child from: it opens only what it serves itself. let source = Path::new(image_path.unwrap_or(program)); let image = match (prepared_image, self.boot_image) { (Some(image), _) => Some(image), @@ -385,12 +392,15 @@ impl Command { let copy = toyos_abi::syscall::dup(toyos_abi::RawHandle(handle)).map_err(crate::sys::to_io_error)?; // SAFETY: `dup` answered a fresh handle nothing else answers for. let copy = unsafe { toyos::OwnedHandle::from_raw(copy) }; - let region = toyos::shm::Immutable::adopt(copy).map_err(|why| { - io::Error::new(io::ErrorKind::InvalidInput, format!("the boot image is no immutable region: {why}")) + let region = toyos::shm::Immutable::adopt(copy).map_err(|why| match why { + toyos::shm::NotImmutable::Kernel(e) => crate::sys::to_io_error(e), + toyos::shm::NotImmutable::Writable => { + io::const_error!(io::ErrorKind::InvalidInput, "the boot image is no immutable region") + } })?; Some(toyos::process::Image::immutable(region)) } - (None, None) if crate::sys::fs::is_served(source) => Some(crate::sys::fs::read_image(source)?), + (None, None) if crate::sys::fs::is_served(source) => Some(crate::sys::fs::read_image(source, self.endowed_job())?), (None, None) => None, }; let spawned = helper.spawn(image.as_ref()); diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index 43172b972f4..d99ef75304f 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -36,7 +36,7 @@ use crate::fd::{self, Fd}; use crate::fs; use crate::process::{Image, ImageRefused}; use crate::shm::{Immutable, NotImmutable}; -use crate::OwnedHandle; +use crate::{OwnedHandle, RawHandle}; /// The deepest directory a capability names, in components. pub const MAX_CAPABILITY_DEPTH: usize = 4; @@ -404,6 +404,12 @@ pub fn read_dir(fd: Fd, cursor: u64) -> Result<(Vec, Option), Error> { /// ([`crate::fs::IMAGE`]), and otherwise a copy read into memory of this /// process's own. pub fn read_image(path: &[u8]) -> Result> { + read_image_for(path, None) +} + +/// [`read_image`], a copy counted to the job `job` is a handle to: a +/// child's program, which it keeps (`Image::read_for`). +pub fn read_image_for(path: &[u8], job: Option) -> Result> { let fd = open(At::Cwd, path, Open { read: true, cloexec: true, ..Open::default() }).map_err(ImageRefused::Read)?; // SAFETY: a plain value; a handle answered is this process's alone. let held = decode(unsafe { toyos_fs_image(fd) }).map(|h| OwnedHandle(toyos_abi::RawHandle(h as u32))); @@ -413,7 +419,7 @@ pub fn read_image(path: &[u8]) -> Result> { NotImmutable::Writable => ImageRefused::Read(Error::Served(fs::Error::Malformed)), }), Err(Error::Served(fs::Error::Unsupported) | Error::Kernel(SyscallError::NotSupported)) => match fstat(fd) { - Ok(stat) => Image::read(stat.size, |buf| read(fd, buf)), + Ok(stat) => Image::read_for(job, stat.size, |buf| read(fd, buf)), Err(e) => Err(ImageRefused::Read(e)), }, Err(e) => Err(ImageRefused::Read(e)), diff --git a/sdk/toyos/src/job.rs b/sdk/toyos/src/job.rs index e40ec701888..9918ce01306 100644 --- a/sdk/toyos/src/job.rs +++ b/sdk/toyos/src/job.rs @@ -3,8 +3,12 @@ //! [`JOB_LABEL`] to put its child in it; [`own`] is this process's own, and //! [`SysCap::new_job`](crate::syscap::SysCap::new_job) makes an independent //! one. +//! +//! **A job is also the account its memory is counted to** ([`Job::account`]), +//! and a readable poll on its handle answers once memory it was refused is +//! free for it again: what a launcher waits on before it tries again. -use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::syscall::{self, AccountInfo, SyscallError}; pub use toyos_abi::syscall::JOB_LABEL; @@ -25,6 +29,13 @@ impl Job { syscall::dup(self.0.raw()).map(|raw| Job(unsafe { OwnedHandle::from_raw(raw) })) } + /// What its account holds. + pub fn account(&self) -> Result { + let mut info = AccountInfo::default(); + syscall::account_info(self.0.raw(), &mut info)?; + Ok(info) + } + /// Give up ownership, for a spawn that moves it. pub fn into_raw(self) -> RawHandle { self.0.into_raw() diff --git a/sdk/toyos/src/port.rs b/sdk/toyos/src/port.rs index 285a37a53f1..b623676a86b 100644 --- a/sdk/toyos/src/port.rs +++ b/sdk/toyos/src/port.rs @@ -41,6 +41,16 @@ impl Acceptor { syscall::accept(self.0.raw()).map(|h| Connection(OwnedHandle(h))) } + /// A connection already queued, or `None` where none is: for a loop that + /// must never wait on a client. + pub fn try_accept(&self) -> Result, SyscallError> { + match syscall::accept_nonblock(self.0.raw()) { + Ok(h) => Ok(Some(Connection(OwnedHandle(h)))), + Err(SyscallError::WouldBlock) => Ok(None), + Err(e) => Err(e), + } + } + /// A connector to this port whose every connection carries `badge`, which /// [`Self::badge`] reads back: what this holder granted whoever holds the /// connector, never who that is. diff --git a/sdk/toyos/src/process.rs b/sdk/toyos/src/process.rs index b983ba75c6f..4d4bddcea51 100644 --- a/sdk/toyos/src/process.rs +++ b/sdk/toyos/src/process.rs @@ -131,7 +131,15 @@ pub enum ImageRefused { impl Image { /// `len` bytes, from `read` asked until it has given them all. - pub fn read( + pub fn read(len: u64, read: impl FnMut(&mut [u8]) -> Result) -> Result> { + Self::read_for(None, len, read) + } + + /// [`read`](Self::read), counted to the job `job` is a handle to, rather + /// than this process's: the program of a child started in it, which + /// keeps the image for as long as it runs. + pub fn read_for( + job: Option, len: u64, mut read: impl FnMut(&mut [u8]) -> Result, ) -> Result> { @@ -139,7 +147,11 @@ impl Image { if size == 0 { return Err(ImageRefused::Empty); } - let mut object = SharedMemory::create(size).map_err(ImageRefused::Memory)?; + let created = match job { + Some(job) => SharedMemory::create_for(job, size), + None => SharedMemory::create(size), + }; + let mut object = created.map_err(ImageRefused::Memory)?; let bytes = object.as_mut_slice(); let mut done = 0; while done < size { diff --git a/sdk/toyos/src/shm.rs b/sdk/toyos/src/shm.rs index f123ea6aa28..9cec9e00733 100644 --- a/sdk/toyos/src/shm.rs +++ b/sdk/toyos/src/shm.rs @@ -49,6 +49,13 @@ impl SharedMemory { Self::adopt(syscall::shm_create(size)?, size) } + /// [`create`](Self::create), counted to the job `job` is a handle to, + /// carrying `WRITE`, rather than this process's own: memory laid out + /// for a child started in it. + pub fn create_for(job: RawHandle, size: usize) -> Result { + Self::adopt(syscall::shm_create_for(job, size)?, size) + } + /// [`create`](Self::create) for a region a device will be lent, by this /// process's driver or by a driver it sends the region to. pub fn create_for_device(size: usize) -> Result { diff --git a/sdk/toyos/src/syscap.rs b/sdk/toyos/src/syscap.rs index b1d342ffc68..d6dde469a87 100644 --- a/sdk/toyos/src/syscap.rs +++ b/sdk/toyos/src/syscap.rs @@ -2,7 +2,8 @@ //! //! Some things are reachable no other way — minting a device claim, entering //! the real-time band, listing every process in the machine, reading what the -//! machine is made of and what its CPUs count, and taking its power away, off or back to firmware — +//! machine is made of and what its CPUs count, holding memory back for a job, +//! and taking its power away, off or back to firmware — //! and each is one bit on a handle to this. The kernel makes exactly one at boot, for the supervisor, so the set of //! processes that can ever do any of them is exactly what the supervisor endowed. @@ -61,6 +62,13 @@ impl SysCap { Ok(crate::job::Job(unsafe { OwnedHandle::from_raw(raw) })) } + /// Hold `bytes` back for `job` however short every other job runs, for + /// as long as `job`'s handle and its duplicates live. Needs + /// [`Rights::FLOOR`], which the supervisor alone holds. + pub fn hold_floor(&self, job: &crate::job::Job, bytes: u64) -> Result<(), SyscallError> { + syscall::job_floor(self.0.raw(), job.as_handle(), bytes) + } + fn mint(&self, request: DeviceRequest) -> Result { let raw = syscall::device_claim(self.0.raw(), request)?; // SAFETY: the kernel installed this handle in this process's table for diff --git a/system/compositor/src/signout.rs b/system/compositor/src/signout.rs index 4d78374096f..fb0a98356ba 100644 --- a/system/compositor/src/signout.rs +++ b/system/compositor/src/signout.rs @@ -72,8 +72,12 @@ impl Desktop { .name("launcher".into()) .spawn(move || launcher(queue, login, sent, woken)) .expect("compositor: no thread to launch on"); + // Connected here, so the control connection is the compositor's own. + let control = endowed.take::(wire::CONTROL_LABEL).and_then(|controls| { + controls.open(wire::CONTROL).inspect_err(|why| eprintln!("compositor: its session's controls refused it: {why}")).ok() + }); let mut desktop = Self { - control: endowed.take::(wire::CONTROL_LABEL), + control, rx: ipc::FrameRx::new(), waiting: None, stopping: false, diff --git a/system/session/src/main.rs b/system/session/src/main.rs index 112138c4844..6353044575f 100644 --- a/system/session/src/main.rs +++ b/system/session/src/main.rs @@ -36,7 +36,12 @@ struct Control { fn main() { let endowed = Endowments::get(); - let link: Connection = endowed.take(wire::LINK_LABEL).expect("session: started without its link to the supervisor"); + // Connected here, so the link is this job's and not the supervisor's. + let link: Connection = endowed + .take::(wire::LINK_LABEL) + .expect("session: started without its link to the supervisor") + .open(wire::LINK) + .expect("session: the supervisor's link refused it"); let launcher: Acceptor = endowed.take(wire::LAUNCHER_ACCEPTOR_LABEL).expect("session: started without its launcher's acceptor"); let controls_acceptor: Acceptor = diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index ca180751a91..09b12cb3d12 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -33,6 +33,12 @@ //! as it is for a server nothing keeps. A program started through `launcher` //! still gets its acceptor by move and is started once per boot. //! +//! **A service keeps one job across its starts**, its row's `system.toml` +//! floor held on it, so a control path started again is owed what it was. +//! **A start refused for memory waits for it to come back**: the service +//! keeps its ports, and its job's readiness, posted when what it lacked is +//! free, is what starts it again ([`Supervisor::memory_returned`]). +//! //! **A `drives` row has one process per PCI function it names**, each holding //! that function's claim and started when the inventory shows the function //! free ([`toyos_supervisor::drives`]). **Every end is readiness on the @@ -252,7 +258,10 @@ const TOKEN_ADMIN_ACCEPTOR: u64 = 9; /// Every session owner's link, and the ends of every session's apps. const TOKEN_LINKS: u64 = 10; const TOKEN_APPS: u64 = 11; -const TOKEN_PENDING_BASE: u64 = 12; +/// The job of every service waiting for memory to come back to start again, +/// watched in the place of the process it has none of. +const TOKEN_MEMORY: u64 = 12; +const TOKEN_PENDING_BASE: u64 = 13; /// Above every pending connection's token, which is a handle's past the base. const TOKEN_POWER_WATCHERS: u64 = u64::MAX; @@ -309,6 +318,24 @@ fn new_ring() -> SharedMemory { region } +/// [`new_ring`] for a program started in `job`, counted to it: a ring is the +/// program's output, and the supervisor's own floor pays for none of it. +/// Refused where `job` has no memory for it. +fn ring_for(job: toyos::RawHandle) -> std::io::Result { + let region = SharedMemory::create_for(job, RING_BYTES).map_err(memory_refused)?; + view(®ion).lay_out_with_placeholder_owner(); + Ok(region) +} + +/// A refusal the kernel answered as an `io::Error` of the kind std gives it, +/// so a start refused for memory is told apart from one refused otherwise. +fn memory_refused(e: SyscallError) -> std::io::Error { + match e { + SyscallError::ResourceExhausted => std::io::Error::new(std::io::ErrorKind::OutOfMemory, "no memory left for it"), + other => std::io::Error::other(format!("{other:?}")), + } +} + fn view(region: &SharedMemory) -> Ring { let base = core::ptr::NonNull::new(region.as_ptr()).expect("a mapped region is not null"); // SAFETY: `region` maps `RING_BYTES` for as long as it is held, and the @@ -537,6 +564,16 @@ fn main() { // For the machine's life, and `'static` so a launch's path can be resolved // against it on the supervisor's file worker. let system: &'static Manifest = Box::leak(Box::new(toyos_manifest::parse(&text))); + // Its own floor, the protected capacity of the machine's controls: held + // through a handle to its own job, kept for the machine's life. + let own = toyos::job::own().expect("supervisor: no handle to its own job"); + if system.supervisor_floor > 0 { + match syscap.hold_floor(&own, system.supervisor_floor) { + Ok(()) => say!("supervisor: {} MiB held back for its own job", system.supervisor_floor >> 20), + Err(e) => toyos::error!("supervisor: its floor of {} MiB could not be held ({e:?}); the controls run unprotected", system.supervisor_floor >> 20), + } + } + let _own: &'static toyos::job::Job = Box::leak(Box::new(own)); // A role no row serves: closed before anything asks it, so a connect to it // answers `ServerGone` rather than waiting in a queue nothing will ever // accept from. @@ -756,6 +793,8 @@ struct Instance { row: &'static Program, function: Pci, child: Child, + /// The job it runs in, kept while it runs: what holds its row's floor. + _job: toyos::job::Job, /// A disk driver's [`toyos_quiesce::DRIVER_STOP`] pipe: closing it tells the /// driver the machine stops. stop: Option, @@ -790,6 +829,12 @@ struct Service<'a> { /// The screen's row's desktop session's launcher and control, minted for /// its next start, which takes them. desktop: Option, + /// The job every process serving it runs in, made at its first start and + /// kept: its floor, where its row has one, outlives each process, so a + /// start after an end is not refused what the row was held. + job: Option, + /// Since when a start after its end has waited for memory to come back. + starved: Option, } impl<'a> Service<'a> { @@ -805,6 +850,8 @@ impl<'a> Service<'a> { session, budget: Budget::default(), desktop: None, + job: None, + starved: None, } } @@ -845,6 +892,11 @@ impl<'a> Service<'a> { storage.endow.push((LAUNCHER, desktop.launcher.into_raw())); storage.endow.push((toyos_session::CONTROL_LABEL, desktop.control.into_raw())); } + let job = match &self.job { + Some(job) => job, + None => self.job.insert(job_for(syscap, self.program)?), + }; + let endowed = job.duplicate().map_err(memory_refused)?.into_raw(); let ports = self.ports.borrow(); let served = Served::Keep { acceptors: &ports, owed }; let command = match source { @@ -866,7 +918,7 @@ impl<'a> Service<'a> { &[], storage, Minting::machine(minted, self.program, self.session), - Output::Boot(log), + Output::Boot { log, job: endowed }, )?; drop(ports); let pid = child.id(); @@ -880,6 +932,22 @@ impl<'a> Service<'a> { self.child.as_ref().map(Child::id) } + /// How many requests its job was refused for memory: a start that raised + /// it waits for memory, and one refused anything else was refused that. + fn refusals(&self) -> u64 { + self.job.as_ref().and_then(|job| job.account().ok()).map_or(0, |account| account.refusals) + } + + /// What its job's account says it would have held had its last refused + /// request been granted, for a line that says why it waits. + fn wanted(&self) -> String { + match self.job.as_ref().map(toyos::job::Job::account) { + Some(Ok(account)) => format!("{} KiB, to hold {} KiB in all", account.wanted.saturating_sub(account.committed + account.kernel) >> 10, account.wanted >> 10), + Some(Err(e)) => format!("what its account would not say: {e:?}"), + None => String::from("memory before it had a job"), + } + } + /// The name its lines and the supervisor's say it under: a file server's role /// beside its row's. fn label(&self) -> String { @@ -1041,6 +1109,7 @@ impl<'a> Supervisor<'a> { Ok(_) => {} Err(StartError::Refused(why)) => { service.ports.borrow_mut().clear(); + service.job = None; say!("supervisor: {} did not start ({why}); its ports are closed this boot", service.label()); } Err(e) => panic!("supervisor: cannot start {}: {e}", program.name), @@ -1074,7 +1143,7 @@ impl<'a> Supervisor<'a> { for act in self.drives.decide(&rows, &records, boot_controller(), Instant::now()) { match act { Act::Start { row, entry, function } => match self.start_instance(row, entry, function) { - Ok((child, stop)) => self.instances.push(Instance { row, function, child, stop }), + Ok((child, stop, job)) => self.instances.push(Instance { row, function, child, stop, _job: job }), Err(why) => { say!("supervisor: {} was not started on PCI {}: {why}", row.name, pci_at(function.at)); refused.push(format!("{entry} at PCI {}", pci_at(function.at))); @@ -1109,7 +1178,7 @@ impl<'a> Supervisor<'a> { /// One `drives` process of `row`, holding `function`'s claim under /// `entry`'s spelling. - fn start_instance(&mut self, row: &'static Program, entry: PciMatch, function: Pci) -> Result<(Child, Option), String> { + fn start_instance(&mut self, row: &'static Program, entry: PciMatch, function: Pci) -> Result<(Child, Option, toyos::job::Job), String> { if self.services.len() + self.instances.len() >= MAX_CHILDREN { return Err(format!("the supervisor watches {MAX_CHILDREN} processes already")); } @@ -1130,10 +1199,12 @@ impl<'a> Supervisor<'a> { } let launcher = Minting::machine(&self.minted, row, self.sessions.machine()); let served = Served::Keep { acceptors: &[], owed: &[] }; - let output = Output::Boot(&mut self.log); let path = row.source.path().ok_or_else(|| format!("{} runs {}, and a `drives` row runs a file", row.name, row.source))?; + let job = job_for(self.syscap, row).map_err(|e| e.to_string())?; + let endowed = job.duplicate().map_err(|e| format!("its job would not duplicate: {e:?}"))?.into_raw(); + let output = Output::Boot { log: &mut self.log, job: endowed }; start(Command::new(path), row, self.system, self.syscap, served, &self.connectors, &self.grants, &[], extra, launcher, output) - .map(|(child, _)| (child, stop)) + .map(|(child, _)| (child, stop, job)) .map_err(|e| e.to_string()) } @@ -1329,14 +1400,17 @@ impl<'a> Supervisor<'a> { /// call a server's end left waiting in the port's queue goes on to the /// process started in its place. /// `Err` is the call still unanswered at [`FILES_BOUND`], which the worker - /// goes on waiting for alone, or the worker still waiting on an earlier - /// one. + /// goes on waiting for alone, the worker still waiting on an earlier + /// one, or a storage service waiting for memory to start again: no call + /// may be answered until it has, and the loop answers the controls + /// meanwhile. fn files(&mut self, what: &str, work: impl FnOnce() -> T + Send + 'static) -> Result { const DONE: u64 = 0; self.worker.settle(); if self.worker.owed > 0 { return Err(format!("the supervisor's file worker still waits on {}", self.worker.owed_what)); } + self.storage_fed()?; let (answer, answered) = std::sync::mpsc::channel(); let job: Job = Box::new(move || { // The receiver is gone once the bound has passed. @@ -1361,6 +1435,7 @@ impl<'a> Supervisor<'a> { if self.worker.owed == 0 { return Ok(answered.recv().expect("supervisor: a finished file job answered")); } + self.storage_fed()?; if Instant::now() >= deadline { return Err(format!("{what} was not answered in {} s", FILES_BOUND.as_secs())); } @@ -1381,6 +1456,7 @@ impl<'a> Supervisor<'a> { /// service it has just killed to finish ending, which is the kernel's /// teardown and no client's. fn serve_forever(&mut self, swap: &Acceptor, power: &Acceptor) -> ! { + self.say_floors(); let poller = Poller::new(Poller::MAX_HANDLES); let mut pending: Vec = Vec::new(); let mut flight: Option = None; @@ -1393,7 +1469,7 @@ impl<'a> Supervisor<'a> { poller.watch(&self.minted.admin, READABLE, TOKEN_ADMIN_ACCEPTOR); poller.watch(&self.watch.ended, READABLE, TOKEN_APPS); for login in &self.logins.open { - poller.watch(&login.link, READABLE, TOKEN_LINKS); + poller.watch_raw(login.link.handle(), READABLE, TOKEN_LINKS); } poller.watch(swap, READABLE, TOKEN_SWAP_ACCEPTOR); poller.watch(power, READABLE, TOKEN_POWER_ACCEPTOR); @@ -1402,6 +1478,10 @@ impl<'a> Supervisor<'a> { poller.watch(&watcher.conn, READABLE | writable, TOKEN_POWER_WATCHERS); } self.watch_children(&poller); + for service in self.services.iter().filter(|service| service.starved.is_some()) { + let job = service.job.as_ref().expect("supervisor: a service waiting for memory has its job"); + poller.watch(job, READABLE, TOKEN_MEMORY); + } if let Some(Flight { phase: Phase::Answered { conn, .. }, .. }) = &flight { poller.watch(conn, READABLE, TOKEN_HANGUP); } @@ -1552,6 +1632,9 @@ impl<'a> Supervisor<'a> { if ready.contains(&TOKEN_APPS) { self.apps_ended(); } + if ready.contains(&TOKEN_MEMORY) { + self.memory_returned(); + } if ready.contains(&TOKEN_ENDED) { self.ended(); } @@ -1908,30 +1991,34 @@ impl<'a> Supervisor<'a> { let (label, pid) = (service.label(), service.pid().unwrap_or(0)); service.child = None; match restart::after_end(service.program, exit, &mut service.budget, Instant::now()) { - AfterEnd::Close => service.ports.borrow_mut().clear(), + AfterEnd::Close => { + service.ports.borrow_mut().clear(); + service.job = None; + } AfterEnd::GiveUp => { service.ports.borrow_mut().clear(); + service.job = None; say!( "supervisor: {label} ended {} times in {} s; its ports are closed, its clients answered Gone, and a program started from now on is given none of its directories", restart::RESTARTS + 1, restart::WINDOW.as_secs() ); } - AfterEnd::Restart => { - let (source, owed, program) = (service.source.clone(), service.devices.clone(), service.program); - self.make_state_dir(program); - self.desktop_for(index); - let block = self.block_args(); - let service = &mut self.services[index]; - let (system, syscap, connectors, grants) = (self.system, self.syscap, &self.connectors, &self.grants); - match service.spawn(&source, &owed, block, system, syscap, connectors, grants, &self.minted, self.boot, &mut self.log) { - Ok(new) => say!("supervisor: {label} (pid {pid}) ended; started again as pid {new}"), - Err(e) => { - service.ports.borrow_mut().clear(); - say!("supervisor: {label} ended and would not start again ({e}); its ports are closed"); - } + AfterEnd::Restart => match self.start_again(index) { + Ok(new) => say!("supervisor: {label} (pid {pid}) ended; started again as pid {new}"), + Err((e, true)) => { + let wanted = self.services[index].wanted(); + self.services[index].starved = Some(Instant::now()); + say!( + "supervisor: {label} (pid {pid}) ended and could not start again for want of memory ({e}; it lacked {wanted}); \ + it starts again once memory comes back, and its clients wait on its ports meanwhile" + ); } - } + Err((e, false)) => { + self.close_for_good(index); + say!("supervisor: {label} ended and would not start again ({e}{}); its ports are closed", unrecorded(&e)); + } + }, } } let mut at = 0; @@ -1952,6 +2039,80 @@ impl<'a> Supervisor<'a> { self.restarting = false; } + /// What each floor job holds of its floor once the boot is done: the margin + /// a floor's size is judged by. + fn say_floors(&self) { + let own = toyos::job::own().and_then(|job| job.account()); + let services = self.services.iter().filter_map(|s| Some((s.label(), s.job.as_ref()?.account().ok()?))); + for (name, account) in own.ok().map(|a| ("the supervisor".to_string(), a)).into_iter().chain(services) { + if account.floor > 0 { + say!("supervisor: {name} holds {} KiB of its {} KiB floor, at most {} KiB so far", (account.committed + account.kernel) >> 10, account.floor >> 10, account.peak >> 10); + } + } + } + + /// Service `index` serves no more: its ports close, and its job goes, and + /// with it any floor its row was held. + fn close_for_good(&mut self, index: usize) { + let service = &mut self.services[index]; + service.ports.borrow_mut().clear(); + service.starved = None; + service.job = None; + } + + /// `Err` naming a storage service that waits for memory to start again. + fn storage_fed(&self) -> Result<(), String> { + match self.services.iter().find(|s| s.starved.is_some() && is_storage(s.program)) { + Some(starved) => Err(format!("{} waits for memory to start again", starved.label())), + None => Ok(()), + } + } + + /// Start service `index` again on the ports and devices it kept, from + /// what it last ran. A refusal says whether its job was refused memory + /// meanwhile, which is what a wait for memory to come back can answer. + fn start_again(&mut self, index: usize) -> Result { + let before = self.services[index].refusals(); + let started = self.start_from(index); + started.map_err(|e| (e, self.services[index].refusals() > before)) + } + + fn start_from(&mut self, index: usize) -> Result { + let service = &self.services[index]; + let (source, owed, program) = (service.source.clone(), service.devices.clone(), service.program); + self.make_state_dir(program); + self.desktop_for(index); + let block = self.block_args(); + let service = &mut self.services[index]; + let (system, syscap, connectors, grants) = (self.system, self.syscap, &self.connectors, &self.grants); + service.spawn(&source, &owed, block, system, syscap, connectors, grants, &self.minted, self.boot, &mut self.log) + } + + /// Memory came back that a service waiting to start again lacked: each + /// is tried again, and one still short waits for the next time, said + /// with what it lacks now. + fn memory_returned(&mut self) { + for index in 0..self.services.len() { + let Some(since) = self.services[index].starved else { continue }; + let label = self.services[index].label(); + let waited = since.elapsed().as_millis(); + match self.start_again(index) { + Ok(pid) => { + self.services[index].starved = None; + say!("supervisor: {label} started again as pid {pid} once memory came back, {waited} ms after its start was refused"); + } + Err((e, true)) => { + let wanted = self.services[index].wanted(); + say!("supervisor: {label} is still short of memory after {waited} ms ({e}; it lacks {wanted}); it waits on"); + } + Err((e, false)) => { + self.close_for_good(index); + say!("supervisor: {label} would not start again once memory came back ({e}{}); its ports are closed", unrecorded(&e)); + } + } + } + } + /// Start the binary a failed swap replaced, or close the service's ports /// when that will not start either. fn restore(&mut self, index: usize, previous: &Source, owed: &[String]) { @@ -1969,6 +2130,7 @@ impl<'a> Supervisor<'a> { Err(e) => { service.swapping = false; service.ports.borrow_mut().clear(); + service.job = None; swapped( &name, Word::Gone, @@ -2189,6 +2351,10 @@ impl Supervisor<'_> { } } + // Endowed before `prepare` reads the program, so its image is read for the + // job the child runs in, which keeps it as long as the child runs. + command.endow(JOB_LABEL, job.0[0].0); + // On the worker, every file the launch reads: a path under `/apps` is // read off its package, and the program and its working directory may // be a file server's, which the supervisor supervises. Whether the @@ -2788,7 +2954,7 @@ fn start<'a>( // A storage row's own arguments first: a file server's role leads its argv. command.args(&args); command.args(&program.args); - let booting = matches!(output, Output::Boot(_)); + let booting = matches!(output, Output::Boot { .. }); // **Set here, over whatever a launching caller carried**: the row decides // where a program's home is, and a service's is made before this @@ -2802,16 +2968,16 @@ fn start<'a>( // class nothing can mint again. A client picks whether a spawn fails — an // unreachable `cwd` is enough — so this is its path as much as a bug's. let mut held = Moved(Vec::new()); - // **Every service the supervisor starts is a job of its own**, made with the - // system capability's `JOBS`; a launch's program joins the job its caller + // **Every service the supervisor starts is a job of its own**, which its + // caller made ([`job_for`]); a launch's program joins the job its caller // named, which is the caller's own unless it holds that right itself. let job = match &output { - Output::Boot(_) => { - syscap.new_job().map_err(|e| std::io::Error::other(format!("no job for it: {e:?}")))?.into_raw() - } - Output::Launch { job, .. } => *job, + Output::Boot { job, .. } | Output::Launch { job, .. } => *job, }; - command.endow(JOB_LABEL, job.0); + // A launch's command was endowed its job before its image was read. + if matches!(output, Output::Boot { .. }) { + command.endow(JOB_LABEL, job.0); + } held.0.push(job); // Acceptors are given *back* rather than closed: a `serves` port whose // acceptor is gone can never be served again. @@ -3001,8 +3167,8 @@ fn start<'a>( let mut ring: Option = None; let mut origins: Option = None; let log: &mut Log = match output { - Output::Boot(log) => { - let own = ring.insert(new_ring()).as_handle(); + Output::Boot { log, .. } => { + let own = ring.insert(ring_for(job)?).as_handle(); command.inherit_handle(0, log.stdin.0); command.inherit_handle(1, own.0); command.inherit_handle(2, own.0); @@ -3029,7 +3195,10 @@ fn start<'a>( .is_ok_and(|stat| stat.file_type == FileType::SharedMemory); match kind { _ if caller_ring => { - let own = ring.get_or_insert_with(new_ring).as_handle(); + let own = match &mut ring { + Some(ring) => ring.as_handle(), + None => ring.insert(ring_for(job)?).as_handle(), + }; command.inherit_handle(fd, own.0); } toyos::fd::Kind::Handle => { @@ -3095,7 +3264,7 @@ fn start<'a>( /// launch's caller's descriptors with any ring among them replaced — and, for /// a launch, the job its caller named, which `start` owns from its first line. enum Output<'l> { - Boot(&'l mut Log), + Boot { log: &'l mut Log, job: toyos::RawHandle }, Launch { log: &'l mut Log, fds: &'l [(u32, toyos::RawHandle, toyos::fd::Kind)], job: toyos::RawHandle }, } @@ -3533,3 +3702,34 @@ fn system_row() -> Program { ..Program::default() } } + +/// What a refusal no memory was recorded for is, beside its own words: a +/// `ResourceExhausted` the job's account did not count is a limit, or the +/// supervisor's own memory, and waiting for memory would not end it. +fn unrecorded(e: &StartError) -> &'static str { + match e { + StartError::Other(e) if e.kind() == std::io::ErrorKind::OutOfMemory => { + "; its job was refused no memory, so it is a limit or the supervisor's own refusal" + } + _ => "", + } +} + +/// A job of its own for a service or driver `row` starts: made with the system +/// capability's `JOBS`, and held its row's floor, which only a signed image's +/// `system.toml` declares. A floor refused is a start refused: a control path +/// started without the memory it is owed would answer until memory ran out +/// and then not. +fn job_for(syscap: &SysCap, row: &Program) -> std::io::Result { + let job = syscap.new_job().map_err(memory_refused)?; + if row.memory_floor > 0 { + syscap.hold_floor(&job, row.memory_floor).map_err(|e| match e { + SyscallError::ResourceExhausted => std::io::Error::new( + std::io::ErrorKind::OutOfMemory, + format!("its floor of {} MiB cannot be held", row.memory_floor >> 20), + ), + other => std::io::Error::other(format!("its floor would not be held: {other:?}")), + })?; + } + Ok(job) +} diff --git a/system/supervisor/src/sessions.rs b/system/supervisor/src/sessions.rs index 5f718a83b78..1e31d11f7ad 100644 --- a/system/supervisor/src/sessions.rs +++ b/system/supervisor/src/sessions.rs @@ -27,7 +27,7 @@ use toyos::ipc::{self, Connection, RxStep}; use toyos::launch::LAUNCHER; use toyos::namespace::{self, Namespace}; use toyos::poller::{Poller, READABLE}; -use toyos::port::{self, Acceptor}; +use toyos::port::{self, Acceptor, Connector}; use toyos::{say, AsHandle, OwnedHandle, Pipe, RawHandle}; use toyos_abi::syscall; use toyos_abi::Rights; @@ -38,9 +38,6 @@ use toyos_supervisor::sessions::{self as decide, Reopens}; use crate::{exit_of, start, Caller, Extra, LaunchRx, Minting, Output, Pending, Port, Served, Supervisor}; -/// The name a session's controls namespace holds its control port under. -const CONTROL: &str = "control"; - /// The login sessions the supervisor watches the owners of at once. Each /// owner is a process its loop watches, beside its services and `drives` /// processes. The last is the desktop's alone, so sessions left ending by an @@ -53,8 +50,8 @@ pub(crate) struct Login { /// The login authority's row that opened it. opener: &'static Program, pub owner: Child, - /// The supervisor's end of the owner's link. - pub link: Connection, + /// The supervisor's end of the owner's link, which the owner connects. + pub link: Link, rx: LaunchRx, /// The owner's handle to itself, which places a detached app under it. place: Option, @@ -62,30 +59,71 @@ pub(crate) struct Login { /// Held here alone, so the session's end closes it and a launch connected /// to it reads its end. pub launcher: Acceptor, - /// A connector to the session's control port, for each control connection - /// a login authority is handed. - controls: Namespace, + /// The session's control port, a namespace of which each login + /// authority is handed to connect its own control connection through. + controls: Connector, /// Whether the screen's row opened it: the desktop's. desktop: bool, state: State, waiting: Names, } -/// The desktop session's launcher and a control connection to it, minted for -/// one start of the screen's row. +/// The desktop session's launcher and its controls, minted for one start of +/// the screen's row. pub(crate) struct Desktop { pub launcher: Namespace, - pub control: Connection, + pub control: Namespace, +} + +/// A session owner's link: the port the owner connects, until it has, and +/// then the connection. The owner connects, so the link is counted to its +/// job. +pub(crate) enum Link { + Waiting(Acceptor), + Open(Connection), +} + +impl Link { + /// What the loop watches for the owner's word, or its connect. + pub fn handle(&self) -> RawHandle { + match self { + Self::Waiting(acceptor) => acceptor.as_handle(), + Self::Open(conn) => conn.as_handle(), + } + } + + /// The connection, taken off the port first if the owner has connected. + fn open(&mut self) -> Option<&Connection> { + if let Self::Waiting(acceptor) = self { + match acceptor.try_accept() { + Ok(Some(conn)) => *self = Self::Open(conn), + Ok(None) => return None, + Err(e) => { + say!("supervisor: a session owner's link was not accepted ({e:?})"); + return None; + } + } + } + self.conn() + } + + /// The connection, once the owner has connected. + fn conn(&self) -> Option<&Connection> { + match self { + Self::Open(conn) => Some(conn), + Self::Waiting(_) => None, + } + } } impl Login { - /// `opener`'s launcher in this session, and a control connection to it. - fn handed_to(&self, opener: &Program) -> Result<(Namespace, Connection), String> { + /// `opener`'s launcher in this session, and its controls to connect to. + fn handed_to(&self, opener: &Program) -> Result<(Namespace, Namespace), String> { let badge = Authority { row: opener.name.clone(), session: self.session }.encode(); let connector = self.launcher.mint(&badge).map_err(|e| format!("no launcher was minted ({e:?})"))?; let launcher = namespace::build().add(LAUNCHER, &connector).finish().map_err(|e| format!("{e:?}"))?; - let control = self.controls.open(CONTROL).map_err(|e| format!("no control connection ({e:?})"))?; - Ok((launcher, control)) + let controls = namespace::build().add(wire::CONTROL, &self.controls).finish().map_err(|e| format!("{e:?}"))?; + Ok((launcher, controls)) } } @@ -128,7 +166,11 @@ impl Logins { /// is said. pub fn tell_all(&self, msg_type: u32) { for login in &self.open { - if let Err(e) = login.link.try_signal(msg_type) { + let Some(link) = login.link.conn() else { + say!("supervisor: session {}'s owner has not connected its link, and is not told {msg_type}", login.session.share()); + continue; + }; + if let Err(e) = link.try_signal(msg_type) { say!("supervisor: session {}'s owner was not told {msg_type} ({e:?})", login.session.share()); } } @@ -261,14 +303,11 @@ impl Supervisor<'_> { let (launcher, _) = port::create().map_err(|e| format!("no launcher port ({e:?})"))?; let (controls_acceptor, controls_connector) = port::create().map_err(|e| format!("no control port ({e:?})"))?; let (link_acceptor, link_connector) = port::create().map_err(|e| format!("no link ({e:?})"))?; - let links = namespace::build().add(CONTROL, &link_connector).finish().map_err(|e| format!("{e:?}"))?; - let link = links.open(CONTROL).map_err(|e| format!("the link would not open ({e:?})"))?; - let owner_end = link_acceptor.accept().map_err(|e| format!("the link was not accepted ({e:?})"))?; - let controls = namespace::build().add(CONTROL, &controls_connector).finish().map_err(|e| format!("{e:?}"))?; + let link = namespace::build().add(wire::LINK, &link_connector).finish().map_err(|e| format!("{e:?}"))?; let accepting = syscall::dup(launcher.as_handle()).map_err(|e| format!("no copy of its launcher ({e:?})"))?; let extra = Extra { endow: vec![ - (wire::LINK_LABEL, owner_end.into_raw()), + (wire::LINK_LABEL, link.into_raw()), (wire::LAUNCHER_ACCEPTOR_LABEL, accepting), (wire::CONTROL_ACCEPTOR_LABEL, controls_acceptor.into_raw()), ], @@ -277,6 +316,7 @@ impl Supervisor<'_> { self.make_state_dir(owner_row); let minting = Minting { minted: &self.minted, session, launches: None }; let served = Served::Keep { acceptors: &[], owed: &[] }; + let job = crate::job_for(self.syscap, owner_row).map_err(|e| format!("no job for its owner: {e}"))?.into_raw(); let (owner, _) = start( Command::new(path), owner_row, @@ -288,7 +328,7 @@ impl Supervisor<'_> { &[], extra, minting, - Output::Boot(&mut self.log), + Output::Boot { log: &mut self.log, job }, ) .map_err(|e| format!("its owner did not start: {e}"))?; say!("supervisor: session {} opened for {}, owned by pid {}", session.share(), opener.name, owner.id()); @@ -296,11 +336,11 @@ impl Supervisor<'_> { session, opener, owner, - link, + link: Link::Waiting(link_acceptor), rx: LaunchRx::new(), place: None, launcher, - controls, + controls: controls_connector, desktop, state: State::Active, waiting: Names::default(), @@ -358,9 +398,9 @@ impl Supervisor<'_> { .map_or_else(|| self.open_session(opener), Ok) .and_then(|at| self.logins.open[at].handed_to(opener)); match answered { - Ok((ns, control)) => { - // SAFETY: the namespace's own handle, given up here and to nothing else. - let handles = [unsafe { OwnedHandle::from_raw(ns.into_raw()) }, control.into_owned()]; + Ok((ns, controls)) => { + // SAFETY: the two namespaces' own handles, given up here and to nothing else. + let handles = unsafe { [OwnedHandle::from_raw(ns.into_raw()), OwnedHandle::from_raw(controls.into_raw())] }; if let Err(e) = conn.try_signal_with(wire::MSG_OPENED, handles) { say!("supervisor: login: {}'s session was opened and not answered ({e:?})", opener.name); } @@ -378,7 +418,8 @@ impl Supervisor<'_> { for at in 0..self.logins.open.len() { loop { let login = &mut self.logins.open[at]; - let (msg_type, payload, batch) = match login.rx.pump(&login.link) { + let Some(link) = login.link.open() else { break }; + let (msg_type, payload, batch) = match login.rx.pump(link) { RxStep::Idle | RxStep::Eof => break, // The supervisor's own child broke its protocol: the session // ends with it, as a failure. @@ -448,7 +489,11 @@ impl Supervisor<'_> { let started = Started { instance, name: name.to_string() }; // SAFETY: a duplicate made above, which nothing else answers for. let managed = unsafe { OwnedHandle::from_raw(managed) }; - if let Err(e) = login.link.try_send_bytes_with(wire::MSG_STARTED, &started.encode(), [managed]) { + let sent = match login.link.conn() { + Some(link) => link.try_send_bytes_with(wire::MSG_STARTED, &started.encode(), [managed]).map_err(|e| format!("{e:?}")), + None => Err("its link is not connected".to_string()), + }; + if let Err(e) = sent { say!("supervisor: session {}'s owner was not told of {name} ({e:?})", session.share()); } // SAFETY: as above. @@ -461,7 +506,9 @@ impl Supervisor<'_> { while self.watch.ended.read_nonblock(&mut drained).is_ok_and(|n| n > 0) {} while let Ok((instance, session)) = self.watch.ends.try_recv() { if let Some(login) = self.logins.of(session) { - let _ = login.link.try_send_bytes(wire::MSG_ENDED, &instance.to_le_bytes()); + if let Some(link) = login.link.conn() { + let _ = link.try_send_bytes(wire::MSG_ENDED, &instance.to_le_bytes()); + } } } } @@ -539,6 +586,8 @@ impl Supervisor<'_> { let SignOut { force } = asked.sign_out; login .link + .conn() + .ok_or_else(|| format!("session {}'s owner has not connected its link", asked.share))? .try_send_bytes(wire::MSG_SIGN_OUT, &asked.sign_out.encode()) .map_err(|e| format!("session {}'s owner was not asked ({e:?})", asked.share))?; Ok(format!("session {} is asked to sign out{}", asked.share, if force { ", by force" } else { "" })) diff --git a/tests/guest/inputsink/Cargo.toml b/tests/guest/inputsink/Cargo.toml new file mode 100644 index 00000000000..d915f221724 --- /dev/null +++ b/tests/guest/inputsink/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "inputsink" +description = "Test-only: an owner of input with no screen, relaying each key a source sends it to the one test that watches, so a test can hold that input reached its owner." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dependencies] +toyos-abi = { path = "../../../sdk/abi" } +toyos = { path = "../../../sdk/toyos" } +toyos-input = { path = "../../../lib/input" } diff --git a/tests/guest/inputsink/src/main.rs b/tests/guest/inputsink/src/main.rs new file mode 100644 index 00000000000..d87b11a307b --- /dev/null +++ b/tests/guest/inputsink/src/main.rs @@ -0,0 +1,93 @@ +//! Test-only: an owner of input with no screen. Every key a source sends on +//! `input` is relayed to the one test connected on `keys`, which asks first, +//! so the relay is a send on a connection that has carried a frame each way +//! and needs no memory of its own. +//! +//! `keys` frames: the test's [`HELLO`] is answered [`HELLO`]; then each key is +//! a [`KEY`] carrying the source's usage and whether it was released. + +use toyos::endow; +use toyos::ipc::{Connection, FrameRx, RxStep}; +use toyos::poller::{Poller, READABLE}; +use toyos_input::wire::{self, Record, MAX_PAYLOAD}; + +/// The port the test watches keys on. +const KEYS: &str = "keys"; +/// The test's first frame, and its answer. +const HELLO: u32 = 1; +/// A key: its usage, then 1 if it was released and 0 if pressed. +const KEY: u32 = 2; + +const TOKEN_INPUT: u64 = 0; +const TOKEN_KEYS: u64 = 1; +const TOKEN_SOURCE: u64 = 2; +const TOKEN_WATCHER: u64 = 3; + +fn main() { + let inputs = endow::acceptor(wire::PORT).expect("inputsink: the manifest declares it serves `input`"); + let keys = endow::acceptor(KEYS).expect("inputsink: the manifest declares it serves `keys`"); + let mut source: Option<(Connection, FrameRx)> = None; + let mut watcher: Option<(Connection, FrameRx<0>)> = None; + let poller = Poller::new(4); + println!("inputsink: serving input"); + loop { + poller.watch(&inputs, READABLE, TOKEN_INPUT); + poller.watch(&keys, READABLE, TOKEN_KEYS); + if let Some((conn, _)) = &source { + poller.watch(conn, READABLE, TOKEN_SOURCE); + } + if let Some((conn, _)) = &watcher { + poller.watch(conn, READABLE, TOKEN_WATCHER); + } + let mut ready = [false; 4]; + poller.wait(1, u64::MAX, |token| ready[token as usize] = true); + + // One source and one watcher: a later one takes the place of the last. + if ready[TOKEN_INPUT as usize] { + if let Ok(conn) = inputs.accept() { + source = Some((conn, FrameRx::new())); + } + } + if ready[TOKEN_KEYS as usize] { + if let Ok(conn) = keys.accept() { + watcher = Some((conn, FrameRx::new())); + } + } + if ready[TOKEN_WATCHER as usize] { + if let Some((conn, rx)) = &mut watcher { + loop { + match rx.pump(conn) { + RxStep::Frame { msg_type: HELLO, .. } => { + let _ = conn.try_signal(HELLO); + } + RxStep::Frame { .. } | RxStep::Idle => break, + RxStep::Eof | RxStep::Malformed => { + watcher = None; + break; + } + } + } + } + } + if ready[TOKEN_SOURCE as usize] { + let Some((conn, rx)) = &mut source else { continue }; + loop { + let record = match rx.pump(conn) { + RxStep::Idle => break, + RxStep::Eof | RxStep::Malformed => { + source = None; + break; + } + RxStep::Frame { msg_type, payload_len, .. } => wire::decode(msg_type, rx.payload(payload_len)), + }; + if let Ok(Record::Key { key, .. }) = record { + let released = u8::from(key.modifiers & toyos_abi::input::MOD_RELEASED != 0); + println!("inputsink: key {:#04x} {}", key.keycode, if released == 1 { "released" } else { "pressed" }); + if let Some((watching, _)) = &watcher { + let _ = watching.try_send_bytes(KEY, &[key.keycode, released]); + } + } + } + } + } +} diff --git a/tests/guest/rust-tests/src/bin/memory_floor.rs b/tests/guest/rust-tests/src/bin/memory_floor.rs new file mode 100644 index 00000000000..71bb50741f6 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/memory_floor.rs @@ -0,0 +1,357 @@ +//! With one ordinary job holding every frame it may, the machine's controls +//! still work, and what the exhaustion stopped starts again once the memory +//! is given back. +//! +//! This job runs in test-runner's, which `tests/images/floorcase.toml` gives a +//! floor: it stands in for the stop UI, a system holder. It starts a greedy +//! job of its own, an ordinary one, which commits every frame it may; then, +//! while that holds them: +//! +//! - a server in an ordinary job answers this job's connection, before any of +//! that, and its account does not move: a connection's rings are its +//! client's; +//! - an ordinary job's spawn is refused, so the exhaustion is real; +//! - the supervisor answers a stop's request on a new connection, which takes +//! nothing of the supervisor's own; +//! - `starveling`, an ordinary service, is asked to take memory and ends, +//! refused it; the supervisor is refused its restart, which it says in the +//! log, and keeps its port, where a new client's request waits; +//! - with `--input`, a key the host types reaches `inputsink` through +//! ps2server, whose floor holds the ring its connection's first write takes; +//! +//! and then it kills the greedy job, and the waiting request is answered by a +//! `starveling` the supervisor started again because memory came back. +//! +//! It needs the guest: what it holds is the kernel's accounting across jobs, +//! the supervisor's restart and the input path together, which no host +//! model has. It holds every frame an ordinary job may, so it has a boot of +//! its own: `memory_floor_holds_the_controls` on QEMU q35, and the +//! `memory_floor` metal row on the T14, without `--input`. + +use std::io::{BufRead, BufReader}; +use std::os::toyos::process::CommandExt; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use toyos::endow::{self, Endowments, SYSCAP_LABEL}; +use toyos::ipc::Connection; +use toyos::job::JOB_LABEL; +use toyos::poller::{Poller, READABLE}; +use toyos::power::{self, Refused}; +use toyos::syscap::SysCap; +use toyos_abi::syscall::{self, SyscallError, VmAt, VmProt}; + +const ME: &str = "/system/bin/test_rs_memory_floor"; +const PAGE: usize = 4096; + +/// `starveling`'s frames. +const PING: u32 = 1; +const SERVED: u32 = 2; +const STARVE: u32 = 3; +const RELEASED: u32 = 4; +const END: u32 = 5; + +/// `inputsink`'s frames on `keys`. +const HELLO: u32 = 1; +const KEY: u32 = 2; + +/// What the supervisor says when its restart of `starveling` is refused for +/// memory, after the pid (`system/supervisor`, `Supervisor::ended`). +const REFUSED_RESTART: &str = "ended and could not start again for want of memory"; + +/// How long each event is waited for before the test calls it lost: a hang +/// ceiling, never a timing claim. +const CEILING: Duration = Duration::from_secs(120); + +/// What the host waits for before it types: the job holds every frame. +const PRESS: &str = "memory_floor: every frame is held; press a key"; + +fn main() { + let args: Vec = std::env::args().collect(); + match args.get(1).map(String::as_str) { + Some("greedy") => greedy(), + Some("idle") => {} + Some("serve") => serve(), + Some("--input") => control(true), + None => control(false), + Some(other) => panic!("memory_floor: no arm {other}"), + } +} + +fn control(input: bool) { + let cap: SysCap = Endowments::get().take(SYSCAP_LABEL).expect("test-runner endows the estate's capability"); + let own = toyos::job::own().expect("a handle to this job"); + let account = own.account().expect("this job's account"); + assert!(account.floor > 0, "this job has no floor: the image gives test-runner one"); + println!(" memory_floor: this job's floor is {} KiB, and it holds {} KiB", account.floor >> 10, (account.committed + account.kernel) >> 10); + + // Every connection that crosses the exhaustion is made and used once + // before it, so neither end allocates a ring for it then. + let keys = input.then(|| { + let keys = endow::service("keys").expect("the image gives test-runner `keys`"); + keys.signal(HELLO).expect("hello to inputsink"); + assert_eq!(frame(&keys).0, HELLO, "inputsink's answer to hello"); + keys + }); + let starved = endow::service("starved").expect("the image gives test-runner `starved`"); + ping(&starved).expect("starveling served before anything is taken"); + + // The log from its first line, which logkeeper, on this image's floor, + // goes on handing over: where the supervisor's refusal is read. + let mut log = Log { pipe: logkeeper_api::read().expect("the log, before anything is taken").pipe, bytes: vec![0; 64 << 10], held: 0 }; + log.drain(); + + rings_are_the_clients(&cap); + + let job = cap.new_job().expect("an ordinary job for the greedy one"); + let mut greedy = Command::new(ME) + .arg("greedy") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .endow(JOB_LABEL, job.duplicate().expect("a second handle to its job").into_raw().0) + .spawn() + .expect("spawn the greedy job"); + let mut said = BufReader::new(greedy.stdout.take().expect("its stdout")); + let mut line = String::with_capacity(256); + for want in ["greedy: taking", "greedy: holding"] { + line.clear(); + said.read_line(&mut line).expect("a line of the greedy job's"); + assert!(line.starts_with(want), "the greedy job said {line:?}, and was to say {want:?}"); + } + print!(" {line}"); + let held = job.account().expect("the greedy job's account"); + println!(" memory_floor: the greedy job holds {} MiB committed", held.committed >> 20); + + // An ordinary job is refused: the exhaustion is the machine's. + let other = cap.new_job().expect("a job is a few bytes this job's floor holds"); + let refused = Command::new(ME) + .arg("idle") + .endow(JOB_LABEL, other.into_raw().0) + .spawn() + .map(|_| ()); + assert!( + refused.as_ref().is_err_and(|e| e.kind() == std::io::ErrorKind::OutOfMemory), + "an ordinary job's spawn with every frame held answered {refused:?}" + ); + println!(" PASS: an ordinary job's spawn is refused for memory"); + + // The stop's request, on a connection the supervisor accepts now. + assert_eq!(power::cancel(), Err(Refused::NoStopPending), "the supervisor's answer to a stop's cancel"); + println!(" PASS: the supervisor answers a stop's request"); + + // The restart: starveling is refused memory, gives back most of its + // stack, and ends. The greedy job takes what came back, so starveling's + // end frees less than another start of it needs, and the restart is + // refused. + let before = job.account().expect("the greedy job's account").committed; + starved.signal(STARVE).expect("ask starveling to take memory"); + let (msg_type, payload) = frame(&starved); + assert_eq!(msg_type, RELEASED, "starveling's answer to being refused memory"); + let given = u64::from_le_bytes(payload[..8].try_into().expect("how much it gave back")); + // Each line the greedy job says is a take. It has taken what came back + // once, after a take since starveling was asked, its job is not ready: + // what it was last refused is not free for it. + let taken = loop { + line.clear(); + said.read_line(&mut line).expect("a line of the greedy job's"); + assert!(line.starts_with("greedy: took "), "the greedy job said {line:?}"); + let taken = job.account().expect("the greedy job's account").committed.saturating_sub(before); + if taken > 0 && !ready_now(&job) { + break taken; + } + }; + println!(" memory_floor: the greedy job took back {} KiB, starveling having given back {} KiB", taken >> 10, given >> 10); + starved.signal(END).expect("tell starveling to end"); + assert!(starved.recv_header().is_err(), "starveling answered, and was to end"); + log.await_line(&["supervisor: starveling (pid ", REFUSED_RESTART]); + println!(" PASS: starveling ended, and its restart was refused for memory"); + // Asked of a port no process serves now: it waits for the next. + let waiting = endow::service("starved").expect("a connection waits on starveling's kept port"); + waiting.signal(PING).expect("a request to the starveling to come"); + + if let Some(keys) = &keys { + println!("{PRESS}"); + await_readable(keys, "key relayed by inputsink"); + let (msg_type, payload) = frame(keys); + assert_eq!(msg_type, KEY, "inputsink relayed {msg_type}, not a key"); + println!(" PASS: key {:#04x} reached the owner of input through ps2server", payload[0]); + } + + greedy.kill().expect("the system holder kills the greedy job"); + greedy.wait().expect("the greedy job's end"); + println!(" PASS: the greedy job is killed"); + + await_readable(&waiting, "starveling started again to answer"); + assert_eq!(frame(&waiting).0, SERVED, "the restarted starveling's answer"); + println!(" PASS: starveling serves again once the memory is given back"); + println!("memory_floor: the controls worked with every frame held, and the starved service came back"); +} + +/// A server in an ordinary job of its own accepts this job's connection and +/// answers it: both of the connection's rings are its client's, so the +/// server's reply costs its job nothing, and no client can spend a server's +/// floor by connecting to it. +fn rings_are_the_clients(cap: &SysCap) { + let (acceptor, connector) = toyos::port::create().expect("a port"); + let job = cap.new_job().expect("a job for the server"); + let mut server = Command::new(ME) + .arg("serve") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .endow(JOB_LABEL, job.duplicate().expect("a second handle to its job").into_raw().0) + .endow("acceptor", acceptor.into_raw().0) + .spawn() + .expect("spawn the server"); + let names = toyos::namespace::build().add("served", &connector).finish().expect("a namespace of the port"); + let conn = names.open("served").expect("connect to the server"); + let mut said = BufReader::new(server.stdout.take().expect("its stdout")); + let mut line = String::new(); + said.read_line(&mut line).expect("the server's line"); + assert_eq!(line.trim(), "serve: accepted"); + let before = job.account().expect("the server's account"); + ping(&conn).expect("the server answers"); + line.clear(); + said.read_line(&mut line).expect("the server's line"); + assert_eq!(line.trim(), "serve: answered"); + let after = job.account().expect("the server's account"); + assert_eq!(after.kernel, before.kernel, "the server's reply was charged to the server"); + server.kill().expect("end the server"); + server.wait().expect("the server's end"); + println!(" PASS: a connection's rings are its client's: the server's reply cost its job nothing"); +} + +/// Accept one connection on the endowed acceptor and answer its [`PING`]. +fn serve() -> ! { + let acceptor: toyos::port::Acceptor = Endowments::get().take("acceptor").expect("serve: an acceptor"); + let poller = Poller::new(1); + poller.watch(&acceptor, READABLE, 0); + poller.wait(1, u64::MAX, |_| {}); + let conn = acceptor.accept().expect("serve: the connection"); + println!("serve: accepted"); + assert_eq!(frame(&conn).0, PING, "serve: the request"); + conn.signal(SERVED).expect("serve: the answer"); + println!("serve: answered"); + let mut stdin = [0u8; 1]; + loop { + let _ = std::io::Read::read(&mut std::io::stdin(), &mut stdin); + } +} + +/// Commit every page a promise can still be made for, and take again what +/// comes back, each time its job is told memory it was refused is free, until +/// it is killed. +fn greedy() -> ! { + println!("greedy: taking every frame"); + // Before anything is taken: a poller is an inbox's ring of memory. + let job = toyos::job::own().expect("greedy: a handle to its own job"); + let poller = Poller::new(1); + println!("greedy: holding {} MiB, refused a page more", take() >> 20); + loop { + poller.watch(&job, READABLE, 0); + poller.wait(1, u64::MAX, |_| {}); + let took = take(); + if took > 0 { + println!("greedy: took {} KiB more", took >> 10); + } + } +} + +/// Every page a promise can still be made for, committed and never released: +/// how many bytes. +fn take() -> usize { + let mut committed = 0usize; + let mut len = 1usize << 40; + while len >= PAGE { + // SAFETY: placed anywhere, so it replaces nothing; never released. + match unsafe { syscall::vm_reserve(len, VmProt::ReadWrite, VmAt::ANYWHERE) } { + Ok(_) => committed += len, + Err(SyscallError::ResourceExhausted | SyscallError::InvalidArgument) => len /= 2, + Err(e) => panic!("greedy: a commitment of {len} bytes answered {e:?}"), + } + } + committed +} + +/// Whether `job` is readable now: what it was last refused is free for it. +fn ready_now(job: &toyos::job::Job) -> bool { + let poller = Poller::new(1); + poller.watch(job, READABLE, 0); + let mut woke = false; + poller.wait(1, 0, |_| woke = true); + woke +} + +/// [`PING`] answered [`SERVED`]. +fn ping(conn: &Connection) -> Result<(), String> { + conn.signal(PING).map_err(|e| format!("{e:?}"))?; + match frame(conn).0 { + SERVED => Ok(()), + other => Err(format!("answered {other}")), + } +} + +/// One whole frame off `conn`, waiting for it. +fn frame(conn: &Connection) -> (u32, Vec) { + let header = conn.recv_header().expect("a frame's header"); + let mut payload = vec![0u8; header.len() as usize]; + let n = conn.recv_bytes(&header, &mut payload).expect("a frame's payload"); + payload.truncate(n); + (header.msg_type, payload) +} + +/// Wait until `conn` has something to read, within [`CEILING`]. +fn await_readable(conn: &Connection, what: &str) { + let poller = Poller::new(1); + poller.watch(conn, READABLE, 0); + let mut woke = false; + poller.wait(1, CEILING.as_nanos() as u64, |_| woke = true); + assert!(woke, "{} s passed with no {what}", CEILING.as_secs()); +} + +/// The log as logkeeper hands it over, read into a buffer made before +/// anything is taken. +struct Log { + pipe: toyos::Pipe, + bytes: Vec, + /// Bytes of an unfinished line kept at the buffer's start. + held: usize, +} + +impl Log { + /// Read what has been handed over so far, keeping none of it. + fn drain(&mut self) { + while self.ready(Duration::ZERO) { + let n = self.pipe.read(&mut self.bytes).expect("read the log"); + assert!(n > 0, "logkeeper let the log go"); + } + } + + /// Whether a read would find bytes within `bound`. + fn ready(&self, bound: Duration) -> bool { + let poller = Poller::new(1); + poller.watch(&self.pipe, READABLE, 0); + let mut woke = false; + poller.wait(1, bound.as_nanos() as u64, |_| woke = true); + woke + } + + /// Wait, within [`CEILING`], for a line holding every one of `parts`. + fn await_line(&mut self, parts: &[&str]) { + let until = Instant::now() + CEILING; + loop { + let left = until.saturating_duration_since(Instant::now()); + assert!(self.ready(left), "{} s passed with no line of {parts:?} in the log", CEILING.as_secs()); + let n = self.pipe.read(&mut self.bytes[self.held..]).expect("read the log"); + assert!(n > 0, "logkeeper let the log go"); + let end = self.held + n; + let last = self.bytes[..end].iter().rposition(|&b| b == b'\n').map_or(0, |at| at + 1); + let whole = String::from_utf8_lossy(&self.bytes[..last]); + if whole.lines().any(|line| parts.iter().all(|part| line.contains(part))) { + return; + } + self.bytes.copy_within(last..end, 0); + self.held = end - last; + assert!(self.held < self.bytes.len(), "a line of the log longer than {} bytes", self.bytes.len()); + } + } +} diff --git a/tests/guest/starveling/Cargo.toml b/tests/guest/starveling/Cargo.toml new file mode 100644 index 00000000000..fe7ba2d226b --- /dev/null +++ b/tests/guest/starveling/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "starveling" +description = "Test-only: a service that ends when it is refused memory and serves again once started, so a test can hold that a restart refused for memory is retried when memory comes back." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dependencies] +toyos-abi = { path = "../../../sdk/abi" } +toyos = { path = "../../../sdk/toyos" } diff --git a/tests/guest/starveling/src/main.rs b/tests/guest/starveling/src/main.rs new file mode 100644 index 00000000000..2da85c7d5cb --- /dev/null +++ b/tests/guest/starveling/src/main.rs @@ -0,0 +1,89 @@ +//! Test-only: a service on `starved` that answers [`PING`] with [`SERVED`], +//! and on [`STARVE`] asks for memory. Refused it, as on a machine another job +//! has taken, it gives back what it can of its own stack, answers +//! [`RELEASED`] with how much, and ends, exit 3, on the next [`END`]: so +//! what its end frees is less than another start of it needs, whoever took +//! what it gave back. Granted, the premise is gone, and it ends exit 4. + +use toyos::endow; +use toyos::ipc::{Connection, FrameRx, RxStep}; +use toyos::poller::{Poller, READABLE}; +use toyos_abi::syscall::{self, SyscallError, VmAt, VmProt}; + +const PING: u32 = 1; +const SERVED: u32 = 2; +const STARVE: u32 = 3; +const RELEASED: u32 = 4; +const END: u32 = 5; + +/// One commitment of what the test's greedy job left nothing of. +const ASKED: usize = 64 << 20; +const PAGE: usize = 4096; +/// What of the 8 MiB main stack it gives back: everything but the top +/// [`KEPT`] below where it runs, which this program never reaches. +const GIVEN: usize = 6 << 20; +const KEPT: usize = 256 << 10; + +const TOKEN_ACCEPT: u64 = 0; +const TOKEN_CLIENT: u64 = 1; + +fn main() { + let acceptor = endow::acceptor("starved").expect("starveling: the manifest declares it serves `starved`"); + let mut client: Option<(Connection, FrameRx<0>)> = None; + let poller = Poller::new(2); + println!("starveling: serving"); + loop { + poller.watch(&acceptor, READABLE, TOKEN_ACCEPT); + if let Some((conn, _)) = &client { + poller.watch(conn, READABLE, TOKEN_CLIENT); + } + let mut ready = [false; 2]; + poller.wait(1, u64::MAX, |token| ready[token as usize] = true); + if ready[TOKEN_ACCEPT as usize] { + if let Ok(conn) = acceptor.accept() { + client = Some((conn, FrameRx::new())); + } + } + if !ready[TOKEN_CLIENT as usize] { + continue; + } + let Some((conn, rx)) = &mut client else { continue }; + loop { + match rx.pump(conn) { + RxStep::Frame { msg_type: PING, .. } => { + let _ = conn.try_signal(SERVED); + } + RxStep::Frame { msg_type: STARVE, .. } => starve(conn), + RxStep::Frame { msg_type: END, .. } => { + println!("starveling: ending, refused memory"); + std::process::exit(3); + } + RxStep::Frame { .. } | RxStep::Idle => break, + RxStep::Eof | RxStep::Malformed => { + client = None; + break; + } + } + } + } +} + +/// Ask for [`ASKED`]; refused, give back [`GIVEN`] of the stack and say so. +fn starve(conn: &Connection) { + // SAFETY: placed anywhere, so it replaces nothing. + match unsafe { syscall::vm_reserve(ASKED, VmProt::ReadWrite, VmAt::ANYWHERE) } { + Err(SyscallError::ResourceExhausted) => {} + answer => { + println!("starveling: {ASKED} bytes answered {:?}, which nothing left should have", answer.map(|_| ())); + std::process::exit(4); + } + } + let here = core::ptr::addr_of!(conn) as usize; + let top = (here - KEPT) & !(PAGE - 1); + // SAFETY: the pages below `top` are this thread's own stack, which it + // never reaches again; a touch there faults as an overflow, not as + // another range's memory. + unsafe { syscall::vm_decommit((top - GIVEN) as *mut u8, GIVEN) }.expect("starveling: give back its stack"); + println!("starveling: refused {ASKED} bytes; gave back {GIVEN} of its stack"); + let _ = conn.try_send_bytes(RELEASED, &(GIVEN as u64).to_le_bytes()); +} diff --git a/tests/images/floorcase.toml b/tests/images/floorcase.toml new file mode 100644 index 00000000000..a58a1fe20cb --- /dev/null +++ b/tests/images/floorcase.toml @@ -0,0 +1,83 @@ +# The boot `memory_floor` runs alone: one ordinary job holds every frame it +# may, and the control paths keep working: ps2server and test-runner on their +# floors, which no other job's allocation can take. The supervisor's floor +# mirrors the shipped images' shape at a smaller size; the test claims nothing +# for it. +# +# test-runner stands in for the stop UI, a system holder with a floor: its +# job kills the greedy job. `inputsink` stands in for the owner of input, and +# `starveling` is the ordinary service the exhaustion ends, which the +# supervisor starts again once the memory is given back. Neither has a floor. + +[boot] +start = ["logkeeper", "acpiserver", "diskserver", "fileserver", "ps2server", "inputsink", "starveling", "test-runner"] + +[supervisor] +memory-floor-mib = 64 + +# `every_boot_config_runs_logkeeper` is what refuses a boot config without it. +# Its floor is this test's, not `images/system.toml`'s: the job reads the +# supervisor's refusal off the log while every frame is held, so the log has +# to keep being handed over. +[programs.logkeeper] +service = true +syscap = ["logread"] +serves = ["log"] +memory-floor-mib = 32 + +# The machine's ACPI fixed hardware, as `images/system.toml` serves it: the +# boot's last job hands the machine back to firmware through it. +[programs.acpiserver] +service = true +devices = ["acpi"] +receives = ["power"] + +# The i8042, an input server on its floor as `images/system.toml`'s is. +[programs.ps2server] +restart = true +receives = ["input"] +devices = ["isa:0060,0064:1,12"] +memory-floor-mib = 24 + +[programs.inputsink] +service = true +serves = ["input", "keys"] + +[programs.starveling] +service = true +restart = true +serves = ["starved"] + +# `dup` and `device` as every test estate's; `jobs` because the job starts its +# greedy job as an ordinary job of its own; `power` for the stop it asks and +# the boot's last job; `log` for the supervisor's refusal. +[programs.test-runner] +receives = ["power", "keys", "starved", "log"] +syscap = ["device", "dup", "power", "jobs"] +memory-floor-mib = 64 + +[programs.toybox] + +[symlinks] +"bin/reboot" = "/system/bin/toybox" + +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +[programs.nvme] +drives = ["pci:01:08:02"] +receives = ["disk"] + +# The USB driver: QEMU's `nec-usb-xhci`, and the controller firmware booted +# through, which on a bench is the one holding the stick. No floor: it drives +# disks too, so it serves ordinary work. +[programs.usbd] +drives = ["pci:0c:03:30/1033:0194", "pci:0c:03:30@boot"] +receives = ["disk"] + +[programs.fileserver] +restart = true +roles = ["data", "log", "boot"] +receives = ["block"] diff --git a/tests/toyos.rs b/tests/toyos.rs index d5afe06f454..d0fdd743c06 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -170,6 +170,10 @@ const RUST_SKIP: &[&str] = &[ // of it: `kernel_objects_refused_alone` runs it on QEMU and the // `kernel_objects_refused` metal row on the T14. "kernel_objects_refused", + // It holds every frame an ordinary job may, and needs the floors of + // `tests/images/floorcase.toml`: `memory_floor_holds_the_controls` runs it + // on QEMU and the `memory_floor` metal row on the T14. + "memory_floor", // Fills /tmp to the VFS listing limit, so it needs a boot nothing else // shares — every later `read_dir("/tmp")` in it would be refused. // `readdir_bound` gives it one. @@ -654,6 +658,16 @@ const MACHINE_TESTS: &[&str] = &[ // refused meanwhile. QEMU establishes the refusals and that the machine // goes on; nothing in it is timing. "kernel_objects_refused_alone", + // With one ordinary job holding every frame it may, the supervisor answers + // a stop's request, a key reaches the owner of input through ps2server, a + // system holder kills the greedy job, and a service the exhaustion ended is + // serving again once the memory is given back: the kernel's accounts and + // floors, the supervisor's restart and the input path together, which no + // host model has. On the two-CPU, 4 GiB q35 guest in a boot of its own, + // `tests/images/floorcase.toml`; QEMU types the key. Order and completion + // only: nothing in it is timing. The `memory_floor` metal row runs it on the + // T14 without the key. + "memory_floor_holds_the_controls", // The parity instrument's two builds in one boot, behind ring's known // answers: that every row runs and both builds print the same whole table. // Its timings decide nothing here; the `crypto_instrument` metal row takes @@ -742,6 +756,10 @@ const METAL: &[(&str, metal::Metal)] = &[ judge: |b| b[0].job_passed("test_rs_kernel_objects_refused"), }, ), + ( + "memory_floor", + metal::Metal { arms: FLOORCASE, judge: |b| b[0].job_passed("test_rs_memory_floor") }, + ), ( "readdir_bound", metal::Metal { @@ -1669,6 +1687,12 @@ const TESTCASES_HELD: &[metal::Arm] = &[metal::Arm { const TESTCASES_REFUSED: &[metal::Arm] = &[metal::once("testcases-refused", "tests/images/testcases.toml", &[], &["test_rs_kernel_objects_refused"])]; +/// Its own boot: `memory_floor` holds every frame an ordinary job may, on +/// `tests/images/floorcase.toml`'s floors. No key is typed on the bench, so +/// the input arm is QEMU's alone. +const FLOORCASE: &[metal::Arm] = + &[metal::once("floorcase", "tests/images/floorcase.toml", &[], &["test_rs_memory_floor"])]; + /// Its own boot: `readdir_bound` fills `/tmp` to the VFS listing limit, and /// every later `read_dir("/tmp")` in a boot it shared would be refused. const TESTCASES_READDIR: &[metal::Arm] = @@ -4392,6 +4416,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "vector_state_survives_a_switch" => vector_state_survives_a_switch(test_config), "std_threads_count_the_cpus_and_give_back_ended_threads" => std_threads_count_the_cpus_and_give_back_ended_threads(test_config), "kernel_objects_refused_alone" => kernel_objects_refused_alone(test_config), + "memory_floor_holds_the_controls" => memory_floor_holds_the_controls(), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), "crypto_instrument_prints_its_table" => crypto_instrument_prints_its_table(test_config), "the_compositor_drives_the_virtio_gpu" => the_compositor_drives_the_virtio_gpu(), @@ -5168,6 +5193,60 @@ fn kernel_objects_refused_alone(test_config: &Path) -> Result<(), String> { Ok(()) } +/// `memory_floor --input` alone in `tests/images/floorcase.toml`'s boot, a key +/// typed once every frame is held: the job holds what the controls did, and the +/// supervisor said it was refused the starved service's restart, then started +/// it once memory came back. +fn memory_floor_holds_the_controls() -> Result<(), String> { + const JOB: &str = "memory_floor"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options( + &compile::repo_root().join("tests/images/floorcase.toml"), + &[], + &[(JOB.to_string(), bin)], + BootOptions { profile: qemu::Profile::Metal, qmp: true, ..Default::default() }, + ); + let mut console = format!("{}\n", qemu.boot_log()); + await_marker(&mut qemu, &mut console, ps2server::ARMED_BOTH, "ps2server to arm the i8042")?; + let from = console.len(); + writeln!(qemu.stdin_mut(), "run test_rs_{JOB} --input").map_err(|e| e.to_string())?; + qemu.flush_stdin(); + let said = |console: &str, why: String| format!("{why}\nthe boot said:\n{}", &console[from..]); + await_marker_new(&mut qemu, &mut console, MEMORY_FLOOR_PRESS, from, "the greedy job to hold every frame").map_err(|why| said(&console, why))?; + { + let mut input = qemu::QmpInput::open(qemu.qmp_socket()); + input.keys(&[("a", true)]); + input.keys(&[("a", false)]); + } + await_marker_new(&mut qemu, &mut console, &format!("===TEST_END test_rs_{JOB} exit=0==="), from, "the job's end").map_err(|why| said(&console, why))?; + let said = &console[from..]; + for line in [ + "could not start again for want of memory", + "supervisor: starveling started again as pid ", + ] { + if !said.contains(line) { + return Err(format!("the supervisor never said {line:?}\n{said}")); + } + } + // Whatever else the exhaustion ended, a file server among them, starts + // again once the memory is given back. + for refused in said.lines().filter(|l| l.contains("could not start again for want of memory")) { + let Some(service) = refused.split("supervisor: ").nth(1).and_then(|rest| rest.split(" (pid").next()) else { continue }; + if !said.contains(&format!("supervisor: {service} started again as pid ")) { + return Err(format!("{service} was refused its restart for memory and never started again\n{said}")); + } + eprintln!(" [floor] {service} was refused its restart for memory and started again"); + } + for line in said.lines().filter(|l| l.contains("PASS: ") || l.contains("memory_floor: ") || l.contains("supervisor: starveling")) { + eprintln!(" [floor] {}", line.trim()); + } + serial::Serial::named("the memory_floor boot", console).must_be_clean() +} + +/// What `memory_floor` says once its greedy job holds every frame, the host's +/// cue to type (`tests/guest/rust-tests/src/bin/memory_floor.rs`). +const MEMORY_FLOOR_PRESS: &str = "memory_floor: every frame is held; press a key"; + /// Ring's known answers, then both builds of the parity instrument, in one /// boot: each ends cleanly and their lines read back as one whole table. fn crypto_instrument_prints_its_table(test_config: &Path) -> Result<(), String> {