diff --git a/.github/workflows/crates.yml b/.github/workflows/crates.yml index cfa4d26b7..856e348d8 100644 --- a/.github/workflows/crates.yml +++ b/.github/workflows/crates.yml @@ -49,7 +49,7 @@ jobs: set -o pipefail actual="$(cargo metadata --no-deps --format-version 1 \ | jq -r '[.packages[] | select(.publish == null) | .name] | sort | join(" ")')" - if [ "$actual" != "powerio powerio-cli powerio-core powerio-dist powerio-matrix powerio-prob powerio-synth powerio-tnd powerio-tx" ]; then + if [ "$actual" != "powerio powerio-archive powerio-cli powerio-core powerio-dist powerio-matrix powerio-prob powerio-synth powerio-tnd powerio-tx" ]; then echo "publishable workspace members ($actual) do not match the publish list in crates.yml" >&2 exit 1 fi @@ -61,7 +61,7 @@ jobs: UA: powerio release workflow (https://github.com/eigenergy/powerio) OWNER: samtalki run: | - for crate in powerio-core powerio-tx powerio-dist powerio-tnd powerio-prob powerio-synth powerio-matrix powerio powerio-cli; do + for crate in powerio-core powerio-archive powerio-tx powerio-dist powerio-tnd powerio-prob powerio-synth powerio-matrix powerio powerio-cli; do resp="$RUNNER_TEMP/$crate-owners.json" code="$(curl -sS -A "$UA" --max-time 30 -o "$resp" -w '%{http_code}' \ "https://crates.io/api/v1/crates/$crate/owners")" || code=unreachable @@ -123,7 +123,7 @@ jobs: # skip-existing), so a re-run after a partial publish only publishes # what's missing. run: | - for crate in powerio-core powerio-tx powerio-dist powerio-tnd powerio-prob powerio-synth powerio-matrix powerio powerio-cli; do + for crate in powerio-core powerio-archive powerio-tx powerio-dist powerio-tnd powerio-prob powerio-synth powerio-matrix powerio powerio-cli; do resp="$RUNNER_TEMP/$crate.json" code="$(curl -sS -A "$UA" --max-time 30 -o "$resp" -w '%{http_code}' \ "https://crates.io/api/v1/crates/$crate/$VERSION")" || code=unreachable diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a9b7789fe..721c355cc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,7 @@ jobs: - name: Build docs env: RUSTDOCFLAGS: "-D warnings" - run: cargo doc --no-deps --all-features -p powerio-tnd -p powerio -p powerio-core -p powerio-tx -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-dist -p powerio-capi -p powerio-py -p powerio-wasm + run: cargo doc --no-deps --all-features -p powerio-tnd -p powerio -p powerio-core -p powerio-archive -p powerio-tx -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-dist -p powerio-capi -p powerio-py -p powerio-wasm - name: Build Python API docs run: | python -m pip install --upgrade pip @@ -143,6 +143,7 @@ jobs:
released / docs.rs
powerio powerio-core + powerio-archive powerio-tx powerio-dist powerio-prob @@ -152,6 +153,7 @@ jobs:
development / main
powerio powerio-core + powerio-archive powerio-tx powerio-dist powerio-synth diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 37ae0e559..1dcdfba5c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -182,7 +182,7 @@ jobs: # Bare `cargo test` only covers the default-members; name the C ABI crate # explicitly so its end-to-end ABI tests run in CI too. - name: Run tests - run: cargo test -p powerio -p powerio-tx -p powerio-core -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-cli -p powerio-capi -p powerio-dist -p powerio-tnd --verbose + run: cargo test -p powerio -p powerio-tx -p powerio-core -p powerio-archive -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-cli -p powerio-capi -p powerio-dist -p powerio-tnd --verbose - name: Test problem matrix feature run: cargo test -p powerio --features matrix --verbose - name: Test gridfm reader feature @@ -265,7 +265,7 @@ jobs: - name: Check wasm32 component crates run: | cargo check --target wasm32-unknown-unknown -p powerio -p powerio-tx -p powerio-dist \ - -p powerio-core -p powerio-prob -p powerio-matrix + -p powerio-core -p powerio-archive -p powerio-prob -p powerio-matrix cargo check --target wasm32-unknown-unknown -p powerio --features matrix # A browser consumer that sets `default-features = false` builds with no C # compiler: SQLite enters only through the default-on `sincal` feature. diff --git a/AGENTS.md b/AGENTS.md index a47ca1aa8..320eda3de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,9 @@ check the current source, tests, and release notes. `Diagnostic`, `Error`, `PioModule`, common module records, `TimeSeries`, `ScenarioSet`, and output destination types. It has no electrical network, matrix, or solver dependencies. +- **`powerio-archive`**: bounded in-memory ZIP transport over core, with explicit + limits, namespace validation, and lazy verified reads; no electrical types + or case discovery. - **`powerio-tx`**: the format neutral `BalancedNetwork`, the balanced format readers and writers, normalization, and derived indexed views. - **`powerio-tnd`**: integrated transmission/distribution composition, explicit @@ -161,8 +164,8 @@ release on its own. its platform. Only then does it stage a **draft** GitHub release with the five assets. 4. A human inspects and publishes the draft. Publishing starts `python.yml` - (PyPI) and `crates.yml` (crates.io: `powerio-core`, `powerio-tx`, `powerio-dist`, - `powerio-tnd`, `powerio-prob`, `powerio-matrix`, `powerio`, and + (PyPI) and `crates.yml` (crates.io: `powerio-core`, `powerio-archive`, + `powerio-tx`, `powerio-dist`, `powerio-tnd`, `powerio-prob`, `powerio-matrix`, `powerio`, and `powerio-cli`, in dependency order). Both deploy through the `pypi` and `crates-io` environments, which wait for the maintainer's approval. A rerun skips files and versions already published. A crate's first @@ -183,6 +186,7 @@ release on its own. ## Layout ``` +powerio-archive/src lib.rs (ZIP index/cache/limits), preflight.rs, local.rs powerio-core/src module.rs (PioModule), source.rs, output.rs, diagnostic.rs, codes.rs, error.rs, records.rs, time_series.rs, scenario.rs, component_id.rs, diff --git a/CHANGELOG.md b/CHANGELOG.md index 769750923..1bac64c14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Add `powerio-archive`: bounded ZIP indexing and verified member reads with + explicit limits and namespace checks, as shared infrastructure for #774. + ## 0.11.7 **AC optimal transmission switching** diff --git a/Cargo.lock b/Cargo.lock index 5907f85ed..d6c035e1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2431,6 +2431,14 @@ dependencies = [ "zip", ] +[[package]] +name = "powerio-archive" +version = "0.11.7" +dependencies = [ + "powerio-core", + "zip", +] + [[package]] name = "powerio-capi" version = "0.11.7" diff --git a/Cargo.toml b/Cargo.toml index f7717405e..901c2d492 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "powerio", "powerio-tx", "powerio-core", + "powerio-archive", "powerio-tnd", "powerio-matrix", "powerio-synth", @@ -24,6 +25,7 @@ default-members = [ "powerio", "powerio-tx", "powerio-core", + "powerio-archive", "powerio-tnd", "powerio-matrix", "powerio-synth", @@ -55,6 +57,7 @@ powerio = { path = "powerio", version = "0.11.7" } # Model crates remain independent; the facade enables optional format adapters. powerio-tx = { path = "powerio-tx", version = "0.11.7", default-features = false } powerio-core = { path = "powerio-core", version = "0.11.7" } +powerio-archive = { path = "powerio-archive", version = "0.11.7" } powerio-matrix = { path = "powerio-matrix", version = "0.11.7" } powerio-synth = { path = "powerio-synth", version = "0.11.7" } powerio-prob = { path = "powerio-prob", version = "0.11.7" } diff --git a/README.md b/README.md index 4588aaadd..4f6944ff9 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ depend on just the layer they use. |---|---| | `powerio` | `PioValue`, `parse`, `emit`, `serialize`, `deserialize` | | `powerio-core` | `Source`, `PioModule`, diagnostics, collections, destinations | +| `powerio-archive` | Bounded ZIP indexing and verified member reads | | `powerio-tx` | `BalancedNetwork` and balanced format readers and writers | | `powerio-dist` | `MulticonductorNetwork` and OpenDSS, PMD, BMOPF, CIM converters | | `powerio-prob` | Operating points, updates, calculation instances and solutions | diff --git a/docs/diagrams/architecture.dot b/docs/diagrams/architecture.dot index 0c3c7c293..4f819cf60 100644 --- a/docs/diagrams/architecture.dot +++ b/docs/diagrams/architecture.dot @@ -36,6 +36,8 @@ digraph powerio_architecture { "powerio-core" [label = "powerio-core\nSource, Diagnostic, Error,\nPioModule, series, output"]; "powerio-tnd" [label = "powerio-tnd\nIntegratedNetwork + transforms\nSINCAL adapters"]; + "powerio-archive" [label = "powerio-archive\nbounded ZIP transport"]; + "powerio-archive" -> "powerio-core"; "powerio-synth" -> "powerio-tx"; "powerio-tx" -> "powerio-core"; "powerio-tnd" -> "powerio-tx"; diff --git a/docs/src/assets/architecture.svg b/docs/src/assets/architecture.svg index 7002ebfc7..ca4f029ba 100644 --- a/docs/src/assets/architecture.svg +++ b/docs/src/assets/architecture.svg @@ -1,238 +1,249 @@ - + powerio_architecture - + cluster_consumers - -representative consumers + +representative consumers solvers - -solvers -(Tellegen, ExaModelsPower, -BMOPFTools) + +solvers +(Tellegen, ExaModelsPower, +BMOPFTools) - + powerio-prob - -powerio-prob -operating points, 7 instances, -8 solutions + +powerio-prob +operating points, 7 instances, +8 solutions - + solvers->powerio-prob - - -consume instances + + +consume instances gridfm_ml - -GridFM pipeline + +GridFM pipeline powerio-matrix - -powerio-matrix -sparse matrices, graph data + +powerio-matrix +sparse matrices, graph data - + gridfm_ml->powerio-matrix - - -Parquet datasets + + +Parquet datasets powerio-cli - -powerio-cli -CLI + TUI + +powerio-cli +CLI + TUI powerio - -powerio (facade) -parse, emit, serialize, deserialize, -PioValue + +powerio (facade) +parse, emit, serialize, deserialize, +PioValue - + powerio-cli->powerio - - + + powerio-capi - -powerio-capi -C ABI 7 (Julia, C, C++) + +powerio-capi +C ABI 7 (Julia, C, C++) - + powerio-capi->powerio - - + + powerio-py - -powerio-py -Python package + MCP + +powerio-py +Python package + MCP - + powerio-py->powerio - - + + powerio-wasm - -powerio-wasm -browser conversion + +powerio-wasm +browser conversion - + powerio-wasm->powerio - - + + - + powerio->powerio-matrix - - -matrix feature + + +matrix feature - + powerio->powerio-prob - - + + - + powerio-tx - -powerio-tx -BalancedNetwork -+ balanced formats + +powerio-tx +BalancedNetwork ++ balanced formats - + powerio->powerio-tx - - + + - + powerio-dist - -powerio-dist -MulticonductorNetwork -+ OpenDSS, PMD, BMOPF + +powerio-dist +MulticonductorNetwork ++ OpenDSS, PMD, BMOPF - + powerio->powerio-dist - - + + - + powerio-core - -powerio-core -Source, Diagnostic, Error, -PioModule<T>, series, output + +powerio-core +Source, Diagnostic, Error, +PioModule<T>, series, output - + powerio->powerio-core - - + + - + powerio-tnd - -powerio-tnd -IntegratedNetwork + transforms -SINCAL adapters + +powerio-tnd +IntegratedNetwork + transforms +SINCAL adapters - + powerio->powerio-tnd - - + + - + +powerio-synth + +powerio-synth +deterministic topology fixtures + + +powerio-matrix->powerio-synth + + + + powerio-matrix->powerio-prob - - + + - + powerio-matrix->powerio-tx - - + + - + powerio-matrix->powerio-dist - - + + - + powerio-matrix->powerio-core - - + + - + +powerio-synth->powerio-tx + + + + powerio-prob->powerio-tx - - + + - + powerio-prob->powerio-dist - - + + - + powerio-prob->powerio-core - - + + - + powerio-tx->powerio-core - - + + - + powerio-dist->powerio-core - - + + - + powerio-tnd->powerio-tx - - + + - + powerio-tnd->powerio-dist - - + + - + powerio-tnd->powerio-core - - + + - -powerio-synth - -powerio-synth -deterministic topology fixtures - -powerio-matrix->powerio-synth - - + +powerio-archive + +powerio-archive +bounded ZIP transport - - -powerio-synth->powerio-tx - - + +powerio-archive->powerio-core + + diff --git a/docs/src/crate-graph.md b/docs/src/crate-graph.md index 1ab5f359f..6d7f3c469 100644 --- a/docs/src/crate-graph.md +++ b/docs/src/crate-graph.md @@ -7,6 +7,7 @@ network crates do not depend on each other. ```text powerio-core Source, FormatId, Diagnostic, Error, PioModule, TimePoint, TimeSeries, ScenarioSet, Destination +├── powerio-archive bounded ZIP indexing and verified reads; no case discovery ├── powerio-tx BalancedNetwork and the balanced readers and writers └── powerio-dist MulticonductorNetwork and the OpenDSS, PMD, BMOPF, and CIM converters diff --git a/powerio-archive/Cargo.toml b/powerio-archive/Cargo.toml new file mode 100644 index 000000000..d67e5effd --- /dev/null +++ b/powerio-archive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "powerio-archive" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +description = "Bounded in-memory ZIP transport for PowerIO inputs." +readme = "README.md" +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation = "https://docs.rs/powerio-archive" +keywords = ["zip", "archive", "powerio"] +categories = ["compression", "parsing"] + +[dependencies] +powerio-core.workspace = true +zip.workspace = true + +[lints] +workspace = true diff --git a/powerio-archive/LICENSE-APACHE b/powerio-archive/LICENSE-APACHE new file mode 100644 index 000000000..57bfa63a0 --- /dev/null +++ b/powerio-archive/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. diff --git a/powerio-archive/LICENSE-MIT b/powerio-archive/LICENSE-MIT new file mode 100644 index 000000000..6505c9a15 --- /dev/null +++ b/powerio-archive/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright (c) 2026 eigenergy contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/powerio-archive/README.md b/powerio-archive/README.md new file mode 100644 index 000000000..f70c22de1 --- /dev/null +++ b/powerio-archive/README.md @@ -0,0 +1,30 @@ +# powerio-archive + +Bounded in-memory ZIP indexing and member reads. This crate depends on +`powerio-core` and ZIP support, with no electrical model or format discovery. + +`Archive::open` validates the complete index, namespace, local headers, and +declared expansion budgets before any member is decompressed. `Archive::read` +validates the inflated size and CRC before caching bytes. Repeated reads share +one immutable buffer. Input and limits are explicit; no filesystem, environment, +network, or extraction operation is used. + +The supported profile is single-volume ZIP32/ZIP64 with UTF-8 paths and stored +or deflated members. Self-extracting prefixes, ambiguous end records in metadata, +encrypted entries, special files, path collisions, and unsafe paths are refused. +Nested containers remain opaque bytes. This crate neither recurses into them nor +selects case files. Index validation does not certify unread payload CRCs. + +Names preserve exact UTF-8 bytes with slash separators; Unicode normalization +and OS path lookup are not performed. Lowercased name/ancestor collisions, +file/directory conflicts, Windows device names, forbidden characters and +trailing dots/spaces are refused. Paths are capped at 4096 bytes and 64 segments +(with core's per-segment bound). Export to a filesystem still needs the +destination's own validation. + +Defaults cap compressed input and cumulative declared expansion at 64 MiB each, +entries at 4096, and each member's expansion ratio at 200:1. Callers can supply +other positive platform-sized limits and a smaller per-read cap. Cached bytes +cannot exceed the declared expansion cap; transient decoder/index allocations +are additionally bounded by input and entry counts. These caps do not bound +downstream decoded tables, models, or repeated parser work. diff --git a/powerio-archive/src/lib.rs b/powerio-archive/src/lib.rs new file mode 100644 index 000000000..92fd05140 --- /dev/null +++ b/powerio-archive/src/lib.rs @@ -0,0 +1,407 @@ +//! Bounded, model-neutral ZIP transport. See [`Archive`] for validation stages. + +use std::collections::BTreeMap; +use std::io::{Cursor, Read}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use powerio_core::{ArtifactPath, Error}; +use zip::{ + CompressionMethod, ZipArchive, + read::{ArchiveOffset, Config}, +}; + +mod local; +mod preflight; + +/// Stable diagnostic identities for archive transport. +pub mod codes { + powerio_core::diagnostic_codes! { + INVALID_LIMIT = "REQUEST.ARCHIVE.INVALID_LIMIT", Error, + "archive limits must be positive and fit this platform", category = Request; + MALFORMED = "READ.ARCHIVE.MALFORMED", Error, + "ZIP metadata or payload integrity is invalid", category = Data; + UNSAFE = "READ.ARCHIVE.UNSAFE", Error, + "the archive namespace or member layout is unsafe", category = Data; + LIMIT = "READ.ARCHIVE.LIMIT", Error, + "the archive exceeds an explicit resource limit", category = Data; + UNSUPPORTED = "READ.ARCHIVE.UNSUPPORTED", Error, + "the archive uses an unsupported transport feature", category = Data; + UNKNOWN_ENTRY = "REQUEST.ARCHIVE.UNKNOWN_ENTRY", Error, + "the selected archive member does not exist", category = Request; + } +} + +type Result = std::result::Result; +fn malformed(message: impl Into) -> Error { + Error::new(&codes::MALFORMED, message) +} +fn unsafe_archive(message: impl Into) -> Error { + Error::new(&codes::UNSAFE, message) +} +fn limit(message: impl Into) -> Error { + Error::new(&codes::LIMIT, message) +} +fn unsupported(message: impl Into) -> Error { + Error::new(&codes::UNSUPPORTED, message) +} + +/// Explicit input, entry-count, cumulative expansion, and per-member ratio caps. +/// Defaults: 64 MiB input/expanded bytes, 4096 entries, expansion ratio 200:1. +/// These limits do not read the environment or bound downstream model memory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ArchiveLimits { + input: u64, + entries: usize, + expanded: u64, + ratio: u64, +} + +impl Default for ArchiveLimits { + fn default() -> Self { + Self { + input: 64 << 20, + entries: 4096, + expanded: 64 << 20, + ratio: 200, + } + } +} +impl ArchiveLimits { + /// Set positive limits. Byte counts and the index capacity must fit this + /// platform. Paths additionally have fixed 4096-byte/64-segment bounds. + pub fn new(input: u64, entries: usize, expanded: u64, ratio: u64) -> Result { + if input == 0 + || entries == 0 + || expanded == 0 + || ratio == 0 + || isize::try_from(input).is_err() + || isize::try_from(expanded).is_err() + || entries + .checked_mul(std::mem::size_of::()) + .is_none_or(|n| n > isize::MAX as usize) + { + return Err(Error::new( + &codes::INVALID_LIMIT, + "archive limits must be positive and platform sized", + )); + } + Ok(Self { + input, + entries, + expanded, + ratio, + }) + } +} + +/// An indexed member. Names retain their exact UTF-8 spelling; directories +/// have their terminal slash removed. Indexing does not validate payload CRCs. +#[derive(Clone, Debug)] +pub struct ArchiveEntry { + path: ArtifactPath, + directory: bool, + size: u64, + compressed: u64, + index: usize, + flags: u16, + method: u16, + crc: u32, + header: usize, + data: usize, +} +impl ArchiveEntry { + #[must_use] + pub fn path(&self) -> &ArtifactPath { + &self.path + } + #[must_use] + pub const fn is_directory(&self) -> bool { + self.directory + } + #[must_use] + pub const fn size(&self) -> u64 { + self.size + } + #[must_use] + pub const fn compressed_size(&self) -> u64 { + self.compressed + } +} + +/// An immutable ZIP input with a validated index and lazily verified members. +/// +/// Opening validates namespace, counts, declared expansion, local headers, +/// descriptors, and nonoverlapping member regions before decompressing files. +/// Reading validates actual size and CRC before publishing/caching bytes. +/// Repeated reads return the same immutable allocation. Nothing is extracted, +/// executed, discovered as a case, or recursively opened as another archive. +/// +/// The supported profile is single-volume ZIP32/ZIP64 with stored/deflated +/// members, UTF-8 paths and no prepended executable. Ambiguous metadata end +/// records, encryption, symlinks, special files and unsafe paths are refused. +#[derive(Debug)] +pub struct Archive { + bytes: Arc<[u8]>, + zip: ZipArchive, + entries: BTreeMap, + cache: BTreeMap>, +} +impl Archive { + /// Index caller-owned bytes under explicit limits without filesystem I/O. + /// An Arc input is retained without copying. Unread payload CRCs are not + /// certified. Expansion limits apply to the sum of all declared members. + pub fn open(bytes: impl AsRef<[u8]> + Into>, limits: ArchiveLimits) -> Result { + if bytes.as_ref().len() as u64 > limits.input { + return Err(limit("ZIP input byte limit exceeded")); + } + let bytes = bytes.into(); + let directory = preflight::inspect(&bytes, limits)?; + let indexing = Arc::new(AtomicBool::new(true)); + let reader = preflight::IndexReader { + cursor: Cursor::new(Arc::clone(&bytes)), + indexing: Arc::clone(&indexing), + central_start: directory.start, + }; + let mut zip = ZipArchive::with_config( + Config { + archive_offset: ArchiveOffset::Known(0), + }, + reader, + ) + .map_err(|e| malformed(format!("cannot index ZIP: {e}")))?; + indexing.store(false, Ordering::Relaxed); + if zip.len() != directory.headers.len() + || zip.central_directory_start() != directory.start as u64 + { + return Err(malformed( + "ZIP index disagrees with its bounded central directory", + )); + } + let mut entries = BTreeMap::new(); + let mut nodes = BTreeMap::new(); + let mut expanded = 0u64; + let mut regions = Vec::new(); + for (index, &(flags, method)) in directory.headers.iter().enumerate() { + let file = zip + .by_index_raw(index) + .map_err(|e| malformed(format!("invalid ZIP member: {e}")))?; + validate_type(&file, flags)?; + let name = std::str::from_utf8(file.name_raw()) + .map_err(|_| unsupported("ZIP member names must be UTF-8"))?; + if file.name().map_err(|e| malformed(e.to_string()))?.as_ref() != name { + return Err(unsafe_archive( + "ZIP Unicode name override disagrees with its raw name", + )); + } + let is_dir = file.is_dir(); + expanded = expanded + .checked_add(file.size()) + .ok_or_else(|| limit("ZIP expansion size overflow"))?; + if expanded > limits.expanded { + return Err(limit("ZIP cumulative expanded byte limit exceeded")); + } + if file.size() > 0 + && (file.compressed_size() == 0 + || file.size() > file.compressed_size().saturating_mul(limits.ratio)) + { + return Err(limit("ZIP member compression ratio exceeded")); + } + let path = validate_path(name, is_dir, &mut nodes)?; + let entry = ArchiveEntry { + path: path.clone(), + directory: is_dir, + size: file.size(), + compressed: file.compressed_size(), + index, + flags, + method, + crc: file.crc32(), + header: usize::try_from(file.header_start()) + .map_err(|_| malformed("invalid local header offset"))?, + data: file + .data_start() + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| malformed("invalid member data offset"))?, + }; + regions.push(local::validate( + &bytes, + &entry, + name.as_bytes(), + directory.start, + )?); + entries.insert(path, entry); + } + local::validate_regions(&mut regions, directory.start)?; + Ok(Self { + bytes, + zip, + entries, + cache: BTreeMap::new(), + }) + } + + /// Members in deterministic path order, including explicit directories. + pub fn entries(&self) -> impl ExactSizeIterator { + self.entries.values() + } + + /// Exact original ZIP bytes, including its comment and compression data. + #[must_use] + pub fn original_bytes(&self) -> Arc<[u8]> { + Arc::clone(&self.bytes) + } + + /// Read and CRC-verify a member, caching only a complete valid payload. + pub fn read(&mut self, path: &ArtifactPath) -> Result> { + self.read_with_limit(path, u64::MAX) + } + + /// Apply an additional per-read byte cap, including on cached members. + /// A refusal does not populate the cache. Zero permits only empty members. + pub fn read_with_limit(&mut self, path: &ArtifactPath, max_bytes: u64) -> Result> { + let entry = self.entries.get(path).ok_or_else(|| { + Error::new( + &codes::UNKNOWN_ENTRY, + format!("unknown ZIP member `{path}`"), + ) + })?; + if entry.size > max_bytes { + return Err(limit("ZIP member exceeds the caller's byte limit")); + } + if let Some(bytes) = self.cache.get(path) { + return Ok(Arc::clone(bytes)); + } + let mut file = self + .zip + .by_index(entry.index) + .map_err(|e| malformed(format!("cannot open ZIP member `{path}`: {e}")))?; + let mut content = Vec::new(); + content + .try_reserve_exact( + usize::try_from(entry.size) + .map_err(|_| limit("member length exceeds this platform"))?, + ) + .map_err(|_| limit("cannot reserve ZIP member bytes"))?; + content.resize(entry.size as usize, 0); + file.read_exact(&mut content) + .map_err(|e| malformed(format!("cannot verify ZIP member `{path}`: {e}")))?; + // Force EOF even for zero-length members, so CRC validation runs. + if file + .read(&mut [0u8; 1]) + .map_err(|e| malformed(e.to_string()))? + != 0 + { + return Err(malformed("ZIP member inflates beyond its declared size")); + } + let content: Arc<[u8]> = content.into(); + self.cache.insert(path.clone(), Arc::clone(&content)); + Ok(content) + } +} + +// Store segments rather than every full ancestor prefix: namespace storage +// stays linear in supplied name bytes even for deep, disjoint paths. +struct NameNode { + id: usize, + spelling: String, + directory: bool, + explicit: bool, +} + +fn validate_path( + name: &str, + directory: bool, + nodes: &mut BTreeMap<(usize, String), NameNode>, +) -> Result { + let name = if directory { + name.strip_suffix('/').unwrap_or(name) + } else { + name + }; + let path = + ArtifactPath::new(name).map_err(|_| unsafe_archive("ZIP member has an unsafe path"))?; + if name.split('/').count() > 64 { + return Err(limit("ZIP member path depth exceeded")); + } + let parts: Vec<_> = name.split('/').collect(); + let mut parent = 0; + for (index, part) in parts.iter().enumerate() { + let stem = part + .split('.') + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + let device_number = stem + .strip_prefix("com") + .or_else(|| stem.strip_prefix("lpt")) + .is_some_and(|suffix| { + suffix.chars().count() == 1 + && suffix + .chars() + .all(|c| c.is_ascii_digit() || matches!(c, '¹' | '²' | '³')) + }); + if part.ends_with(['.', ' ']) + || part.contains(['<', '>', '"', '|', '?', '*']) + || matches!(stem.as_str(), "con" | "prn" | "aux" | "nul") + || device_number + { + return Err(unsafe_archive( + "ZIP path is not portable across supported platforms", + )); + } + let is_directory = directory || index + 1 < parts.len(); + let next_id = nodes.len() + 1; + let node = nodes + .entry((parent, part.to_lowercase())) + .or_insert_with(|| NameNode { + id: next_id, + spelling: (*part).to_owned(), + directory: is_directory, + explicit: false, + }); + if node.spelling != *part || node.directory != is_directory { + return Err(unsafe_archive( + "ZIP file/directory or ancestor case collision", + )); + } + if index + 1 == parts.len() { + if node.explicit { + return Err(unsafe_archive("duplicate ZIP member path")); + } + node.explicit = true; + } + parent = node.id; + } + Ok(path) +} + +fn validate_type(file: &zip::read::ZipFile<'_, impl Read>, flags: u16) -> Result<()> { + if file.encrypted() || flags & (1 | 64 | 8192) != 0 { + return Err(unsupported("encrypted ZIP members are unsupported")); + } + if !matches!( + file.compression(), + CompressionMethod::Stored | CompressionMethod::Deflated + ) { + return Err(unsupported( + "only stored and deflated ZIP members are supported", + )); + } + if let Some(mode) = file.unix_mode() { + let kind = mode & 0o170_000; + if !matches!(kind, 0 | 0o100_000 | 0o040_000) + || (kind != 0 && (kind == 0o040_000) != file.is_dir()) + { + return Err(unsafe_archive( + "ZIP links and special files are unsupported", + )); + } + } + if file.is_dir() && file.size() != 0 { + return Err(unsafe_archive("directory member contains file data")); + } + Ok(()) +} diff --git a/powerio-archive/src/local.rs b/powerio-archive/src/local.rs new file mode 100644 index 000000000..0f57afaad --- /dev/null +++ b/powerio-archive/src/local.rs @@ -0,0 +1,126 @@ +//! Local headers and descriptors must agree with the indexed member. +use crate::{ + ArchiveEntry, Result, malformed, + preflight::{u16_at, u32_at, u64_at}, +}; + +pub(crate) fn validate( + bytes: &[u8], + entry: &ArchiveEntry, + name: &[u8], + central: usize, +) -> Result<(usize, usize)> { + let header = bytes + .get(entry.header..central) + .filter(|h| h.len() >= 30 && h.starts_with(b"PK\x03\x04")) + .ok_or_else(|| malformed("missing or misplaced ZIP local header"))?; + let name_len = usize::from(u16_at(header, 26)); + let extra_len = usize::from(u16_at(header, 28)); + let header_len = 30 + name_len + extra_len; + if header_len > header.len() + || header.get(30..30 + name_len) != Some(name) + || u16_at(header, 6) != entry.flags + || u16_at(header, 8) != entry.method + || entry.header.checked_add(header_len) != Some(entry.data) + { + return Err(malformed("ZIP local and central member headers disagree")); + } + let mut sizes = [u64::from(u32_at(header, 22)), u64::from(u32_at(header, 18))]; + let mut extra = &header[30 + name_len..header_len]; + let mut zip64 = false; + while !extra.is_empty() { + if extra.len() < 4 { + return Err(malformed("truncated ZIP local extra field")); + } + let n = usize::from(u16_at(extra, 2)); + let value = extra + .get(4..4 + n) + .ok_or_else(|| malformed("ZIP local extra field exceeds its header"))?; + if u16_at(extra, 0) == 1 { + if zip64 { + return Err(malformed("duplicate ZIP64 local extra field")); + } + zip64 = true; + let mut offset = 0; + for size in &mut sizes { + if *size == u64::from(u32::MAX) { + let wide = value + .get(offset..offset + 8) + .ok_or_else(|| malformed("truncated ZIP64 local size"))?; + *size = u64_at(wide, 0); + offset += 8; + } + } + } + extra = &extra[4 + n..]; + } + let end = entry + .data + .checked_add( + usize::try_from(entry.compressed) + .map_err(|_| malformed("invalid ZIP compressed size"))?, + ) + .filter(|&end| end <= central) + .ok_or_else(|| malformed("ZIP member leaves the local data region"))?; + if entry.flags & 8 == 0 { + if sizes != [entry.size, entry.compressed] || u32_at(header, 14) != entry.crc { + return Err(malformed( + "ZIP local sizes or CRC disagree with central directory", + )); + } + return Ok((entry.header, end)); + } + if sizes + .iter() + .zip([entry.size, entry.compressed]) + .any(|(&local, indexed)| local != 0 && local != indexed) + || ![0, entry.crc].contains(&u32_at(header, 14)) + { + return Err(malformed( + "streaming ZIP local sizes or CRC contradict its index", + )); + } + let descriptor = &bytes[end..central]; + let width = + if zip64 || entry.size > u64::from(u32::MAX) || entry.compressed > u64::from(u32::MAX) { + 8 + } else { + 4 + }; + let matches = |offset: usize| -> Option { + let record = descriptor.get(offset..offset + 4 + 2 * width)?; + let values = if width == 8 { + [u64_at(record, 4), u64_at(record, 12)] + } else { + [u64::from(u32_at(record, 4)), u64::from(u32_at(record, 8))] + }; + (u32_at(record, 0) == entry.crc && values == [entry.compressed, entry.size]) + .then_some(end + offset + record.len()) + }; + let with_signature = descriptor + .starts_with(b"PK\x07\x08") + .then(|| matches(4)) + .flatten(); + let without_signature = matches(0); + let ((Some(tail), None) | (None, Some(tail))) = (without_signature, with_signature) else { + return Err(malformed( + "missing, ambiguous or inconsistent ZIP data descriptor", + )); + }; + Ok((entry.header, tail)) +} + +pub(crate) fn validate_regions(regions: &mut [(usize, usize)], central: usize) -> Result<()> { + regions.sort_unstable(); + if regions.first().map_or(central != 0, |region| region.0 != 0) { + return Err(crate::unsupported( + "prepended data and self-extracting ZIPs are unsupported", + )); + } + if regions.windows(2).any(|pair| pair[1].0 < pair[0].1) { + return Err(crate::unsafe_archive( + "ZIP members have overlapping header/data regions", + )); + } + Ok(()) +} diff --git a/powerio-archive/src/preflight.rs b/powerio-archive/src/preflight.rs new file mode 100644 index 000000000..a731e7101 --- /dev/null +++ b/powerio-archive/src/preflight.rs @@ -0,0 +1,182 @@ +//! Bound the decoder's metadata work before it constructs its index. + +use std::io::{self, Cursor, Read, Seek, SeekFrom}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use crate::{ArchiveLimits, Result, limit, malformed, unsupported}; + +pub(crate) fn u16_at(bytes: &[u8], at: usize) -> u16 { + u16::from_le_bytes(bytes[at..at + 2].try_into().expect("checked record")) +} +pub(crate) fn u32_at(bytes: &[u8], at: usize) -> u32 { + u32::from_le_bytes(bytes[at..at + 4].try_into().expect("checked record")) +} +pub(crate) fn u64_at(bytes: &[u8], at: usize) -> u64 { + u64::from_le_bytes(bytes[at..at + 8].try_into().expect("checked record")) +} + +pub(crate) struct Directory { + pub start: usize, + pub headers: Vec<(u16, u16)>, +} + +fn usize_at(value: u64) -> Result { + usize::try_from(value).map_err(|_| limit("ZIP offset exceeds this platform")) +} + +pub(crate) fn inspect(bytes: &[u8], limits: ArchiveLimits) -> Result { + let mut end = None; + // A terminal end record has a 16-bit comment length. Requiring its comment + // to end at EOF rules out trailing data and ambiguous terminal candidates. + for at in bytes.len().saturating_sub(65_557)..bytes.len().saturating_sub(21) { + if bytes[at..].starts_with(b"PK\x05\x06") + && at + 22 + usize::from(u16_at(bytes, at + 20)) == bytes.len() + && end.replace(at).is_some() + { + return Err(malformed("ambiguous terminal ZIP end records")); + } + } + let end = end.ok_or_else(|| malformed("missing terminal ZIP end record"))?; + if u16_at(bytes, end + 4) != 0 || u16_at(bytes, end + 6) != 0 { + return Err(unsupported("multi-volume ZIP archives are unsupported")); + } + let mut count = u64::from(u16_at(bytes, end + 10)); + let mut size = u64::from(u32_at(bytes, end + 12)); + let mut offset = u64::from(u32_at(bytes, end + 16)); + let mut footer = end; + if end >= 20 && bytes[end - 20..].starts_with(b"PK\x06\x07") { + let locator = end - 20; + if u32_at(bytes, locator + 4) != 0 || u32_at(bytes, locator + 16) != 1 { + return Err(unsupported("multi-volume ZIP64 archives are unsupported")); + } + footer = usize_at(u64_at(bytes, locator + 8))?; + let record = bytes + .get(footer..locator) + .filter(|record| record.len() >= 56 && record.starts_with(b"PK\x06\x06")) + .ok_or_else(|| malformed("invalid ZIP64 end-record location"))?; + if u64_at(record, 4).checked_add(12) != Some(record.len() as u64) + || u32_at(record, 16) != 0 + || u32_at(record, 20) != 0 + || u64_at(record, 24) != u64_at(record, 32) + { + return Err(malformed("inconsistent ZIP64 end record")); + } + let values = [u64_at(record, 32), u64_at(record, 40), u64_at(record, 48)]; + for (legacy, wide, sentinel) in [ + (count, values[0], u64::from(u16::MAX)), + (size, values[1], u64::from(u32::MAX)), + (offset, values[2], u64::from(u32::MAX)), + ] { + if legacy != sentinel && legacy != wide { + return Err(malformed("ZIP32 and ZIP64 end records disagree")); + } + } + [count, size, offset] = values; + } else if count == u64::from(u16::MAX) + || size == u64::from(u32::MAX) + || offset == u64::from(u32::MAX) + { + return Err(malformed("missing ZIP64 end records")); + } + if u16_at(bytes, end + 8) != u16_at(bytes, end + 10) { + return Err(malformed("end-record entry counts disagree")); + } + if count > limits.entries as u64 { + return Err(limit("ZIP entry-count limit exceeded")); + } + let start = usize_at(offset)?; + if offset.checked_add(size) != Some(footer as u64) || start > footer { + return Err(malformed("central directory does not meet its end records")); + } + // The index reader below masks member data, so embedded ZIP signatures in + // opaque payloads cannot trigger decoder fallback. Metadata has one EOCD. + if bytes[start..] + .windows(4) + .enumerate() + .any(|(at, magic)| magic == b"PK\x05\x06" && start + at != end) + { + return Err(malformed("ambiguous ZIP end record in metadata")); + } + let mut at = start; + let mut headers = Vec::new(); + headers + .try_reserve_exact(usize_at(count)?) + .map_err(|_| limit("cannot reserve ZIP index"))?; + for _ in 0..count { + let header = bytes + .get(at..footer) + .filter(|h| h.len() >= 46 && h.starts_with(b"PK\x01\x02")) + .ok_or_else(|| malformed("missing central-directory member header"))?; + if u16_at(header, 34) != 0 { + return Err(unsupported("multi-volume ZIP member")); + } + let name = usize::from(u16_at(header, 28)); + if name == 0 || name > 4096 { + return Err(limit("ZIP member path length limit exceeded")); + } + headers.push((u16_at(header, 8), u16_at(header, 10))); + at += 46 + name + usize::from(u16_at(header, 30)) + usize::from(u16_at(header, 32)); + if at > footer { + return Err(malformed("central-directory member exceeds its bounds")); + } + validate_extra(&header[46 + name..46 + name + usize::from(u16_at(header, 30))])?; + } + if at != footer { + return Err(malformed( + "central-directory length or entry count disagrees", + )); + } + Ok(Directory { start, headers }) +} + +fn validate_extra(mut fields: &[u8]) -> Result<()> { + let mut zip64 = false; + while !fields.is_empty() { + if fields.len() < 4 { + return Err(malformed("truncated ZIP central extra field")); + } + let length = 4 + usize::from(u16_at(fields, 2)); + if length > fields.len() { + return Err(malformed("ZIP central extra field exceeds its header")); + } + if u16_at(fields, 0) == 1 { + if zip64 { + return Err(malformed("duplicate ZIP64 central extra field")); + } + zip64 = true; + } + fields = &fields[length..]; + } + Ok(()) +} + +/// While ZIP indexes metadata, bytes before the central directory read as +/// zero. This hides all fallback end-record candidates inside opaque payloads +/// without copying or rewriting those payloads. Local headers and member data +/// are visible again before validation/reads; no unsafe metadata transplant. +#[derive(Debug)] +pub(crate) struct IndexReader { + pub cursor: Cursor>, + pub indexing: Arc, + pub central_start: usize, +} + +impl Read for IndexReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + let start = usize::try_from(self.cursor.position()).unwrap_or(usize::MAX); + let n = self.cursor.read(output)?; + if self.indexing.load(Ordering::Relaxed) { + let hidden = self.central_start.saturating_sub(start).min(n); + output[..hidden].fill(0); + } + Ok(n) + } +} +impl Seek for IndexReader { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.cursor.seek(position) + } +} diff --git a/powerio-archive/tests/transport.rs b/powerio-archive/tests/transport.rs new file mode 100644 index 000000000..2246df45e --- /dev/null +++ b/powerio-archive/tests/transport.rs @@ -0,0 +1,453 @@ +use powerio_archive::{Archive, ArchiveLimits, codes}; +use powerio_core::{ArtifactPath, DiagnosticInfo}; +use std::{ + io::{Cursor, Write}, + sync::Arc, +}; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +fn path(name: &str) -> ArtifactPath { + ArtifactPath::new(name).unwrap() +} +fn open(bytes: Vec) -> Result { + Archive::open(bytes, ArchiveLimits::default()) +} +fn zip(files: &[(&str, &[u8])], method: CompressionMethod) -> Vec { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for &(name, bytes) in files { + writer + .start_file( + name, + SimpleFileOptions::default().compression_method(method), + ) + .unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap().into_inner() +} +fn at(bytes: &[u8], magic: &[u8]) -> usize { + bytes.windows(4).position(|b| b == magic).unwrap() +} +fn put16(bytes: &mut [u8], at: usize, value: u16) { + bytes[at..at + 2].copy_from_slice(&value.to_le_bytes()); +} +fn put32(bytes: &mut [u8], at: usize, value: u32) { + bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); +} +fn get32(bytes: &[u8], at: usize) -> u32 { + u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()) +} +fn refused(bytes: Vec, code: &'static DiagnosticInfo) { + let error = open(bytes).unwrap_err(); + assert_eq!(error.info().unwrap().code, code.code, "{error}"); +} + +#[test] +fn reads_preserve_bytes_order_and_shared_ownership() { + assert!(powerio_core::check_registry(codes::ALL.iter().copied()).is_empty()); + for method in [CompressionMethod::Stored, CompressionMethod::Deflated] { + let nested = zip(&[("uninterpreted.bin", b"opaque")], method); + let bytes: Arc<[u8]> = zip( + &[ + ("z.zip", &nested), + ("é/a.dss", b"arbitrary\0bytes"), + ("empty", b""), + ], + method, + ) + .into(); + let mut archive = Archive::open(Arc::clone(&bytes), ArchiveLimits::default()).unwrap(); + assert!(Arc::ptr_eq(&bytes, &archive.original_bytes())); + assert_eq!( + archive + .entries() + .map(|e| e.path().as_str()) + .collect::>(), + ["empty", "z.zip", "é/a.dss"] + ); + let first = archive.read(&path("z.zip")).unwrap(); + assert_eq!(&*first, nested); + assert!(Arc::ptr_eq(&first, &archive.read(&path("z.zip")).unwrap())); + assert!(archive.read_with_limit(&path("z.zip"), 1).is_err()); + assert_eq!( + &*archive.read(&path("é/a.dss")).unwrap(), + b"arbitrary\0bytes" + ); + assert!( + archive + .read_with_limit(&path("empty"), 0) + .unwrap() + .is_empty() + ); + assert_eq!( + archive + .read(&path("missing")) + .unwrap_err() + .info() + .unwrap() + .code, + codes::UNKNOWN_ENTRY.code + ); + } + assert_eq!( + open(zip(&[], CompressionMethod::Stored)) + .unwrap() + .entries() + .len(), + 0 + ); +} + +#[test] +fn explicit_budgets_apply_at_their_boundaries() { + let bytes = zip(&[("a", b"123"), ("b", b"456")], CompressionMethod::Stored); + let n = bytes.len() as u64; + for limits in [(n - 1, 2, 6, 1), (n, 1, 6, 1), (n, 2, 5, 1)] { + let limits = ArchiveLimits::new(limits.0, limits.1, limits.2, limits.3).unwrap(); + assert_eq!( + Archive::open(bytes.clone(), limits) + .unwrap_err() + .info() + .unwrap() + .code, + codes::LIMIT.code + ); + } + let mut archive = Archive::open(bytes, ArchiveLimits::new(n, 2, 6, 1).unwrap()).unwrap(); + assert!(archive.read_with_limit(&path("a"), 2).is_err()); + assert_eq!(&*archive.read_with_limit(&path("a"), 3).unwrap(), b"123"); + for values in [ + (0, 1, 1, 1), + (1, 0, 1, 1), + (1, 1, 0, 1), + (1, 1, 1, 0), + (u64::MAX, 1, 1, 1), + (1, usize::MAX, 1, 1), + ] { + assert!(ArchiveLimits::new(values.0, values.1, values.2, values.3).is_err()); + } + let bomb = zip(&[("bomb", &[0; 4096])], CompressionMethod::Deflated); + let limits = ArchiveLimits::new(10000, 1, 4096, 1).unwrap(); + assert_eq!( + Archive::open(bomb, limits) + .unwrap_err() + .info() + .unwrap() + .code, + codes::LIMIT.code + ); +} + +#[test] +fn rejects_nonportable_names_and_ancestor_collisions() { + for name in [ + "../a", + "/a", + "a/../b", + "a/./b", + "a//b", + "C:a", + "a\\b", + "a\0b", + "con.txt", + "x/AUX", + "lpt1", + "COM¹.txt", + "a*", + "a?", + "a|", + "a<", + "a>", + "a\"", + "a.", + "a ", + ] { + refused( + zip(&[(name, b"x")], CompressionMethod::Stored), + &codes::UNSAFE, + ); + } + for names in [["A", "a"], ["a", "a/b"], ["A/b", "a/c"], ["a/b", "a"]] { + refused( + zip( + &[(names[0], b""), (names[1], b"")], + CompressionMethod::Stored, + ), + &codes::UNSAFE, + ); + } + let deep = ["a"; 65].join("/"); + refused( + zip(&[(&deep, b"")], CompressionMethod::Stored), + &codes::LIMIT, + ); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink("link", "target", SimpleFileOptions::default()) + .unwrap(); + refused(writer.finish().unwrap().into_inner(), &codes::UNSAFE); +} + +#[test] +fn directories_and_comments_remain_transport_metadata() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_comment("archive comment").unwrap(); + writer + .add_directory("dir/", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("dir/file", SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"data").unwrap(); + let mut archive = open(writer.finish().unwrap().into_inner()).unwrap(); + assert!(archive.entries().next().unwrap().is_directory()); + assert!(archive.read(&path("dir")).unwrap().is_empty()); + assert_eq!(&*archive.read(&path("dir/file")).unwrap(), b"data"); +} + +#[test] +fn deep_names_and_late_directory_entries_share_a_namespace() { + let first = format!("root/{}", vec!["a".repeat(60); 63].join("/")); + let second = first.replace("root/", "other/"); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for name in [&first, &second] { + writer + .start_file(name, SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"x").unwrap(); + } + writer + .add_directory("root/", SimpleFileOptions::default()) + .unwrap(); + let mut archive = open(writer.finish().unwrap().into_inner()).unwrap(); + assert_eq!(archive.entries().len(), 3); + assert_eq!(&*archive.read(&path(&first)).unwrap(), b"x"); + assert_eq!(&*archive.read(&path(&second)).unwrap(), b"x"); + assert!(archive.read(&path("root")).unwrap().is_empty()); +} + +#[test] +fn corruption_is_lazy_and_never_cached() { + for method in [CompressionMethod::Stored, CompressionMethod::Deflated] { + let base = zip(&[("a", b"payload"), ("valid", b"still readable")], method); + let central = at(&base, b"PK\x01\x02"); + for (local_offset, central_offset, value) in [(14, 16, 123), (22, 24, 6), (22, 24, 8)] { + let mut bytes = base.clone(); + put32(&mut bytes, local_offset, value); + put32(&mut bytes, central + central_offset, value); + let mut archive = open(bytes).unwrap(); + for _ in 0..2 { + assert!(archive.read(&path("a")).is_err()); + } + assert_eq!(&*archive.read(&path("valid")).unwrap(), b"still readable"); + } + } +} + +#[test] +fn local_layout_and_transport_features_are_checked_before_reads() { + let base = zip( + &[("a", b"payload"), ("b", b"second")], + CompressionMethod::Stored, + ); + let central = at(&base, b"PK\x01\x02"); + for offset in [6, 8, 14, 18, 22, 26, 28, 30] { + let mut bytes = base.clone(); + bytes[offset] ^= 1; + assert!(open(bytes).is_err(), "local header byte {offset}"); + } + for (flag, method) in [(1, 0), (64, 0), (8192, 0), (0, 99)] { + let mut bytes = base.clone(); + put16(&mut bytes, 6, flag); + put16(&mut bytes, central + 8, flag); + put16(&mut bytes, 8, method); + put16(&mut bytes, central + 10, method); + assert!(open(bytes).is_err()); + } + let mut overlap = base.clone(); + let size = (central - 31) as u32; + for offset in [18, 22, central + 20, central + 24] { + put32(&mut overlap, offset, size); + } + refused(overlap, &codes::UNSAFE); + let mut special = base; + put32(&mut special, central + 38, 0o060_000 << 16); + refused(special, &codes::UNSAFE); +} + +#[test] +fn central_directory_preflight_refuses_forged_and_ambiguous_metadata() { + let base = zip(&[("a", b"payload")], CompressionMethod::Stored); + let end = base.len() - 22; + for offset in [ + end + 4, + end + 6, + end + 8, + end + 10, + end + 12, + end + 16, + end + 20, + ] { + let mut bytes = base.clone(); + bytes[offset] ^= 1; + assert!(open(bytes).is_err(), "end record byte {offset}"); + } + let mut too_many = base.clone(); + put16(&mut too_many, end + 8, 5000); + put16(&mut too_many, end + 10, 5000); + refused(too_many, &codes::LIMIT); + let central = at(&base, b"PK\x01\x02"); + for offset in [ + central, + central + 28, + central + 30, + central + 32, + central + 34, + central + 42, + ] { + let mut bytes = base.clone(); + bytes[offset] ^= 1; + assert!(open(bytes).is_err(), "central record byte {offset}"); + } + let mut ambiguous = base.clone(); + put16(&mut ambiguous, end + 20, 22); + ambiguous.extend_from_slice(&base[end..]); + refused(ambiguous, &codes::MALFORMED); + let mut trailing = base.clone(); + trailing.push(0); + refused(trailing, &codes::MALFORMED); + for len in 0..base.len() { + assert!(open(base[..len].to_vec()).is_err()); + } +} + +#[test] +fn embedded_end_signatures_are_opaque_payloads() { + let payload = b"PK\x05\x06PK\x06\x06PK\x06\x07".repeat(1000); + let mut archive = open(zip(&[("nested.zip", &payload)], CompressionMethod::Stored)).unwrap(); + assert_eq!(&*archive.read(&path("nested.zip")).unwrap(), payload); +} + +#[test] +fn bit_mutations_never_panic_while_indexing_or_reading() { + for method in [CompressionMethod::Stored, CompressionMethod::Deflated] { + let base = zip(&[("a", b"payload")], method); + for at in 0..base.len() { + for bit in 0..8 { + let mut bytes = base.clone(); + bytes[at] ^= 1 << bit; + let limits = ArchiveLimits::new(2048, 8, 4096, 200).unwrap(); + if let Ok(mut archive) = Archive::open(bytes, limits) { + let paths: Vec<_> = archive.entries().map(|e| e.path().clone()).collect(); + for path in paths { + let _ = archive.read(&path); + } + } + } + } + } +} + +#[test] +fn zip64_end_records_and_members_have_checked_bounds() { + let base = zip(&[("a", b"payload")], CompressionMethod::Stored); + let end = base.len() - 22; + let mut bytes = base[..end].to_vec(); + bytes.extend_from_slice(b"PK\x06\x06"); + bytes.extend_from_slice(&44u64.to_le_bytes()); + bytes.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + for value in [ + 1, + 1, + u64::from(get32(&base, end + 12)), + u64::from(get32(&base, end + 16)), + ] { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes.extend_from_slice(b"PK\x06\x07"); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&(end as u64).to_le_bytes()); + bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&base[end..]); + let mut archive = open(bytes.clone()).unwrap(); + assert_eq!(&*archive.read(&path("a")).unwrap(), b"payload"); + let mut saturated = bytes.clone(); + let legacy = saturated.len() - 22; + for offset in [legacy + 8, legacy + 10] { + put16(&mut saturated, offset, u16::MAX); + } + for offset in [legacy + 12, legacy + 16] { + put32(&mut saturated, offset, u32::MAX); + } + assert_eq!( + &*open(saturated.clone()).unwrap().read(&path("a")).unwrap(), + b"payload" + ); + let mut excessive = saturated; + for offset in [end + 24, end + 32] { + excessive[offset..offset + 8].copy_from_slice(&5000u64.to_le_bytes()); + } + refused(excessive, &codes::LIMIT); + for offset in [ + end + 4, + end + 16, + end + 24, + end + 32, + end + 40, + end + 48, + end + 64, + end + 72, + ] { + let mut bad = bytes.clone(); + bad[offset] ^= 1; + assert!(open(bad).is_err(), "ZIP64 footer byte {offset}"); + } + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("large", SimpleFileOptions::default().large_file(true)) + .unwrap(); + writer + .write_all(b"small payload with ZIP64 local sizes") + .unwrap(); + let mut archive = open(writer.finish().unwrap().into_inner()).unwrap(); + assert_eq!( + &*archive.read(&path("large")).unwrap(), + b"small payload with ZIP64 local sizes" + ); +} + +#[test] +fn streaming_descriptors_are_verified_in_both_widths() { + for large in [false, true] { + let mut writer = ZipWriter::new_stream(Vec::new()); + writer + .start_file("a", SimpleFileOptions::default().large_file(large)) + .unwrap(); + writer.write_all(b"stream").unwrap(); + let bytes = writer.finish().unwrap().into_inner(); + let descriptor = at(&bytes, b"PK\x07\x08"); + for offset in [14, 18, 22] { + let mut bad = bytes.clone(); + bad[offset] ^= 1; + refused(bad, &codes::MALFORMED); + } + let mut archive = open(bytes.clone()).unwrap(); + assert_eq!(&*archive.read(&path("a")).unwrap(), b"stream"); + for offset in [ + descriptor + 4, + descriptor + 8, + descriptor + if large { 16 } else { 12 }, + ] { + let mut bad = bytes.clone(); + bad[offset] ^= 1; + refused(bad, &codes::MALFORMED); + } + let central = at(&bytes, b"PK\x01\x02"); + let mut unsigned = bytes; + unsigned.drain(descriptor..descriptor + 4); + let end = unsigned.len() - 22; + put32(&mut unsigned, end + 16, (central - 4) as u32); + let mut archive = open(unsigned).unwrap(); + assert_eq!(&*archive.read(&path("a")).unwrap(), b"stream"); + } +} diff --git a/powerio/tests/crate_graph.rs b/powerio/tests/crate_graph.rs index 395089319..06143a850 100644 --- a/powerio/tests/crate_graph.rs +++ b/powerio/tests/crate_graph.rs @@ -54,6 +54,12 @@ fn the_dependency_graph_matches_the_settled_layout() { // The foundation depends on no workspace crate. assert_eq!(workspace_deps_of(&graph, "powerio-core"), BTreeSet::new()); + // Archive transport shares core infrastructure without electrical models. + assert_eq!( + workspace_deps_of(&graph, "powerio-archive"), + BTreeSet::from(["powerio-core"]) + ); + // Integration composes both model crates above the foundation. assert_eq!( workspace_deps_of(&graph, "powerio-tnd"), diff --git a/scripts/check-release-versions.sh b/scripts/check-release-versions.sh index 69bd88703..347637eee 100755 --- a/scripts/check-release-versions.sh +++ b/scripts/check-release-versions.sh @@ -82,7 +82,7 @@ if [ "$schema_name" != "pio-ir" ]; then exit 1 fi -for manifest in powerio/Cargo.toml powerio-core/Cargo.toml powerio-tx/Cargo.toml \ +for manifest in powerio/Cargo.toml powerio-core/Cargo.toml powerio-archive/Cargo.toml powerio-tx/Cargo.toml \ powerio-dist/Cargo.toml powerio-tnd/Cargo.toml powerio-matrix/Cargo.toml powerio-prob/Cargo.toml \ powerio-cli/Cargo.toml; do grep -q '^version.workspace = true' "$manifest" \ diff --git a/scripts/check-semver.sh b/scripts/check-semver.sh index ed01e981b..1809b672c 100755 --- a/scripts/check-semver.sh +++ b/scripts/check-semver.sh @@ -42,6 +42,11 @@ echo "comparing the publishable crates at $version against $baseline" # API is changed by this workaround. # Compare the new crate once the selected release baseline contains it. excluded=(--exclude powerio-py --exclude powerio-capi) +if [ -f powerio-archive/Cargo.toml ] \ + && ! git cat-file -e "$baseline:powerio-archive/Cargo.toml" 2>/dev/null; then + echo "powerio-archive is new since $baseline; no previous API to compare" + excluded+=(--exclude powerio-archive) +fi if [ -f powerio-tnd/Cargo.toml ] \ && ! git cat-file -e "$baseline:powerio-tnd/Cargo.toml" 2>/dev/null; then echo "powerio-tnd is new since $baseline; no previous API to compare" diff --git a/scripts/ci-clippy.sh b/scripts/ci-clippy.sh index 8ddc8f7d6..7f14c139d 100755 --- a/scripts/ci-clippy.sh +++ b/scripts/ci-clippy.sh @@ -45,6 +45,7 @@ case "$target" in -p powerio \ -p powerio-tx \ -p powerio-core \ + -p powerio-archive \ -p powerio-matrix \ -p powerio-prob \ -p powerio-synth \ diff --git a/scripts/ci-mirror.sh b/scripts/ci-mirror.sh index 60de1e787..c273a563c 100755 --- a/scripts/ci-mirror.sh +++ b/scripts/ci-mirror.sh @@ -60,7 +60,7 @@ fi # is not installed (rustup target add wasm32-unknown-unknown). if rustup target list --installed 2>/dev/null | grep -qx wasm32-unknown-unknown; then run cargo check --target wasm32-unknown-unknown -p powerio -p powerio-tx -p powerio-dist \ - -p powerio-core -p powerio-prob -p powerio-matrix + -p powerio-core -p powerio-archive -p powerio-prob -p powerio-matrix run cargo check --target wasm32-unknown-unknown -p powerio --features matrix # Without the default-on sincal feature the facade must build with no C # compiler, so SQLite may not appear in that dependency graph. Package names @@ -89,7 +89,7 @@ else echo "=== skipped: cargo-semver-checks not installed ===" fi -run cargo test -p powerio -p powerio-tx -p powerio-core -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-cli \ +run cargo test -p powerio -p powerio-tx -p powerio-core -p powerio-archive -p powerio-synth -p powerio-matrix -p powerio-prob -p powerio-cli \ -p powerio-capi -p powerio-dist -p powerio-tnd run cargo test -p powerio --features matrix run cargo test -p powerio --features gridfm diff --git a/scripts/release-verified.py b/scripts/release-verified.py index fde3578c9..14e537adf 100644 --- a/scripts/release-verified.py +++ b/scripts/release-verified.py @@ -38,6 +38,7 @@ CRATE_OWNER = "samtalki" CRATES = ( "powerio-core", + "powerio-archive", "powerio-tx", "powerio-dist", "powerio-tnd",