From 64abd0e3068c61f217049da09d4a44fa281f0b82 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 3 Oct 2026 22:55:11 +0000 Subject: [PATCH 1/9] feat: add the bundle GGUF format and its build script A bundle packs several single-model GGUF files into one file, so a user passes one file. Every tensor and key of a component is stored with the prefix . and the header lists the components with their kind, licence, licence URL, source, credit and changes. Tensors are copied byte for byte, so each component keeps its quantisation. docs/bundle.md specifies the layout, the versioning and compatibility rules (plain files unchanged, old readers refuse a bundle), the selection rules and the licence handling. scripts/bundle_gguf.py builds a bundle from a JSON manifest. It checks every input, refuses a component with missing licence metadata, a non-commercial licence or a licence that contradicts the input file, records the sha256 of each input and a content digest of each component, and writes the same bytes for the same inputs. --list, --verify (with optional comparison against the source files) and --notice inspect a bundle. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- docs/bundle.md | 254 +++++++++++++++++++ scripts/bundle_gguf.py | 458 +++++++++++++++++++++++++++++++++++ tests/python/check_bundle.py | 212 ++++++++++++++++ 3 files changed, 924 insertions(+) create mode 100644 docs/bundle.md create mode 100644 scripts/bundle_gguf.py create mode 100644 tests/python/check_bundle.py diff --git a/docs/bundle.md b/docs/bundle.md new file mode 100644 index 0000000..269ae75 --- /dev/null +++ b/docs/bundle.md @@ -0,0 +1,254 @@ +# Bundle GGUF + +A bundle is one GGUF file that holds several single-model GGUF files, so a user +passes one file instead of several. Phase 1 supports two component kinds: an +ASR model (with its own VAD head, if it has one) and a Silero VAD model. + +A bundle is a packaging format. It does not change how any model runs, and it +does not convert or re-quantise any weight: each tensor is copied byte for byte +with its original type. An F16 ASR model can sit next to an F16 Silero model, or +a Q8_0 ASR model next to either, and each keeps the type it was published with. + +Single-model GGUF files are unchanged. Every existing loader, converter and +published file works as before. + +## Layout + +A bundle is a normal GGUF file (version 3) with these rules. + +### Header keys + +The header keys are the only keys that are not inside a component. + +| Key | Type | Meaning | +|---|---|---| +| `general.architecture` | string | `parakeet-bundle` | +| `general.name` | string | Name of the bundle | +| `general.license` | string | Always `other`. The real licences are per component | +| `general.license.name` | string | Always `per-component` | +| `parakeet.bundle.version` | uint32 | Format version. This document describes version 1 | +| `parakeet.bundle.components` | string array | Component names, in file order. At least one. No duplicates | + +### Component names + +A component name is 1 to 32 characters from `a-z`, `0-9` and `_`, and starts with +a letter. The names `general`, `parakeet` and `bundle` are reserved. A name +never contains a dot, so `.` is an unambiguous prefix. + +By convention the ASR component is named `asr` and the Silero component is +named `vad`. The name has no meaning to the loader: the `kind` key does. + +### Namespacing + +Every tensor and every key of the source file is stored with the prefix +`.`, with no exception and no renaming. + +| Source file | Bundle | +|---|---| +| tensor `encoder.layers.0.norm.weight` | tensor `asr.encoder.layers.0.norm.weight` | +| key `parakeet.encoder.d_model` | key `asr.parakeet.encoder.d_model` | +| key `general.architecture` (value `parakeet`) | key `asr.general.architecture` | +| key `general.license` | key `asr.general.license` | +| tensor `vad16k.stft.forward_basis_buffer` | tensor `vad.vad16k.stft.forward_basis_buffer` | +| key `silero_vad.sample_rates` | key `vad.silero_vad.sample_rates` | + +The one key that the bundle drops is `general.alignment`, which GGUF defines per +file. The bundle has its own alignment. + +A component loader reads the keys and tensors under its prefix and sees them +under their original names. After that it behaves as if it had read the source +file. Tensor names are limited by ggml to 63 bytes, and this limit applies to +the prefixed name. The build script checks it. + +A tensor or key outside the header keys and the component prefixes is an error +(`bundle_gguf.py --verify` reports it). + +### Per-component keys + +Every component has these keys, all strings. All of them except the two hashes +are required. + +| Key | Meaning | +|---|---| +| `parakeet.bundle..kind` | `asr` or `vad` in version 1 (see "Kinds") | +| `parakeet.bundle..license` | SPDX identifier, or `LicenseRef-` | +| `parakeet.bundle..license_url` | `http(s)` URL of the licence text | +| `parakeet.bundle..source` | Upstream model id or URL | +| `parakeet.bundle..attribution` | Credit line the licence asks to keep (for example the copyright line) | +| `parakeet.bundle..changes` | What was changed from the upstream model (CC-BY and similar licences need this) | +| `parakeet.bundle..source_sha256` | sha256 of the single-model GGUF the component was built from | +| `parakeet.bundle..content_sha256` | Digest of the component tensors (below) | + +The licence keys are the source of truth for the licences of a bundle. A tool +that shows or checks licences reads these keys, not `general.license`, which +cannot hold more than one value. + +`content_sha256` is the sha256 of the following byte stream, over the tensors of +the component in file order: the tensor name without the prefix as UTF-8 and a +zero byte, the ggml type id as 4 bytes (little endian), the number of dimensions +as 4 bytes, each dimension as 8 bytes (little endian, in GGUF order), and the +raw tensor data. `bundle_gguf.py --verify` recomputes it, so it detects a +damaged file without the source files. + +### Kinds + +| Kind | Content | Loaded by | +|---|---|---| +| `asr` | A parakeet ASR GGUF (`general.architecture` `parakeet`, any family that `Model::load` accepts). If it has a VAD head (Ultra, Redux), the head is part of the component | `ModelLoader::load_component`, `Model::load(path, name)` | +| `vad` | A Silero VAD GGUF (`general.architecture` `silero_vad`) | `SileroVad::load(path, &err, name)` | + +The names `diarization`, `sound` and `voice` are reserved for later phases. A +reader skips components of a kind it does not know: they are listed, but it does +not load them and does not fail because they exist. + +## Compatibility rules + +1. **Plain files are unchanged.** A single-model GGUF has no bundle keys. Every + loader opens it as before. Passing a component name for a plain file is an + error. +2. **Old readers refuse a bundle.** A bundle has no top-level `parakeet.*` model + keys (no `parakeet.encoder.d_model`, no `parakeet.vocab_size`) and no + `silero_vad.*` keys, and `general.architecture` is `parakeet-bundle`, which no + earlier loader accepts. A reader from before this format fails on a missing + key and reports a load failure. It never reads the wrong weights. Tools that + only list the GGUF header still work and show the bundle keys. +3. **New readers refuse a plain call on a bundle with a clear message** instead + of failing on a missing key: `ModelLoader::load` and `SileroVad::load` + without a component say that the file is a bundle and that a component must + be named. +4. **Version.** `parakeet.bundle.version` is 1. A reader refuses a bundle with a + higher version and says which version it knows. A future change that keeps + old readers working (new optional keys, new kinds) does not change the + version. A change that would make a version 1 reader misread a file does. +5. **Unknown keys and kinds are ignored** by a reader. A bundle may carry extra + `parakeet.bundle..*` keys. +6. **Header is read alone.** Listing the components, licences and sizes reads the + header and the tensor table only, never tensor data. + +## Selection rules + +A component is chosen by name or by default. + +* **Default component** (`parakeet_capi_load`, `transcribe` without + `--component`): the only component of kind `asr`. A bundle with no `asr` + component but exactly one loadable component of another kind opens that one + (a Silero-only bundle opens as a VAD). If more than one candidate exists the + load fails and the message lists the component names. There is no silent + choice. +* **By name** (`parakeet_capi_load_component`, `--component`): that component, + whatever its kind. The context has the kind the component declares. +* **VAD source** for `parakeet-cli vad` and for segmented transcription + (`transcribe --vad`) on a bundle: the Silero component if the bundle has one, + otherwise the VAD head of the ASR component. `--component` (for `vad`) and + `--vad-component` (for `transcribe`) override the choice; naming the ASR + component selects its head. A Silero model passed with `--vad-model` takes + precedence over the bundle. + +## Partial loading + +A loader reads the header and the tensor table, then reads only the tensors of +the component it was asked for. The ASR loader reads them into one private +memory block, and the Silero loader seeks to each of its tensors. The bytes of +the other components are never read. A test checks the number of bytes read from +the file for each component. + +Memory: a loaded component uses the same memory as the same single-model file. +The memory of components that are not loaded is not used. The file stays on disk +and is not memory mapped. + +## Building, inspecting and verifying + +`scripts/bundle_gguf.py` (Python with `gguf` and `numpy`, as for the converter): + +``` +bundle_gguf.py --out bundle.gguf --manifest manifest.json +bundle_gguf.py --list bundle.gguf [--json] +bundle_gguf.py --verify bundle.gguf [--source asr=asr.gguf --source vad=silero.gguf] +bundle_gguf.py --notice bundle.gguf +``` + +The manifest is a JSON file. `file` is relative to the manifest. + +```json +{"name": "parakeet-bundle-standard", + "components": [ + {"name": "asr", "file": "tdt-0.6b-v3-q8_0.gguf", "kind": "asr", + "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", + "source": "nvidia/parakeet-tdt-0.6b-v3", + "attribution": "Parakeet TDT 0.6B v3 by NVIDIA, licensed CC-BY-4.0", + "changes": "Converted from the NeMo checkpoint to GGUF; weights quantised to Q8_0"}, + {"name": "vad", "file": "silero-vad-f16.gguf", "kind": "vad", + "license": "MIT", "license_url": "https://github.com/snakers4/silero-vad/blob/master/LICENSE", + "source": "https://github.com/snakers4/silero-vad", + "attribution": "Copyright (c) 2020-present Silero Team", + "changes": "Converted from ONNX to GGUF (F16)"}]} +``` + +The build: + +* checks every input: it is a GGUF, it is not a bundle, its architecture matches + the kind, the keys the loader needs exist, tensor names are unique and short + enough; +* refuses a component with a missing or empty `license`, `license_url`, + `source`, `attribution` or `changes`; +* refuses a licence that is not an SPDX-style identifier, and refuses licences + that restrict commercial use or derivatives (`-NC-`, `-ND-`); +* refuses a component whose input file declares a `general.license` that + differs from the manifest (the same licence in another spelling is accepted); +* records the sha256 of each input file and the content digest of each + component; +* is deterministic: the same inputs and manifest give the same file, byte for + byte (no timestamps, keys and tensors in source order); +* writes to a temporary file and renames it, and never overwrites an input. + +`--verify` checks the structure (no tensor or key outside a component, valid +names, complete licence keys, tensors inside the file), recomputes every +`content_sha256`, and with `--source` compares the tensors (name, type, shape, +bytes) and the keys with the single-model file and its recorded sha256. It +exits 1 on any problem. + +`--notice` prints a NOTICE text from the licence keys: for each component the +model, licence and URL, credit line, changes and input hash. Ship this text and +the licence texts with the file. + +## Licences + +Each component keeps the licence of the model it was converted from. A bundle +is a collection, so it does not have one licence. Rules the tools apply: + +* every component states its licence, source, credit and changes in the header; +* `general.license` is `other`, and a tool must read the per-component keys; +* components under licences that restrict commercial use or derivatives are not + bundled; +* the NOTICE text and the licence texts must be distributed with the file, as + the licences require (CC-BY-4.0 asks for credit, a licence link and a note of + changes; MIT asks that the copyright notice and licence text stay with copies). + +## C-API and CLI + +```c +parakeet_ctx* parakeet_capi_load(const char* path); // bundle: default ASR component +parakeet_ctx* parakeet_capi_load_component(const char* path, const char* name); +char* parakeet_capi_bundle_components_json(const char* path); // free with parakeet_capi_free_string +const char* parakeet_capi_load_error(void); // reason of the last failed load on this thread +``` + +``` +parakeet-cli info bundle.gguf [--component NAME] +parakeet-cli transcribe --model bundle.gguf --input a.wav [--component NAME] +parakeet-cli transcribe --model bundle.gguf --input a.wav --vad [--vad-component NAME] +parakeet-cli vad --model bundle.gguf --input a.wav [--component NAME] +``` + +`parakeet-cli info` without `--component` lists the components, kinds, licences, +sources, credits and sizes from the header. `--component` prints the details of +one component. Other subcommands (bench, scene, streaming) do not take a bundle +yet: they refuse it with a message. + +## Not covered in phase 1 + +* diarization, sound-event and speaker-identification components (the kinds are + reserved; their loaders are separate libraries); +* cache-aware streaming from a bundle component; +* GPU backends were not run with bundles (the loader and the compute path are + the same as for single-model files, but there is no test). diff --git a/scripts/bundle_gguf.py b/scripts/bundle_gguf.py new file mode 100644 index 0000000..278d443 --- /dev/null +++ b/scripts/bundle_gguf.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Build, inspect and verify bundle GGUF files (format: docs/bundle.md). + +A bundle merges single-model GGUF files into one file. No weight is converted or +re-quantised: every tensor is copied byte for byte with its original type, so a +component keeps the quantisation it was published with. + +Build (the manifest is JSON; "file" is relative to the manifest): + + {"name": "parakeet-bundle-standard", + "components": [ + {"name": "asr", "file": "asr.gguf", "kind": "asr", + "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", + "source": "nvidia/parakeet-tdt-0.6b-v3", + "attribution": "Parakeet TDT 0.6B v3 by NVIDIA, CC-BY-4.0", + "changes": "Converted to GGUF and quantised to Q8_0"}, + {"name": "vad", "file": "silero-vad-f16.gguf", "kind": "vad", ...}]} + + bundle_gguf.py --out bundle.gguf --manifest manifest.json + +Inspect and verify: + + bundle_gguf.py --list bundle.gguf [--json] + bundle_gguf.py --verify bundle.gguf [--source asr=asr.gguf --source vad=silero.gguf] + bundle_gguf.py --notice bundle.gguf + +The output is deterministic: the same inputs and manifest give the same bytes. +The script refuses a component without full licence metadata, a licence that +contradicts the licence the input file declares, and non-commercial licences. +""" +import argparse +import hashlib +import json +import os +import re +import sys + +import numpy as np +from gguf import GGUFReader, GGUFValueType, GGUFWriter + +ARCH = "parakeet-bundle" +VERSION = 1 +KINDS = ("asr", "vad") # kinds this tool can build (docs/bundle.md lists the reserved ones) +RESERVED_NAMES = {"general", "parakeet", "bundle"} +NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$") +LICENSE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+-]*$") +REQUIRED = ("license", "license_url", "source", "attribution", "changes") +MANIFEST_KEYS = {"name", "file", "kind", *REQUIRED} +MAX_TENSOR_NAME = 63 # ggml keeps names in 64 bytes including the NUL +SKIP_KEYS = {"general.alignment"} + + +class BundleError(Exception): + pass + + +# --------------------------------------------------------------------------- helpers + +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def tensor_bytes(t): + return np.ascontiguousarray(t.data).reshape(-1).view(np.uint8) + + +def content_digest(tensors): + """sha256 over the tensors of one component, in file order. + + Per tensor: name (utf-8), 0x00, ggml type id (u32 LE), number of dims (u32 LE), + each dim (u64 LE, GGUF order), then the raw data bytes.""" + h = hashlib.sha256() + for name, ttype, dims, data in tensors: + h.update(name.encode("utf-8") + b"\0") + h.update(int(ttype).to_bytes(4, "little")) + h.update(len(dims).to_bytes(4, "little")) + for d in dims: + h.update(int(d).to_bytes(8, "little")) + h.update(data) + return h.hexdigest() + + +def reader_tensors(reader, prefix=""): + """(name without prefix, type id, dims, data) for the tensors that start with prefix.""" + out = [] + for t in reader.tensors: + if not t.name.startswith(prefix): + continue + out.append((t.name[len(prefix):], int(t.tensor_type), [int(x) for x in t.shape], tensor_bytes(t))) + return out + + +def field_value(field): + """(value, type, sub_type) ready for GGUFWriter.add_key_value.""" + t = field.types[0] + if t == GGUFValueType.ARRAY: + sub = field.types[-1] + if sub == GGUFValueType.STRING: + return [str(bytes(field.parts[i]), "utf-8") for i in field.data], t, sub + return field.contents(), t, sub + return field.contents(), t, None + + +def str_field(reader, key): + f = reader.fields.get(key) + if f is None or f.types[0] != GGUFValueType.STRING: + return None + return str(bytes(f.parts[-1]), "utf-8") + + +def open_gguf(path): + try: + return GGUFReader(path) + except Exception as e: # not a GGUF file, truncated, ... + raise BundleError(f"{path}: cannot read as a GGUF file ({e})") + + +def norm_license(s): + return re.sub(r"[\s_]+", "-", s.strip().lower()) + + +def check_license_metadata(where, meta): + for k in REQUIRED: + v = meta.get(k) + if not isinstance(v, str) or not v.strip(): + raise BundleError(f"{where}: missing or empty '{k}' (every component must carry its licence, source, " + f"attribution and changes)") + if not LICENSE_RE.match(meta["license"]): + raise BundleError(f"{where}: license '{meta['license']}' is not a SPDX-style identifier " + f"(use for example CC-BY-4.0, MIT, Apache-2.0 or LicenseRef-)") + if re.search(r"(^|-)(nc|nd)(-|$)|noncommercial|non-commercial", meta["license"].lower()): + raise BundleError(f"{where}: license '{meta['license']}' restricts commercial use or derivatives; " + f"such models must not be bundled") + if not re.match(r"^https?://", meta["license_url"]): + raise BundleError(f"{where}: license_url must be an http(s) URL") + + +# --------------------------------------------------------------------------- build + +def load_manifest(path): + try: + with open(path, "r", encoding="utf-8") as f: + m = json.load(f) + except (OSError, ValueError) as e: + raise BundleError(f"{path}: cannot read manifest ({e})") + if not isinstance(m, dict) or not isinstance(m.get("name"), str) or not m["name"].strip(): + raise BundleError("manifest needs a non-empty 'name'") + unknown = set(m) - {"name", "components"} + if unknown: + raise BundleError(f"manifest: unknown keys {sorted(unknown)}") + comps = m.get("components") + if not isinstance(comps, list) or not comps: + raise BundleError("manifest needs a non-empty 'components' list") + base = os.path.dirname(os.path.abspath(path)) + seen = set() + for c in comps: + if not isinstance(c, dict): + raise BundleError("each component must be a JSON object") + unknown = set(c) - MANIFEST_KEYS + if unknown: + raise BundleError(f"component {c.get('name')!r}: unknown keys {sorted(unknown)}") + n = c.get("name") + if not isinstance(n, str) or not NAME_RE.match(n) or n in RESERVED_NAMES: + raise BundleError(f"bad component name {n!r} (lower case letters, digits and _, starting with a letter; " + f"not one of {sorted(RESERVED_NAMES)})") + if n in seen: + raise BundleError(f"duplicate component name '{n}'") + seen.add(n) + if c.get("kind") not in KINDS: + raise BundleError(f"component {n}: kind must be one of {list(KINDS)}, got {c.get('kind')!r}") + if not isinstance(c.get("file"), str) or not c["file"]: + raise BundleError(f"component {n}: 'file' is required") + c["path"] = c["file"] if os.path.isabs(c["file"]) else os.path.join(base, c["file"]) + check_license_metadata(f"component {n}", c) + return m + + +def check_input(name, kind, reader, meta): + """Reject an input that is not what its kind says, or whose licence disagrees with the manifest.""" + arch = str_field(reader, "general.architecture") + if arch is None: + raise BundleError(f"component {name}: input has no general.architecture") + if arch == ARCH: + raise BundleError(f"component {name}: input is already a bundle; nesting is not supported") + if kind == "asr": + if arch != "parakeet" or "parakeet.arch" not in reader.fields: + raise BundleError(f"component {name}: kind asr needs a parakeet ASR GGUF " + f"(general.architecture is '{arch}')") + if str_field(reader, "parakeet.arch") == "diarization": + raise BundleError(f"component {name}: input is a diarization model, not ASR (a later phase)") + for k in ("parakeet.encoder.d_model", "parakeet.vocab_size"): + if k not in reader.fields: + raise BundleError(f"component {name}: input lacks the key {k}") + elif kind == "vad": + if arch != "silero_vad" or "silero_vad.sample_rates" not in reader.fields: + raise BundleError(f"component {name}: kind vad needs a Silero VAD GGUF " + f"(general.architecture is '{arch}')") + declared = str_field(reader, "general.license") + if declared is not None and norm_license(declared) != norm_license(meta["license"]): + raise BundleError(f"component {name}: the input file declares general.license '{declared}' but the manifest " + f"says '{meta['license']}'; resolve the conflict before bundling") + if not reader.tensors: + raise BundleError(f"component {name}: input has no tensors") + seen = set() + for t in reader.tensors: + if t.name in seen: + raise BundleError(f"component {name}: duplicate tensor name {t.name}") + seen.add(t.name) + if len(f"{name}.{t.name}") > MAX_TENSOR_NAME: + raise BundleError(f"component {name}: tensor name {name}.{t.name} is longer than {MAX_TENSOR_NAME} bytes") + + +def build(manifest_path, out_path): + m = load_manifest(manifest_path) + comps = m["components"] + inputs = [] + for c in comps: + if not os.path.isfile(c["path"]): + raise BundleError(f"component {c['name']}: input file {c['file']} does not exist") + if os.path.exists(out_path) and os.path.samefile(out_path, c["path"]): + raise BundleError("the output file is also an input") + r = open_gguf(c["path"]) + check_input(c["name"], c["kind"], r, c) + inputs.append(r) + + tmp = out_path + ".tmp" + w = GGUFWriter(tmp, ARCH, use_temp_file=False) + w.add_string("general.name", m["name"]) + w.add_string("general.license", "other") + w.add_string("general.license.name", "per-component") + w.add_uint32("parakeet.bundle.version", VERSION) + w.add_array("parakeet.bundle.components", [c["name"] for c in comps]) + total = 0 + for c, r in zip(comps, inputs): + n = c["name"] + digest = content_digest(reader_tensors(r)) + w.add_string(f"parakeet.bundle.{n}.kind", c["kind"]) + for k in REQUIRED: + w.add_string(f"parakeet.bundle.{n}.{k}", c[k]) + w.add_string(f"parakeet.bundle.{n}.source_sha256", sha256_file(c["path"])) + w.add_string(f"parakeet.bundle.{n}.content_sha256", digest) + for key, f in r.fields.items(): + if key.startswith("GGUF.") or key in SKIP_KEYS: # GGUF.* are reader internals + continue + val, vt, sub = field_value(f) + w.add_key_value(f"{n}.{key}", val, vt, sub) + for t in r.tensors: + w.add_tensor(f"{n}.{t.name}", t.data, raw_shape=t.data.shape, raw_dtype=t.tensor_type) + total += int(t.n_bytes) + print(f"{n}: {len(r.tensors)} tensors from {c['file']}", file=sys.stderr) + try: + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file(progress=False) + w.close() + os.replace(tmp, out_path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + print(f"wrote {out_path} ({total / 1e6:.1f} MB of tensor data)", file=sys.stderr) + + +# --------------------------------------------------------------------------- read + +def components_of(reader): + f = reader.fields.get("parakeet.bundle.components") + if f is None or f.types[0] != GGUFValueType.ARRAY or f.types[-1] != GGUFValueType.STRING: + raise BundleError("no parakeet.bundle.components string array") + return field_value(f)[0] + + +def load_bundle(path): + r = open_gguf(path) + if str_field(r, "general.architecture") != ARCH: + raise BundleError(f"{path}: not a bundle (general.architecture is not '{ARCH}')") + f = r.fields.get("parakeet.bundle.version") + if f is None: + raise BundleError("no parakeet.bundle.version") + ver = int(f.contents()) + if ver < 1 or ver > VERSION: + raise BundleError(f"bundle format version {ver} is not supported (this tool knows {VERSION})") + return r + + +def component_info(r, n): + info = {} + for k in ("kind",) + REQUIRED + ("source_sha256", "content_sha256"): + info[k] = str_field(r, f"parakeet.bundle.{n}.{k}") + tens = [t for t in r.tensors if t.name.startswith(n + ".")] + info["tensors"] = len(tens) + info["bytes"] = sum(int(t.n_bytes) for t in tens) + info["types"] = sorted({t.tensor_type.name for t in tens}) + return info + + +def list_bundle(path, as_json): + r = load_bundle(path) + out = {"name": str_field(r, "general.name"), "version": int(r.fields["parakeet.bundle.version"].contents()), + "file_bytes": os.path.getsize(path), "components": {}} + for n in components_of(r): + out["components"][n] = component_info(r, n) + if as_json: + print(json.dumps(out, indent=2, sort_keys=True)) + return + print(f"{out['name']} (format v{out['version']}, {out['file_bytes'] / 1e6:.1f} MB)") + for n, i in out["components"].items(): + print(f" {n}: kind={i['kind']} licence={i['license']} tensors={i['tensors']} " + f"{i['bytes'] / 1e6:.1f} MB types={','.join(i['types'])}") + print(f" source={i['source']} content_sha256={i['content_sha256']}") + + +def verify_bundle(path, sources): + r = load_bundle(path) + names = components_of(r) + problems = [] + if len(set(names)) != len(names): + problems.append("duplicate component names") + for n in names: + if not NAME_RE.match(n) or n in RESERVED_NAMES: + problems.append(f"bad component name '{n}'") + owned = tuple(n + "." for n in names) + allowed_top = tuple(["general.", "parakeet.bundle."]) + owned + for key in r.fields: + if key.startswith("GGUF."): + continue + if not key.startswith(allowed_top): + problems.append(f"key outside any component: {key}") + size = os.path.getsize(path) + for t in r.tensors: + if not t.name.startswith(owned): + problems.append(f"tensor outside any component: {t.name}") + if int(t.data_offset) + int(t.n_bytes) > size: + problems.append(f"tensor {t.name} extends past the end of the file") + for n in names: + i = component_info(r, n) + if i["kind"] is None: + problems.append(f"{n}: no kind") + for k in REQUIRED: + if not i[k]: + problems.append(f"{n}: missing '{k}'") + if i["license"]: + try: + check_license_metadata(n, {k: (i[k] or "") for k in REQUIRED}) + except BundleError as e: + problems.append(str(e)) + if not i["tensors"]: + problems.append(f"{n}: no tensors") + if f"{n}.general.architecture" not in r.fields: + problems.append(f"{n}: no {n}.general.architecture key") + got = content_digest(reader_tensors(r, n + ".")) + if i["content_sha256"] != got: + problems.append(f"{n}: content_sha256 does not match the tensors (recorded {i['content_sha256']}, " + f"computed {got})") + for spec in sources: + n, _, src = spec.partition("=") + if n not in names or not src: + problems.append(f"--source {spec}: not a component=path pair of this bundle") + continue + problems += compare_with_source(r, n, src) + if problems: + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + raise BundleError(f"{len(problems)} problem(s) in {path}") + extra = f", {len(sources)} source(s) match byte for byte" if sources else "" + print(f"OK: {path}: {len(names)} component(s), {len(r.tensors)} tensors{extra}") + + +def compare_with_source(r, n, src): + problems = [] + if not os.path.isfile(src): + return [f"{n}: source file {src} does not exist"] + s = open_gguf(src) + rec = str_field(r, f"parakeet.bundle.{n}.source_sha256") + if rec != sha256_file(src): + problems.append(f"{n}: source file sha256 differs from the recorded source_sha256") + a = reader_tensors(s) + b = reader_tensors(r, n + ".") + if [x[:3] for x in a] != [x[:3] for x in b]: + problems.append(f"{n}: tensor names, types or shapes differ from the source") + else: + for x, y in zip(a, b): + if not np.array_equal(x[3], y[3]): + problems.append(f"{n}: tensor {x[0]} bytes differ from the source") + for key, f in s.fields.items(): + if key.startswith("GGUF.") or key in SKIP_KEYS: + continue + g = r.fields.get(f"{n}.{key}") + if g is None: + problems.append(f"{n}: key {key} missing in the bundle") + continue + v1, v2 = field_value(f)[0], field_value(g)[0] + if not np.array_equal(np.asarray(v1, dtype=object), np.asarray(v2, dtype=object)): + problems.append(f"{n}: key {key} differs from the source") + return problems + + +def notice(path): + r = load_bundle(path) + lines = [ + "NOTICE", + "", + f"{str_field(r, 'general.name')} is a single file that holds several machine-learning models.", + "Each model keeps its own licence and attribution, listed below. They are also stored in the", + "file header under parakeet.bundle..*. The models were converted to GGUF and", + "merged without further changes to their weights, except where a component says otherwise.", + "The full text of each licence is at the URL given for it.", + "", + ] + for n in components_of(r): + i = component_info(r, n) + lines += [ + f"Component {n} ({i['kind']})", + f" Model: {i['source']}", + f" Licence: {i['license']} {i['license_url']}", + f" Credit: {i['attribution']}", + f" Changes: {i['changes']}", + f" Input file sha256: {i['source_sha256']}", + "", + ] + sys.stdout.write("\n".join(lines)) + + +# --------------------------------------------------------------------------- main + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--out", help="bundle file to write (with --manifest)") + ap.add_argument("--manifest", help="JSON manifest of the components") + ap.add_argument("--list", metavar="BUNDLE", help="print the components of a bundle") + ap.add_argument("--json", action="store_true", help="with --list: JSON output") + ap.add_argument("--verify", metavar="BUNDLE", help="check a bundle; exit 1 on any problem") + ap.add_argument("--source", action="append", default=[], metavar="COMP=FILE", + help="with --verify: also compare a component with its single-model file") + ap.add_argument("--notice", metavar="BUNDLE", help="print a NOTICE text from the licence keys") + args = ap.parse_args(argv) + try: + if args.list: + list_bundle(args.list, args.json) + elif args.verify: + verify_bundle(args.verify, args.source) + elif args.notice: + notice(args.notice) + elif args.out and args.manifest: + build(args.manifest, args.out) + else: + ap.error("give --out and --manifest, or one of --list, --verify, --notice") + except BundleError as e: + print(f"bundle_gguf: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/python/check_bundle.py b/tests/python/check_bundle.py new file mode 100644 index 0000000..b29f35a --- /dev/null +++ b/tests/python/check_bundle.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""scripts/bundle_gguf.py: build, --list, --verify, --notice and the refusals. + +Model independent: it writes small synthetic GGUF files itself. +""" +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile + +try: + import numpy as np + from gguf import GGMLQuantizationType, GGUFWriter +except ImportError: + print("skip: numpy and gguf are needed") + sys.exit(77) + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SCRIPT = os.path.join(ROOT, "scripts", "bundle_gguf.py") +failures = 0 + + +def check(cond, msg): + global failures + if not cond: + print(f"FAIL: {msg}", file=sys.stderr) + failures += 1 + + +def run(*args): + p = subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + + +def write_asr(path, arch="parakeet", parakeet_arch="tdt", license=None, extra_key=True): + w = GGUFWriter(path, arch) + w.add_string("general.name", "stand-in/asr") + if license: + w.add_string("general.license", license) + if parakeet_arch: + w.add_string("parakeet.arch", parakeet_arch) + w.add_uint32("parakeet.encoder.d_model", 16) + w.add_uint32("parakeet.vocab_size", 5) + w.add_float32("parakeet.preprocessor.preemph", 0.97) + w.add_bool("parakeet.encoder.xscaling", True) + w.add_array("parakeet.tdt.durations", [0, 1, 2, 3, 4]) + w.add_array("parakeet.tokenizer.pieces", ["", "a", "b", "c", "d"]) + rng = np.random.default_rng(1) + w.add_tensor("encoder.w", rng.standard_normal((4, 16)).astype(np.float32)) + w.add_tensor("encoder.h", rng.standard_normal((8, 8)).astype(np.float16)) + # a quantised-looking tensor: raw bytes with a block type + q = rng.integers(0, 255, size=(4, 34), dtype=np.uint8) + w.add_tensor("encoder.q", q, raw_shape=q.shape, raw_dtype=GGMLQuantizationType.Q8_0) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + + +def write_vad(path, license="MIT"): + w = GGUFWriter(path, "silero_vad") + w.add_string("general.name", "silero-vad") + if license: + w.add_string("general.license", license) + w.add_array("silero_vad.sample_rates", [16000, 8000]) + w.add_uint32("silero_vad.lstm.hidden", 128) + rng = np.random.default_rng(2) + w.add_tensor("vad16k.decoder.rnn.bias_ih", rng.standard_normal(32).astype(np.float32)) + w.add_tensor("vad8k.decoder.rnn.bias_ih", rng.standard_normal(32).astype(np.float32)) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + + +ASR_META = {"kind": "asr", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", + "source": "nvidia/stand-in", "attribution": "Stand-in model by NVIDIA", "changes": "Converted to GGUF"} +VAD_META = {"kind": "vad", "license": "MIT", "license_url": "https://github.com/snakers4/silero-vad/blob/master/LICENSE", + "source": "https://github.com/snakers4/silero-vad", "attribution": "Copyright (c) 2020-present Silero Team", + "changes": "Converted to GGUF"} + + +def manifest(path, comps, name="test-bundle", extra=None): + m = {"name": name, "components": comps} + if extra: + m.update(extra) + with open(path, "w") as f: + json.dump(m, f) + + +def comp(name, file, meta, **over): + c = {"name": name, "file": file, **meta} + c.update(over) + return c + + +def sha(path): + return hashlib.sha256(open(path, "rb").read()).hexdigest() + + +def main(): + d = tempfile.mkdtemp(prefix="check_bundle_") + try: + os.chdir(d) + write_asr("asr.gguf") + write_vad("vad.gguf") + manifest("m.json", [comp("asr", "asr.gguf", ASR_META), comp("vad", "vad.gguf", VAD_META)]) + + # --- roundtrip, determinism, verify against the sources --- + rc, out, err = run("--out", "b1.gguf", "--manifest", "m.json") + check(rc == 0, f"build failed: {err}") + rc, out, err = run("--out", "b2.gguf", "--manifest", "m.json") + check(rc == 0 and sha("b1.gguf") == sha("b2.gguf"), "the output is not deterministic") + rc, out, err = run("--verify", "b1.gguf", "--source", "asr=asr.gguf", "--source", "vad=vad.gguf") + check(rc == 0 and "OK" in out, f"verify with sources failed: {err}") + rc, out, err = run("--verify", "b1.gguf") + check(rc == 0, f"verify failed: {err}") + + rc, out, err = run("--list", "b1.gguf", "--json") + j = json.loads(out) if rc == 0 else {} + check(list(j.get("components", {})) == ["asr", "vad"], "list order or content") + a = j.get("components", {}).get("asr", {}) + check(a.get("license") == "CC-BY-4.0" and a.get("tensors") == 3 and "Q8_0" in a.get("types", []), "list: asr info") + check(a.get("source_sha256") == sha("asr.gguf"), "source_sha256 is not the input's sha256") + rc, out, err = run("--list", "b1.gguf") + check(rc == 0 and "asr:" in out and "vad:" in out, "list text") + + rc, out, err = run("--notice", "b1.gguf") + check(rc == 0 and "CC-BY-4.0" in out and "Copyright (c) 2020-present Silero Team" in out and "NVIDIA" in out, + "notice text lacks a licence or credit") + check(sha("asr.gguf") in out, "notice lacks the input sha256") + + # --- a tampered bundle fails verify --- + data = bytearray(open("b1.gguf", "rb").read()) + data[-5] ^= 0xFF + open("tampered.gguf", "wb").write(bytes(data)) + rc, out, err = run("--verify", "tampered.gguf") + check(rc == 1 and "content_sha256" in err, "a flipped tensor byte must fail verify") + open("short.gguf", "wb").write(bytes(data[:-200])) + rc, out, err = run("--verify", "short.gguf") + check(rc != 0, "a truncated bundle must fail verify") + open("junk.gguf", "wb").write(b"not a gguf") + check(run("--verify", "junk.gguf")[0] == 1 and run("--list", "junk.gguf")[0] == 1, "junk must fail cleanly") + check(run("--verify", "asr.gguf")[0] == 1, "a plain GGUF is not a bundle") + # verify with the wrong source + rc, out, err = run("--verify", "b1.gguf", "--source", "asr=vad.gguf") + check(rc == 1, "a source that differs must fail verify") + + # --- refusals at build time --- + def refuse(label, comps, needle, **kw): + manifest("bad.json", comps, **kw) + rc, out, err = run("--out", "bad.gguf", "--manifest", "bad.json") + check(rc == 1 and needle in err, f"{label}: want a refusal with '{needle}', got rc={rc} {err!r}") + check(not os.path.exists("bad.gguf") and not os.path.exists("bad.gguf.tmp"), f"{label}: left a file behind") + + for k in ("license", "license_url", "source", "attribution", "changes"): + meta = {x: y for x, y in ASR_META.items() if x != k} + refuse(f"missing {k}", [comp("asr", "asr.gguf", meta)], f"'{k}'") + refuse(f"empty {k}", [comp("asr", "asr.gguf", ASR_META, **{k: " "})], f"'{k}'") + refuse("non-commercial", [comp("asr", "asr.gguf", ASR_META, license="CC-BY-NC-SA-4.0")], "must not be bundled") + refuse("no derivatives", [comp("asr", "asr.gguf", ASR_META, license="CC-BY-ND-4.0")], "must not be bundled") + refuse("bad licence id", [comp("asr", "asr.gguf", ASR_META, license="some licence")], "SPDX") + refuse("bad licence url", [comp("asr", "asr.gguf", ASR_META, license_url="file:///x")], "http") + write_asr("asr_lic.gguf", license="Apache-2.0") + refuse("input licence conflicts with the manifest", [comp("asr", "asr_lic.gguf", ASR_META)], "declares general.license") + write_asr("asr_lic_same.gguf", license="cc by 4.0") # same licence, other spelling: allowed + manifest("ok2.json", [comp("asr", "asr_lic_same.gguf", {**ASR_META, "license": "CC-BY-4.0"})]) + check(run("--out", "ok2.gguf", "--manifest", "ok2.json")[0] == 0, "an input with the same licence must be accepted") + write_vad("vad_bad_lic.gguf", license="GPL-3.0") + refuse("silero licence conflict", [comp("vad", "vad_bad_lic.gguf", VAD_META)], "declares general.license") + refuse("kind does not match the file", [comp("asr", "vad.gguf", ASR_META)], "kind asr") + refuse("vad kind on an ASR file", [comp("vad", "asr.gguf", VAD_META)], "kind vad") + write_asr("diar.gguf", parakeet_arch="diarization") + refuse("diarization is a later phase", [comp("asr", "diar.gguf", ASR_META)], "diarization") + refuse("unknown kind", [comp("x", "asr.gguf", ASR_META, kind="diarization")], "kind must be") + refuse("duplicate names", [comp("asr", "asr.gguf", ASR_META), comp("asr", "asr.gguf", ASR_META)], "duplicate") + for bad in ("Asr", "a.b", "general", "parakeet", "bundle", "1a", ""): + refuse(f"bad name {bad!r}", [comp(bad, "asr.gguf", ASR_META)], "component name") + refuse("missing file", [comp("asr", "nope.gguf", ASR_META)], "does not exist") + refuse("unknown component key", [comp("asr", "asr.gguf", ASR_META, colour="red")], "unknown keys") + refuse("unknown manifest key", [comp("asr", "asr.gguf", ASR_META)], "unknown keys", extra={"x": 1}) + refuse("no components", [], "components") + open("notgguf.gguf", "wb").write(b"hello") + refuse("input is not a GGUF", [comp("asr", "notgguf.gguf", ASR_META)], "cannot read") + refuse("input is a bundle", [comp("asr", "b1.gguf", ASR_META)], "already a bundle") + write_asr("asr_noarch.gguf", parakeet_arch=None) + refuse("input has no parakeet.arch", [comp("asr", "asr_noarch.gguf", ASR_META)], "kind asr") + # a long component name makes a tensor name too long + w = GGUFWriter("longname.gguf", "parakeet") + w.add_string("parakeet.arch", "tdt"); w.add_uint32("parakeet.encoder.d_model", 4); w.add_uint32("parakeet.vocab_size", 5) + w.add_tensor("t" * 60, np.zeros(4, dtype=np.float32)) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + refuse("tensor name too long", [comp("asr", "longname.gguf", ASR_META)], "longer than") + # the output must not be an input + manifest("same.json", [comp("asr", "asr.gguf", ASR_META)]) + rc, out, err = run("--out", "asr.gguf", "--manifest", "same.json") + check(rc == 1 and "also an input" in err, "output equal to an input must be refused") + check(run("--verify", "b1.gguf", "--source", "asr=asr.gguf")[0] == 0, "the input was damaged by the refused build") + + # mixed licences in one bundle are fine and are reported per component + check(a.get("license") != j["components"]["vad"]["license"], "the bundle mixes two licences") + # a mode is required + check(run()[0] != 0, "no mode must be an error") + finally: + os.chdir("/") + shutil.rmtree(d, ignore_errors=True) + if failures: + print(f"{failures} check(s) failed", file=sys.stderr) + return 1 + print("check_bundle OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0a8e547a713c183c4fcca4ef599ddad7f2276b29 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 3 Oct 2026 22:55:18 +0000 Subject: [PATCH 2/9] feat: load components of a bundle GGUF ModelLoader::load_component opens a bundle with no_alloc, then reads only the tensors of one component into a private context. The component's keys and tensors are seen under their original names, so the rest of the engine treats it like a single-model file. Model::load and SileroVad::load take an optional component name. ModelLoader::load and SileroVad::load without a component refuse a bundle with a message that says so. Plain files take the same path as before. C-API, all additive (no ABI bump): parakeet_ctx* parakeet_capi_load_component(const char* path, const char* name); char* parakeet_capi_bundle_components_json(const char* path); const char* parakeet_capi_load_error(void); parakeet_capi_load on a bundle opens the only ASR component, or its only loadable component, and fails with a message that lists the names when there is more than one candidate. A Silero component gives a VAD context, so the existing vad and external-VAD functions work with it. test_bundle writes synthetic bundles and checks the header, selection, tensor bytes, the bytes read per component, the Silero component against the plain file, malformed and truncated bundles, the C-API calls and that an old reader finds none of its keys. test_bundle_models compares a real bundle with its single-model files. check_bundle covers the script. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- AGENTS.md | 6 + CMakeLists.txt | 1 + include/parakeet_capi.h | 30 +++ src/bundle.cpp | 204 +++++++++++++++++ src/bundle.hpp | 65 ++++++ src/model.cpp | 4 +- src/model.hpp | 7 +- src/model_loader.cpp | 263 ++++++++++++++-------- src/model_loader.hpp | 6 + src/parakeet_capi.cpp | 91 +++++++- src/silero_vad.cpp | 43 +++- src/silero_vad.hpp | 6 +- tests/CMakeLists.txt | 7 + tests/test_bundle.cpp | 420 +++++++++++++++++++++++++++++++++++ tests/test_bundle_models.cpp | 175 +++++++++++++++ 15 files changed, 1221 insertions(+), 107 deletions(-) create mode 100644 src/bundle.cpp create mode 100644 src/bundle.hpp create mode 100644 tests/test_bundle.cpp create mode 100644 tests/test_bundle_models.cpp diff --git a/AGENTS.md b/AGENTS.md index 7d7f8fc..a09fa8e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,7 @@ src/ libparakeet implementation common.hpp/cpp , logging helpers audio_io.hpp/cpp , dr_wav load + linear resample to 16k model_loader.hpp/cpp, GGUF -> ParakeetConfig + name->tensor + bundle.hpp/cpp , bundle GGUF header: components, licences, default component (docs/bundle.md) mel.cpp , log-mel frontend encoder.cpp / conformer.cpp / relpos_attention.cpp ctc_decoder.cpp , CTC head + greedy decode @@ -112,6 +113,7 @@ examples/cli/ parakeet-cli binary diarize binary: diarize [--stream] scripts/ Python tooling convert_parakeet_to_gguf.py, .nemo/.hf -> GGUF (--dtype f32|f16|q8_0) + bundle_gguf.py , merge GGUFs into one bundle; --list/--verify/--notice (docs/bundle.md) convert_hf_parakeet_to_gguf.py, HF safetensors (moondream/parakeet-ultra, -redux) to GGUF (--template, --ternary keep|dequant, --vad keep|drop) gen_nemo_baseline.py , NeMo intermediates -> baseline.gguf gen_stream_baseline.py , NeMo cache-aware streaming encode+decode -> stream baseline.gguf @@ -128,6 +130,9 @@ tests/ ctest targets test_pool_stress.cpp , N threads x K backends == single-thread results; PARAKEET_STRESS_QUICK=1 for slow builds (PARAKEET_TEST_GGUF, _ULTRA, _REDUX_KEEP) test_pool_shutdown.cpp , pool create/destroy leaves no threads, results equal (PARAKEET_TEST_GGUF) test_model_loader.cpp , config + tensor map (model-dependent) + test_bundle.cpp , bundle header, selection, partial read, Silero component, C-API (model-independent, synthetic files) + test_bundle_models.cpp , real bundle == single-model files: transcript, VAD head, Silero (PARAKEET_TEST_BUNDLE, _ASR, _SILERO) + python/check_bundle.py , bundle_gguf.py build/verify/refusals (model-independent) test_capi.cpp , C-API load -> transcribe -> free (model-dependent) test_transcribe_speech.cpp, end-to-end CTC transcript (model-dependent) test_transcribe_tdt.cpp , TDT transcript on speech fixture (model-dependent) @@ -186,6 +191,7 @@ models/ output dir for converted GGUFs (gitignored; MANIFEST.md tracks the expected published set) docs/ conversion.md , GGUF schema reference + bundle.md , bundle GGUF format: several models in one file, per-component licences quantization.md , quantization allowlist, policy, measured size + WER per type parity.md , full model coverage matrix + per-stage tensor parity concurrency.md , backend pool: concurrent requests, thread rules, measured throughput, TSan recipe diff --git a/CMakeLists.txt b/CMakeLists.txt index e7671d9..6507c33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ set(PARAKEET_SRC src/vad_head.cpp src/vad_segmenter.cpp src/vad_json.cpp + src/bundle.cpp src/parakeet_capi.cpp src/common.cpp src/audio_io.cpp diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index c892ff4..31f5e4b 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -71,12 +71,42 @@ typedef struct parakeet_ctx parakeet_ctx; // parakeet_capi_transcribe_path_json_vad*) and Silero VAD contexts are // additive and keep ABI v10: a caller that needs them checks for the symbols // (dlsym) or for PARAKEET_MODEL_KIND_VAD. +// Bundle GGUF (parakeet_capi_load_component, parakeet_capi_bundle_components_json, +// parakeet_capi_load_error; docs/bundle.md) is additive and keeps ABI v10. int parakeet_capi_abi_version(void); // Load a GGUF model. Returns an owning context, or NULL on failure. // The returned context must be released with parakeet_capi_free. +// +// A bundle GGUF (several models in one file, general.architecture +// "parakeet-bundle"; format in docs/bundle.md) loads its ASR component: the only +// component of kind "asr", or, if it has none, its only loadable component. A +// bundle with several candidates is refused with a message that lists them (see +// parakeet_capi_load_error); open one with parakeet_capi_load_component. Only +// the chosen component's tensors are read from disk. parakeet_ctx* parakeet_capi_load(const char* gguf_path); +// Additive (no ABI bump): open one named component of a bundle GGUF. The +// context is of the kind the component declares: "asr" gives an ASR context, +// "vad" a Silero VAD context (parakeet_capi_model_kind PARAKEET_MODEL_KIND_VAD). +// Returns NULL, with a reason in parakeet_capi_load_error, when `gguf_path` is +// not a bundle, has no such component, or has a component kind this build +// cannot load. A NULL or empty `component` behaves like parakeet_capi_load. +parakeet_ctx* parakeet_capi_load_component(const char* gguf_path, const char* component); + +// Additive: the components of a bundle GGUF as a malloc'd JSON array (free with +// parakeet_capi_free_string), read from the header only. One object per +// component: {"name","kind","license","license_url","source","attribution", +// "changes","source_sha256","content_sha256","tensors","bytes"}. NULL (reason +// in parakeet_capi_load_error) when the file is not a bundle or its header is +// malformed or has a newer format version than this build reads. +char* parakeet_capi_bundle_components_json(const char* gguf_path); + +// Additive: the reason the last parakeet_capi_load, parakeet_capi_load_component +// or parakeet_capi_bundle_components_json call on this thread failed. Valid +// until the next of those calls on the same thread; never NULL. +const char* parakeet_capi_load_error(void); + // Free a context obtained from parakeet_capi_load. Safe on NULL. void parakeet_capi_free(parakeet_ctx* ctx); diff --git a/src/bundle.cpp b/src/bundle.cpp new file mode 100644 index 0000000..3819fc5 --- /dev/null +++ b/src/bundle.cpp @@ -0,0 +1,204 @@ +#include "bundle.hpp" + +#include +#include + +#include "ggml.h" +#include "gguf.h" + +namespace pk { + +namespace { + +void set_err(std::string* err, const std::string& m) { + if (err) *err = m; +} + +struct Gguf { + gguf_context* g = nullptr; + ~Gguf() { + if (g) gguf_free(g); + } +}; + +bool get_str(gguf_context* g, const std::string& key, std::string& out) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_STRING) return false; + out = gguf_get_val_str(g, id); + return true; +} + +bool valid_component_name(const std::string& n) { + if (n.empty() || n.size() > 32 || !(n[0] >= 'a' && n[0] <= 'z')) return false; + for (char c : n) + if (!((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_')) return false; + return n != "general" && n != "parakeet" && n != "bundle"; +} + +std::string json_escape(const std::string& s) { + std::string o; + for (unsigned char c : s) { + switch (c) { + case '"': o += "\\\""; break; + case '\\': o += "\\\\"; break; + case '\n': o += "\\n"; break; + case '\r': o += "\\r"; break; + case '\t': o += "\\t"; break; + default: + if (c < 0x20) { + char b[8]; + std::snprintf(b, sizeof b, "\\u%04x", c); + o += b; + } else { + o += (char)c; + } + } + } + return o; +} + +} // namespace + +const BundleComponent* BundleInfo::find(const std::string& n) const { + for (const auto& c : components) + if (c.name == n) return &c; + return nullptr; +} + +bool gguf_is_bundle(const std::string& path) { + gguf_init_params ip{/*no_alloc=*/true, /*ctx=*/nullptr}; + Gguf f; + f.g = gguf_init_from_file(path.c_str(), ip); + if (!f.g) return false; + std::string arch; + return get_str(f.g, "general.architecture", arch) && arch == kBundleArch; +} + +bool read_bundle_info(const std::string& path, BundleInfo& out, std::string* err) { + out = BundleInfo(); + gguf_init_params ip{/*no_alloc=*/true, /*ctx=*/nullptr}; + Gguf f; + f.g = gguf_init_from_file(path.c_str(), ip); + if (!f.g) { + set_err(err, "cannot read " + path + " as a GGUF file"); + return false; + } + gguf_context* g = f.g; + std::string arch; + if (!get_str(g, "general.architecture", arch) || arch != kBundleArch) { + set_err(err, path + " is not a bundle (general.architecture is not \"" + kBundleArch + "\")"); + return false; + } + const int64_t vid = gguf_find_key(g, "parakeet.bundle.version"); + if (vid < 0 || gguf_get_kv_type(g, vid) != GGUF_TYPE_UINT32) { + set_err(err, "bundle has no parakeet.bundle.version"); + return false; + } + out.version = gguf_get_val_u32(g, vid); + if (out.version == 0 || out.version > kBundleVersion) { + set_err(err, "bundle format version " + std::to_string(out.version) + + " is not supported by this reader (it knows up to " + std::to_string(kBundleVersion) + + "); update parakeet.cpp"); + return false; + } + get_str(g, "general.name", out.name); + const int64_t cid = gguf_find_key(g, "parakeet.bundle.components"); + if (cid < 0 || gguf_get_kv_type(g, cid) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, cid) != GGUF_TYPE_STRING) { + set_err(err, "bundle has no parakeet.bundle.components string array"); + return false; + } + const size_t n = gguf_get_arr_n(g, cid); + if (n == 0) { + set_err(err, "bundle lists no components"); + return false; + } + std::set seen; + for (size_t i = 0; i < n; ++i) { + BundleComponent c; + c.name = gguf_get_arr_str(g, cid, i); + if (!valid_component_name(c.name)) { + set_err(err, "bundle component name \"" + c.name + "\" is not valid"); + return false; + } + if (!seen.insert(c.name).second) { + set_err(err, "bundle lists component \"" + c.name + "\" twice"); + return false; + } + const std::string pre = "parakeet.bundle." + c.name + "."; + if (!get_str(g, pre + "kind", c.kind) || c.kind.empty()) { + set_err(err, "component \"" + c.name + "\" has no " + pre + "kind"); + return false; + } + get_str(g, pre + "license", c.license); + get_str(g, pre + "license_url", c.license_url); + get_str(g, pre + "source", c.source); + get_str(g, pre + "attribution", c.attribution); + get_str(g, pre + "changes", c.changes); + get_str(g, pre + "source_sha256", c.source_sha256); + get_str(g, pre + "content_sha256", c.content_sha256); + out.components.push_back(std::move(c)); + } + const int64_t nt = gguf_get_n_tensors(g); + for (int64_t i = 0; i < nt; ++i) { + const std::string tn = gguf_get_tensor_name(g, i); + const size_t dot = tn.find('.'); + if (dot == std::string::npos) continue; + for (auto& c : out.components) { + if (tn.compare(0, dot, c.name) == 0 && dot == c.name.size()) { + ++c.n_tensors; + c.n_bytes += gguf_get_tensor_size(g, i); + break; + } + } + } + for (const auto& c : out.components) { + if (c.n_tensors == 0) { + set_err(err, "component \"" + c.name + "\" has no tensors"); + return false; + } + } + return true; +} + +std::string bundle_component_names(const BundleInfo& info) { + std::string s; + for (const auto& c : info.components) s += (s.empty() ? "" : ", ") + c.name + " (" + c.kind + ")"; + return s; +} + +bool select_default_component(const BundleInfo& info, std::string& name, std::string* err) { + std::vector asr, known; + for (const auto& c : info.components) { + if (c.kind == kBundleKindAsr) asr.push_back(&c); + if (c.kind == kBundleKindAsr || c.kind == kBundleKindVad) known.push_back(&c); + } + const std::vector& pick = !asr.empty() ? asr : known; + if (pick.size() == 1) { + name = pick[0]->name; + return true; + } + if (pick.empty()) + set_err(err, "bundle has no component this reader can load; components: " + bundle_component_names(info)); + else + set_err(err, std::string("bundle has several ") + (asr.empty() ? "loadable" : "asr") + + " components, pick one by name (parakeet_capi_load_component, or --component); " + "components: " + bundle_component_names(info)); + return false; +} + +std::string bundle_components_json(const BundleInfo& info) { + std::string o = "["; + for (size_t i = 0; i < info.components.size(); ++i) { + const auto& c = info.components[i]; + if (i) o += ","; + o += "{\"name\":\"" + json_escape(c.name) + "\",\"kind\":\"" + json_escape(c.kind) + + "\",\"license\":\"" + json_escape(c.license) + "\",\"license_url\":\"" + json_escape(c.license_url) + + "\",\"source\":\"" + json_escape(c.source) + "\",\"attribution\":\"" + json_escape(c.attribution) + + "\",\"changes\":\"" + json_escape(c.changes) + "\",\"source_sha256\":\"" + json_escape(c.source_sha256) + + "\",\"content_sha256\":\"" + json_escape(c.content_sha256) + "\",\"tensors\":" + + std::to_string(c.n_tensors) + ",\"bytes\":" + std::to_string(c.n_bytes) + "}"; + } + return o + "]"; +} + +} // namespace pk diff --git a/src/bundle.hpp b/src/bundle.hpp new file mode 100644 index 0000000..feebb82 --- /dev/null +++ b/src/bundle.hpp @@ -0,0 +1,65 @@ +#pragma once +// Bundle GGUF: several single-model GGUFs in one file (format in docs/bundle.md). +// +// A bundle has general.architecture "parakeet-bundle". Every tensor and key of +// a component is stored as ".". The header lists the +// components and carries the licence and source of each one. Reading the header +// never touches tensor data. +#include +#include +#include + +namespace pk { + +constexpr const char* kBundleArch = "parakeet-bundle"; +constexpr uint32_t kBundleVersion = 1; // highest format version this reader knows + +// Component kinds understood in phase 1. A bundle may hold other kinds (for +// example from a newer writer); a reader skips them. +constexpr const char* kBundleKindAsr = "asr"; +constexpr const char* kBundleKindVad = "vad"; // Silero VAD + +struct BundleComponent { + std::string name; + std::string kind; + std::string license; + std::string license_url; + std::string source; + std::string attribution; + std::string changes; + std::string source_sha256; // sha256 of the single-model GGUF it was built from + std::string content_sha256; // sha256 over the component's tensors (see docs/bundle.md) + uint64_t n_tensors = 0; + uint64_t n_bytes = 0; // tensor data bytes +}; + +struct BundleInfo { + uint32_t version = 0; + std::string name; + std::vector components; + const BundleComponent* find(const std::string& name) const; +}; + +// True when the file is a GGUF whose general.architecture is "parakeet-bundle". +// Reads the header only. +bool gguf_is_bundle(const std::string& path); + +// Reads and checks the bundle header (no tensor data). Returns false and writes a +// one-line reason to *err on a file that is not a bundle, has a newer format +// version, or has a malformed component table. +bool read_bundle_info(const std::string& path, BundleInfo& out, std::string* err); + +// Picks the component that `parakeet_capi_load` opens for a bundle: the only +// component of kind "asr"; failing that, the only component of any known kind. +// Returns false with a message that names the choices when there is none or +// when it is ambiguous. +bool select_default_component(const BundleInfo& info, std::string& name, std::string* err); + +// JSON array: [{"name","kind","license","license_url","source","attribution", +// "changes","source_sha256","content_sha256","tensors","bytes"}] +std::string bundle_components_json(const BundleInfo& info); + +// " (), ..." for messages. +std::string bundle_component_names(const BundleInfo& info); + +} // namespace pk diff --git a/src/model.cpp b/src/model.cpp index 322e951..8864bd4 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -44,12 +44,12 @@ bool arch_prefers_tdt(const std::string& arch) { } } // namespace -std::unique_ptr Model::load(const std::string& gguf_path) { +std::unique_ptr Model::load(const std::string& gguf_path, const std::string& component) { // unique_ptr via private ctor: construct then load. We avoid // std::make_unique (private ctor) and never throw out of here. std::unique_ptr m(new (std::nothrow) Model()); if (!m) return nullptr; - if (!m->loader_.load(gguf_path)) { + if (!(component.empty() ? m->loader_.load(gguf_path) : m->loader_.load_component(gguf_path, component))) { return nullptr; } // Model is the ASR entry point — reject diarization models so the C-API diff --git a/src/model.hpp b/src/model.hpp index 345695a..23daf68 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -42,7 +42,12 @@ struct NBestTranscription { class Model { public: // Loads the GGUF at `gguf_path`. Returns nullptr on failure (no throw). - static std::unique_ptr load(const std::string& gguf_path); + // With a non-empty `component`, `gguf_path` is a bundle GGUF (docs/bundle.md) + // and only that component is read; the model is then the same as the one + // loaded from the component's single-model file. A plain path without a + // component is the unchanged single-model load; a bundle without a + // component is refused. + static std::unique_ptr load(const std::string& gguf_path, const std::string& component = ""); // Transcribe raw mono float PCM. If `sample_rate != 16000` the audio is // linearly resampled to 16 kHz (via pk::resample_linear) before inference. diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 54861d7..2fc7343 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -1,5 +1,6 @@ #include "model_loader.hpp" #include "common.hpp" +#include "bundle.hpp" #include "ggml.h" #include "ggml-backend.h" #include "ggml-alloc.h" @@ -25,15 +26,22 @@ int PromptCfg::resolve_index_or_throw(const std::string& target_lang) const { return idx; } -static uint32_t kv_u32(gguf_context* g, const char* k, uint32_t d=0){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : (uint32_t)gguf_get_val_u32(g,id); +// Key lookup through an optional component prefix. A bundle GGUF stores the +// keys of component "asr" as "asr."; a plain file has prefix "". +struct KvView { + gguf_context* g; + std::string prefix; + int64_t find(const char* k) const { return gguf_find_key(g, (prefix + k).c_str()); } +}; +static uint32_t kv_u32(const KvView& v, const char* k, uint32_t d=0){ gguf_context* g=v.g; + int64_t id = v.find(k); return id<0 ? d : (uint32_t)gguf_get_val_u32(g,id); } -static int32_t kv_i32(gguf_context* g, const char* k, int32_t d=0){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : gguf_get_val_i32(g,id); +static int32_t kv_i32(const KvView& v, const char* k, int32_t d=0){ gguf_context* g=v.g; + int64_t id = v.find(k); return id<0 ? d : gguf_get_val_i32(g,id); } -static std::vector kv_i32_arr(gguf_context* g, const char* k){ +static std::vector kv_i32_arr(const KvView& v, const char* k){ gguf_context* g=v.g; std::vector out; - int64_t id = gguf_find_key(g,k); + int64_t id = v.find(k); if(id>=0 && gguf_get_arr_type(g,id)==GGUF_TYPE_INT32){ size_t n = gguf_get_arr_n(g,id); const int32_t* a = (const int32_t*)gguf_get_arr_data(g,id); @@ -41,9 +49,9 @@ static std::vector kv_i32_arr(gguf_context* g, const char* k){ } return out; } -static std::vector kv_str_arr(gguf_context* g, const char* k){ +static std::vector kv_str_arr(const KvView& v, const char* k){ gguf_context* g=v.g; std::vector out; - int64_t id = gguf_find_key(g,k); + int64_t id = v.find(k); if(id>=0 && gguf_get_arr_type(g,id)==GGUF_TYPE_STRING){ size_t n = gguf_get_arr_n(g,id); out.resize(n); @@ -51,14 +59,14 @@ static std::vector kv_str_arr(gguf_context* g, const char* k){ } return out; } -static float kv_f32(gguf_context* g, const char* k, float d=0){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : gguf_get_val_f32(g,id); +static float kv_f32(const KvView& v, const char* k, float d=0){ gguf_context* g=v.g; + int64_t id = v.find(k); return id<0 ? d : gguf_get_val_f32(g,id); } -static bool kv_bool(gguf_context* g, const char* k, bool d=false){ - int64_t id = gguf_find_key(g,k); return id<0 ? d : gguf_get_val_bool(g,id); +static bool kv_bool(const KvView& v, const char* k, bool d=false){ gguf_context* g=v.g; + int64_t id = v.find(k); return id<0 ? d : gguf_get_val_bool(g,id); } -static std::string kv_str(gguf_context* g, const char* k, const char* d=""){ - int64_t id = gguf_find_key(g,k); return id<0 ? std::string(d) : std::string(gguf_get_val_str(g,id)); +static std::string kv_str(const KvView& v, const char* k, const char* d=""){ gguf_context* g=v.g; + int64_t id = v.find(k); return id<0 ? std::string(d) : std::string(gguf_get_val_str(g,id)); } ModelLoader::~ModelLoader(){ // Free the weight buffer BEFORE the contexts. For the CPU path it is a @@ -133,62 +141,124 @@ bool ModelLoader::realize_weights(ggml_backend_t backend){ return true; } bool ModelLoader::load(const std::string& path){ + // A bundle holds several models. Say so instead of failing later on a + // missing key, and do not read its tensors. + if(gguf_is_bundle(path)){ + PK_LOG("%s is a bundle GGUF (several models in one file); open one component of it " + "(load_component, parakeet_capi_load_component, or --component)", path.c_str()); + return false; + } struct gguf_init_params p{ /*no_alloc*/false, /*ctx*/&ctx_ }; gguf_ = gguf_init_from_file(path.c_str(), p); if(!gguf_){ PK_LOG("gguf open failed: %s", path.c_str()); return false; } - cfg_.arch = kv_str(gguf_, "parakeet.arch"); - cfg_.feat_in = kv_u32(gguf_, "parakeet.encoder.feat_in"); - cfg_.d_model = kv_u32(gguf_, "parakeet.encoder.d_model"); - cfg_.n_layers = kv_u32(gguf_, "parakeet.encoder.n_layers"); - cfg_.n_heads = kv_u32(gguf_, "parakeet.encoder.n_heads"); - cfg_.ff_dim = kv_u32(gguf_, "parakeet.encoder.ff_dim"); - cfg_.conv_kernel = kv_u32(gguf_, "parakeet.encoder.conv_kernel"); - cfg_.conv_norm_type = kv_str(gguf_, "parakeet.encoder.conv_norm_type", "batch_norm"); - cfg_.subsampling_factor = kv_u32(gguf_, "parakeet.encoder.subsampling_factor"); - cfg_.subsampling_conv_channels = kv_u32(gguf_, "parakeet.encoder.subsampling_conv_channels"); - cfg_.xscaling = kv_bool(gguf_, "parakeet.encoder.xscaling", true); - cfg_.pos_emb_max_len = kv_u32(gguf_, "parakeet.encoder.pos_emb_max_len", 5000); + return parse_config(KvView{gguf_, ""}); +} +// Bundle path: open the file with no_alloc (header and descriptors only), then +// read only the tensors of one component into a private ctx. Components that +// were not asked for are never read from disk. The tensors keep their original +// (unprefixed) names, so the rest of the engine sees a plain single-model file. +bool ModelLoader::load_component(const std::string& path, const std::string& component){ + BundleInfo info; std::string err; + if(!read_bundle_info(path, info, &err)){ PK_LOG("%s", err.c_str()); return false; } + if(!info.find(component)){ + PK_LOG("bundle %s has no component '%s'; components: %s", path.c_str(), component.c_str(), + bundle_component_names(info).c_str()); + return false; + } + struct ggml_context* meta = nullptr; + struct gguf_init_params p{ /*no_alloc*/true, /*ctx*/&meta }; + gguf_ = gguf_init_from_file(path.c_str(), p); + if(!gguf_){ PK_LOG("gguf open failed: %s", path.c_str()); return false; } + const std::string pre = component + "."; + const int64_t nt = gguf_get_n_tensors(gguf_); + size_t need = 0; int64_t count = 0; + for(int64_t i=0;itype, GGML_MAX_DIMS, src->ne) : nullptr; + if(!t){ ok = false; break; } + ggml_set_name(t, nm + pre.size()); + const uint64_t off = gguf_get_data_offset(gguf_) + gguf_get_tensor_offset(gguf_,i); + if(off + ggml_nbytes(t) > file_size){ + PK_LOG("bundle %s is truncated (tensor %s)", path.c_str(), nm); ok = false; break; + } + if(fseeko(f, (off_t)off, SEEK_SET)!=0 || fread(t->data, 1, ggml_nbytes(t), f)!=ggml_nbytes(t)) ok = false; + } + fclose(f); + ggml_free(meta); + if(!ok){ PK_LOG("bundle read failed for component '%s'", component.c_str()); return false; } + return parse_config(KvView{gguf_, pre}); +} +bool ModelLoader::parse_config(const KvView& kv){ + cfg_.arch = kv_str(kv, "parakeet.arch"); + cfg_.feat_in = kv_u32(kv, "parakeet.encoder.feat_in"); + cfg_.d_model = kv_u32(kv, "parakeet.encoder.d_model"); + cfg_.n_layers = kv_u32(kv, "parakeet.encoder.n_layers"); + cfg_.n_heads = kv_u32(kv, "parakeet.encoder.n_heads"); + cfg_.ff_dim = kv_u32(kv, "parakeet.encoder.ff_dim"); + cfg_.conv_kernel = kv_u32(kv, "parakeet.encoder.conv_kernel"); + cfg_.conv_norm_type = kv_str(kv, "parakeet.encoder.conv_norm_type", "batch_norm"); + cfg_.subsampling_factor = kv_u32(kv, "parakeet.encoder.subsampling_factor"); + cfg_.subsampling_conv_channels = kv_u32(kv, "parakeet.encoder.subsampling_conv_channels"); + cfg_.xscaling = kv_bool(kv, "parakeet.encoder.xscaling", true); + cfg_.pos_emb_max_len = kv_u32(kv, "parakeet.encoder.pos_emb_max_len", 5000); // cache-aware streaming / causal config (Phase 5). Absent for offline models // -> offline-safe defaults (regular style, no causal, streaming.present=false). - cfg_.att_context_left = kv_i32(gguf_, "parakeet.encoder.att_context_left", -1); - cfg_.att_context_right = kv_i32(gguf_, "parakeet.encoder.att_context_right", -1); - cfg_.att_context_style = kv_str(gguf_, "parakeet.encoder.att_context_style", "regular"); - cfg_.causal_downsampling = kv_bool(gguf_, "parakeet.encoder.causal_downsampling", false); - cfg_.conv_causal = kv_bool(gguf_, "parakeet.encoder.conv_causal", false); + cfg_.att_context_left = kv_i32(kv, "parakeet.encoder.att_context_left", -1); + cfg_.att_context_right = kv_i32(kv, "parakeet.encoder.att_context_right", -1); + cfg_.att_context_style = kv_str(kv, "parakeet.encoder.att_context_style", "regular"); + cfg_.causal_downsampling = kv_bool(kv, "parakeet.encoder.causal_downsampling", false); + cfg_.conv_causal = kv_bool(kv, "parakeet.encoder.conv_causal", false); // encoder.use_bias: false for nemotron (the attention/FFN linear projections // carry no bias tensor). Defaults true so existing models are unaffected. - cfg_.use_bias = kv_bool(gguf_, "parakeet.encoder.use_bias", true); + cfg_.use_bias = kv_bool(kv, "parakeet.encoder.use_bias", true); // Transformer encoder config (diarization models with RoPE attention). // Absent for ASR (FastConformer) models → safe defaults. - cfg_.self_attention_model = kv_str(gguf_, "parakeet.encoder.self_attention_model", ""); - cfg_.qkv_bias = kv_bool(gguf_, "parakeet.encoder.qkv_bias", false); - cfg_.pre_block_norm = kv_bool(gguf_, "parakeet.encoder.pre_block_norm", true); - cfg_.rope_base = kv_f32(gguf_, "parakeet.encoder.rope_base", 10000.0f); - cfg_.rotary_fraction = kv_f32(gguf_, "parakeet.encoder.rotary_fraction", 1.0f); + cfg_.self_attention_model = kv_str(kv, "parakeet.encoder.self_attention_model", ""); + cfg_.qkv_bias = kv_bool(kv, "parakeet.encoder.qkv_bias", false); + cfg_.pre_block_norm = kv_bool(kv, "parakeet.encoder.pre_block_norm", true); + cfg_.rope_base = kv_f32(kv, "parakeet.encoder.rope_base", 10000.0f); + cfg_.rotary_fraction = kv_f32(kv, "parakeet.encoder.rotary_fraction", 1.0f); // Prompt conditioning (multilingual nemotron). Orthogonal capability flag; // absent -> present=false and the engine skips the prompt stage entirely. - cfg_.prompt.present = kv_bool(gguf_, "parakeet.prompt.present", false); + cfg_.prompt.present = kv_bool(kv, "parakeet.prompt.present", false); if(cfg_.prompt.present){ - cfg_.prompt.num_prompts = kv_u32(gguf_, "parakeet.prompt.num_prompts", 0); - cfg_.prompt.default_lang = kv_str(gguf_, "parakeet.prompt.default_lang", ""); - cfg_.prompt.dict_keys = kv_str_arr(gguf_, "parakeet.prompt.dictionary.keys"); - cfg_.prompt.dict_vals = kv_i32_arr(gguf_, "parakeet.prompt.dictionary.values"); + cfg_.prompt.num_prompts = kv_u32(kv, "parakeet.prompt.num_prompts", 0); + cfg_.prompt.default_lang = kv_str(kv, "parakeet.prompt.default_lang", ""); + cfg_.prompt.dict_keys = kv_str_arr(kv, "parakeet.prompt.dictionary.keys"); + cfg_.prompt.dict_vals = kv_i32_arr(kv, "parakeet.prompt.dictionary.values"); } - cfg_.ternary.present = kv_bool(gguf_, "parakeet.ternary.present", false); + cfg_.ternary.present = kv_bool(kv, "parakeet.ternary.present", false); if(cfg_.ternary.present){ - cfg_.ternary.group_size = kv_u32(gguf_, "parakeet.ternary.group_size", 128); + cfg_.ternary.group_size = kv_u32(kv, "parakeet.ternary.group_size", 128); if(cfg_.ternary.group_size != 128){ PK_LOG("invalid packed ternary GGUF: parakeet.ternary.group_size is %u, only 128 is supported", (unsigned)cfg_.ternary.group_size); return false; } } - cfg_.vad.present = kv_bool(gguf_, "parakeet.vad.present", false); + cfg_.vad.present = kv_bool(kv, "parakeet.vad.present", false); if(cfg_.vad.present){ - cfg_.vad.d_in = kv_u32(gguf_, "parakeet.vad.d_in", 0); - cfg_.vad.hidden = kv_u32(gguf_, "parakeet.vad.hidden", 0); - cfg_.vad.kernel = kv_u32(gguf_, "parakeet.vad.kernel", 0); - cfg_.vad.frame_sec = kv_f32(gguf_, "parakeet.vad.frame_sec", 0.08f); + cfg_.vad.d_in = kv_u32(kv, "parakeet.vad.d_in", 0); + cfg_.vad.hidden = kv_u32(kv, "parakeet.vad.hidden", 0); + cfg_.vad.kernel = kv_u32(kv, "parakeet.vad.kernel", 0); + cfg_.vad.frame_sec = kv_f32(kv, "parakeet.vad.frame_sec", 0.08f); if(!std::isfinite(cfg_.vad.frame_sec) || !(cfg_.vad.frame_sec > 0.0f) || cfg_.vad.d_in==0 || cfg_.vad.hidden==0 || cfg_.vad.kernel==0 || (cfg_.vad.kernel % 2)==0){ PK_LOG("invalid VAD config: frame_sec=%g d_in=%u hidden=%u kernel=%u (need finite frame_sec > 0, " @@ -199,20 +269,20 @@ bool ModelLoader::load(const std::string& path){ } if(cfg_.att_context_style != "regular"){ StreamingCfg& s = cfg_.streaming; - s.chunk_size = kv_i32_arr(gguf_, "parakeet.streaming.chunk_size"); - s.shift_size = kv_i32_arr(gguf_, "parakeet.streaming.shift_size"); - s.pre_encode_cache_size = kv_i32_arr(gguf_, "parakeet.streaming.pre_encode_cache_size"); - s.cache_drop_size = kv_i32(gguf_, "parakeet.streaming.cache_drop_size", 0); - s.last_channel_cache_size = kv_i32(gguf_, "parakeet.streaming.last_channel_cache_size", 0); - s.valid_out_len = kv_i32(gguf_, "parakeet.streaming.valid_out_len", 0); - s.drop_extra_pre_encoded = kv_i32(gguf_, "parakeet.streaming.drop_extra_pre_encoded", 0); + s.chunk_size = kv_i32_arr(kv, "parakeet.streaming.chunk_size"); + s.shift_size = kv_i32_arr(kv, "parakeet.streaming.shift_size"); + s.pre_encode_cache_size = kv_i32_arr(kv, "parakeet.streaming.pre_encode_cache_size"); + s.cache_drop_size = kv_i32(kv, "parakeet.streaming.cache_drop_size", 0); + s.last_channel_cache_size = kv_i32(kv, "parakeet.streaming.last_channel_cache_size", 0); + s.valid_out_len = kv_i32(kv, "parakeet.streaming.valid_out_len", 0); + s.drop_extra_pre_encoded = kv_i32(kv, "parakeet.streaming.drop_extra_pre_encoded", 0); s.present = true; } - cfg_.sample_rate = kv_u32(gguf_, "parakeet.preprocessor.sample_rate", 16000); - cfg_.n_mels = kv_u32(gguf_, "parakeet.preprocessor.n_mels"); - cfg_.n_fft = kv_u32(gguf_, "parakeet.preprocessor.n_fft"); - cfg_.win_length = kv_u32(gguf_, "parakeet.preprocessor.win_length"); - cfg_.hop_length = kv_u32(gguf_, "parakeet.preprocessor.hop_length"); + cfg_.sample_rate = kv_u32(kv, "parakeet.preprocessor.sample_rate", 16000); + cfg_.n_mels = kv_u32(kv, "parakeet.preprocessor.n_mels"); + cfg_.n_fft = kv_u32(kv, "parakeet.preprocessor.n_fft"); + cfg_.win_length = kv_u32(kv, "parakeet.preprocessor.win_length"); + cfg_.hop_length = kv_u32(kv, "parakeet.preprocessor.hop_length"); if(cfg_.vad.present){ const double enc_frame = (double)cfg_.hop_length * (double)cfg_.subsampling_factor / (double)cfg_.sample_rate; const double ratio = enc_frame > 0.0 ? (double)cfg_.vad.frame_sec / enc_frame : 0.0; @@ -222,53 +292,53 @@ bool ModelLoader::load(const std::string& path){ return false; } } - cfg_.preemph = kv_f32(gguf_, "parakeet.preprocessor.preemph", 0.0f); - cfg_.mag_power = kv_f32(gguf_, "parakeet.preprocessor.mag_power", 2.0f); - cfg_.normalize = kv_str(gguf_, "parakeet.preprocessor.normalize", "per_feature"); - cfg_.log_zero_guard = kv_f32(gguf_, "parakeet.preprocessor.log_zero_guard", 0.0f); - cfg_.pred_hidden = kv_u32(gguf_, "parakeet.decoder.pred_hidden"); - cfg_.pred_rnn_layers = kv_u32(gguf_, "parakeet.decoder.pred_rnn_layers"); - cfg_.joint_hidden = kv_u32(gguf_, "parakeet.joint.joint_hidden"); - cfg_.joint_activation = kv_str(gguf_, "parakeet.joint.activation"); - cfg_.max_symbols = kv_u32(gguf_, "parakeet.decoding.max_symbols", 10); - cfg_.vocab_size = kv_u32(gguf_, "parakeet.vocab_size"); - cfg_.blank_id = kv_u32(gguf_, "parakeet.blank_id"); + cfg_.preemph = kv_f32(kv, "parakeet.preprocessor.preemph", 0.0f); + cfg_.mag_power = kv_f32(kv, "parakeet.preprocessor.mag_power", 2.0f); + cfg_.normalize = kv_str(kv, "parakeet.preprocessor.normalize", "per_feature"); + cfg_.log_zero_guard = kv_f32(kv, "parakeet.preprocessor.log_zero_guard", 0.0f); + cfg_.pred_hidden = kv_u32(kv, "parakeet.decoder.pred_hidden"); + cfg_.pred_rnn_layers = kv_u32(kv, "parakeet.decoder.pred_rnn_layers"); + cfg_.joint_hidden = kv_u32(kv, "parakeet.joint.joint_hidden"); + cfg_.joint_activation = kv_str(kv, "parakeet.joint.activation"); + cfg_.max_symbols = kv_u32(kv, "parakeet.decoding.max_symbols", 10); + cfg_.vocab_size = kv_u32(kv, "parakeet.vocab_size"); + cfg_.blank_id = kv_u32(kv, "parakeet.blank_id"); // diarization config (absent for ASR models → present=false) - if (gguf_find_key(gguf_, "parakeet.diar.n_speakers") >= 0) { + if (kv.find("parakeet.diar.n_speakers") >= 0) { auto& d = cfg_.diarization; d.present = true; - d.n_speakers = kv_u32(gguf_, "parakeet.diar.n_speakers"); - d.tf_d_model = kv_u32(gguf_, "parakeet.diar.tf_d_model"); - d.upsample_factor = kv_u32(gguf_, "parakeet.diar.upsample_factor"); - d.frame_resolution_sec = kv_f32(gguf_, "parakeet.diar.frame_resolution_sec", 0.01f); - d.onset_threshold = kv_f32(gguf_, "parakeet.diar.onset_threshold", 0.5f); - d.offset_threshold = kv_f32(gguf_, "parakeet.diar.offset_threshold", 0.5f); + d.n_speakers = kv_u32(kv, "parakeet.diar.n_speakers"); + d.tf_d_model = kv_u32(kv, "parakeet.diar.tf_d_model"); + d.upsample_factor = kv_u32(kv, "parakeet.diar.upsample_factor"); + d.frame_resolution_sec = kv_f32(kv, "parakeet.diar.frame_resolution_sec", 0.01f); + d.onset_threshold = kv_f32(kv, "parakeet.diar.onset_threshold", 0.5f); + d.offset_threshold = kv_f32(kv, "parakeet.diar.offset_threshold", 0.5f); // Streaming (speaker cache) config, in ENCODER frames as in NeMo // SortformerModules. Defaults are the Nemotron-3-Diarization values, // for GGUFs converted before these keys were written. - d.streaming_mode = kv_bool(gguf_, "parakeet.diar.streaming_mode", true); - d.chunk_len = (int32_t)kv_u32(gguf_, "parakeet.diar.chunk_len", 264); - d.spkcache_len = (int32_t)kv_u32(gguf_, "parakeet.diar.spkcache_len", 264); - d.fifo_len = (int32_t)kv_u32(gguf_, "parakeet.diar.fifo_len", 0); - d.spkcache_update_period = (int32_t)kv_u32(gguf_, "parakeet.diar.spkcache_update_period", 264); - d.spkcache_sil_frames_per_spk = (int32_t)kv_u32(gguf_, "parakeet.diar.spkcache_sil_frames_per_spk", 1); - d.sil_threshold = kv_f32(gguf_, "parakeet.diar.sil_threshold", 0.2f); - d.pred_score_threshold = kv_f32(gguf_, "parakeet.diar.pred_score_threshold", 0.25f); - d.scores_boost_latest = kv_f32(gguf_, "parakeet.diar.scores_boost_latest", 0.05f); - d.strong_boost_rate = kv_f32(gguf_, "parakeet.diar.strong_boost_rate", 0.75f); - d.weak_boost_rate = kv_f32(gguf_, "parakeet.diar.weak_boost_rate", 1.5f); - d.min_pos_scores_rate = kv_f32(gguf_, "parakeet.diar.min_pos_scores_rate", 0.5f); - d.use_learnable_sil_emb = kv_bool(gguf_, "parakeet.diar.use_learnable_sil_emb", - gguf_find_tensor(gguf_, "sortformer_modules.learnable_sil_emb") >= 0); + d.streaming_mode = kv_bool(kv, "parakeet.diar.streaming_mode", true); + d.chunk_len = (int32_t)kv_u32(kv, "parakeet.diar.chunk_len", 264); + d.spkcache_len = (int32_t)kv_u32(kv, "parakeet.diar.spkcache_len", 264); + d.fifo_len = (int32_t)kv_u32(kv, "parakeet.diar.fifo_len", 0); + d.spkcache_update_period = (int32_t)kv_u32(kv, "parakeet.diar.spkcache_update_period", 264); + d.spkcache_sil_frames_per_spk = (int32_t)kv_u32(kv, "parakeet.diar.spkcache_sil_frames_per_spk", 1); + d.sil_threshold = kv_f32(kv, "parakeet.diar.sil_threshold", 0.2f); + d.pred_score_threshold = kv_f32(kv, "parakeet.diar.pred_score_threshold", 0.25f); + d.scores_boost_latest = kv_f32(kv, "parakeet.diar.scores_boost_latest", 0.05f); + d.strong_boost_rate = kv_f32(kv, "parakeet.diar.strong_boost_rate", 0.75f); + d.weak_boost_rate = kv_f32(kv, "parakeet.diar.weak_boost_rate", 1.5f); + d.min_pos_scores_rate = kv_f32(kv, "parakeet.diar.min_pos_scores_rate", 0.5f); + d.use_learnable_sil_emb = kv_bool(kv, "parakeet.diar.use_learnable_sil_emb", + gguf_find_tensor(gguf_, (kv.prefix + "sortformer_modules.learnable_sil_emb").c_str()) >= 0); } // durations array (stored as INT32 by the converter) - { int64_t id = gguf_find_key(gguf_, "parakeet.tdt.durations"); + { int64_t id = kv.find("parakeet.tdt.durations"); if(id>=0 && gguf_get_arr_type(gguf_,id)==GGUF_TYPE_INT32){ size_t n = gguf_get_arr_n(gguf_,id); const int32_t* a = (const int32_t*)gguf_get_arr_data(gguf_,id); cfg_.tdt_durations.assign(a, a+n); } } // tokenizer pieces STRING array - { int64_t id = gguf_find_key(gguf_, "parakeet.tokenizer.pieces"); + { int64_t id = kv.find("parakeet.tokenizer.pieces"); if(id>=0 && gguf_get_arr_type(gguf_,id)==GGUF_TYPE_STRING){ size_t n = gguf_get_arr_n(gguf_,id); cfg_.tokenizer_pieces.resize(n); @@ -276,7 +346,10 @@ bool ModelLoader::load(const std::string& path){ cfg_.tokenizer_pieces[i] = gguf_get_arr_str(gguf_,id,i); } } // tensors const int64_t nt = gguf_get_n_tensors(gguf_); + // (a bundle ctx holds only this component's tensors, named without the prefix) for(int64_t i=0;i0 && (cfg_.vocab_size>0 || cfg_.arch=="diarization"); } diff --git a/src/model_loader.hpp b/src/model_loader.hpp index fefa558..e7abc61 100644 --- a/src/model_loader.hpp +++ b/src/model_loader.hpp @@ -13,6 +13,7 @@ typedef struct ggml_backend_buffer* ggml_backend_buffer_t; struct ggml_backend; typedef struct ggml_backend* ggml_backend_t; namespace pk { +struct KvView; // Cache-aware streaming params (Phase 5), populated only for streaming models // (att_context_style != "regular"). Mirrors NeMo CacheAwareStreamingConfig. // List fields are stored as int32 arrays in the GGUF (e.g. chunk_size=[9,16]). @@ -129,6 +130,10 @@ class ModelLoader { ModelLoader() = default; ~ModelLoader(); bool load(const std::string& path); + // Loads one component of a bundle GGUF (docs/bundle.md). Reads only that + // component's tensors; the tensors and keys are seen under their original + // names. load() refuses a bundle with a message that points here. + bool load_component(const std::string& path, const std::string& component); const ParakeetConfig& config() const { return cfg_; } const std::vector& tokenizer_pieces() const { return cfg_.tokenizer_pieces; } ggml_tensor* tensor(const std::string& name) const; // nullptr if absent @@ -158,6 +163,7 @@ class ModelLoader { // tensors they were built from. std::shared_ptr& ternary_store() const { return ternary_store_; } private: + bool parse_config(const KvView& kv); mutable std::shared_ptr ternary_store_; ParakeetConfig cfg_; gguf_context* gguf_ = nullptr; diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index da8c2e0..d13812b 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -22,6 +22,7 @@ #include "transcription_json.hpp" #include "vad_json.hpp" #include "silero_vad.hpp" +#include "bundle.hpp" #include "audio_io.hpp" #include @@ -241,7 +242,66 @@ extern "C" int parakeet_capi_abi_version(void) { return PARAKEET_CAPI_ABI_VERSION; } -extern "C" parakeet_ctx* parakeet_capi_load(const char* gguf_path) { +namespace { + +// Reason for the last failed parakeet_capi_load / _load_component / +// _bundle_components_json on this thread (there is no context to hold it). +thread_local std::string g_load_error; + +parakeet_ctx* load_plain(const char* gguf_path); + +// Opens one component of a bundle GGUF, by the kind recorded in the header. +parakeet_ctx* load_bundle_component(const char* path, const std::string& name) { + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(path, info, &err)) { g_load_error = err; return nullptr; } + const pk::BundleComponent* c = info.find(name); + if (!c) { + g_load_error = std::string("bundle has no component \"") + name + "\"; components: " + + pk::bundle_component_names(info); + return nullptr; + } + std::unique_ptr ctx(new (std::nothrow) parakeet_ctx()); + if (!ctx) { g_load_error = "out of memory"; return nullptr; } + if (c->kind == pk::kBundleKindAsr) { + ctx->model = pk::Model::load(path, name); + if (!ctx->model) { + g_load_error = "cannot load ASR component \"" + name + "\" (see the log for the reason)"; + return nullptr; + } + } else if (c->kind == pk::kBundleKindVad) { + ctx->silero = pk::SileroVad::load(path, &err, name); + if (!ctx->silero) { g_load_error = err; return nullptr; } + } else { + g_load_error = "component \"" + name + "\" has kind \"" + c->kind + "\", which this build cannot load"; + return nullptr; + } + return ctx.release(); +} + +parakeet_ctx* load_impl(const char* path, const char* component) { + g_load_error.clear(); + if (!path) { g_load_error = "path is NULL"; return nullptr; } + const bool bundle = pk::gguf_is_bundle(path); + if (component && *component) { + if (!bundle) { g_load_error = std::string(path) + " is not a bundle GGUF, so it has no components"; return nullptr; } + return load_bundle_component(path, component); + } + if (bundle) { + pk::BundleInfo info; + std::string err, name; + if (!pk::read_bundle_info(path, info, &err) || !pk::select_default_component(info, name, &err)) { + g_load_error = err; + return nullptr; + } + return load_bundle_component(path, name); + } + parakeet_ctx* c = load_plain(path); + if (!c) g_load_error = std::string("cannot load ") + path + " (see the log for the reason)"; + return c; +} + +parakeet_ctx* load_plain(const char* gguf_path) { if (!gguf_path) return nullptr; try { auto* ctx = new (std::nothrow) parakeet_ctx(); @@ -301,6 +361,35 @@ extern "C" parakeet_ctx* parakeet_capi_load(const char* gguf_path) { } } +} // namespace + +extern "C" parakeet_ctx* parakeet_capi_load(const char* gguf_path) { + try { return load_impl(gguf_path, nullptr); } + catch (...) { g_load_error = "unknown error"; return nullptr; } +} + +extern "C" parakeet_ctx* parakeet_capi_load_component(const char* gguf_path, const char* component) { + try { return load_impl(gguf_path, component); } + catch (...) { g_load_error = "unknown error"; return nullptr; } +} + +extern "C" char* parakeet_capi_bundle_components_json(const char* gguf_path) { + g_load_error.clear(); + if (!gguf_path) { g_load_error = "path is NULL"; return nullptr; } + try { + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(gguf_path, info, &err)) { g_load_error = err; return nullptr; } + char* out = dup_to_c(pk::bundle_components_json(info)); + if (!out) g_load_error = "out of memory"; + return out; + } catch (...) { g_load_error = "unknown error"; return nullptr; } +} + +extern "C" const char* parakeet_capi_load_error(void) { + return g_load_error.c_str(); +} + extern "C" void parakeet_capi_free(parakeet_ctx* ctx) { delete ctx; // safe on nullptr; ~unique_ptr releases the model. } diff --git a/src/silero_vad.cpp b/src/silero_vad.cpp index b4ae5b4..e247b76 100644 --- a/src/silero_vad.cpp +++ b/src/silero_vad.cpp @@ -11,6 +11,7 @@ #include "ggml.h" #include "ggml-alloc.h" #include "ggml-backend.h" +#include "bundle.hpp" #include "ggml_graph.hpp" #include "gguf.h" @@ -182,7 +183,29 @@ std::string shape_str(const ggml_tensor* t) { } // namespace -std::unique_ptr SileroVad::load(const std::string& path, std::string* err) { +std::unique_ptr SileroVad::load(const std::string& path, std::string* err, const std::string& component) { + // A component of a bundle GGUF (docs/bundle.md): every key and tensor name has + // the "." prefix, and only those tensors are read. + std::string pfx; + if (!component.empty()) { + BundleInfo info; + std::string berr; + if (!read_bundle_info(path, info, &berr)) { + set_err(err, berr); + return nullptr; + } + const BundleComponent* bc = info.find(component); + if (!bc) { + set_err(err, path + ": no component \"" + component + "\"; components: " + bundle_component_names(info)); + return nullptr; + } + if (bc->kind != kBundleKindVad) { + set_err(err, path + ": component \"" + component + "\" has kind \"" + bc->kind + "\", not \"" + + kBundleKindVad + "\""); + return nullptr; + } + pfx = component + "."; + } ggml_context* meta = nullptr; gguf_init_params gp{/*no_alloc*/ true, /*ctx*/ &meta}; gguf_context* g = gguf_init_from_file(path.c_str(), gp); @@ -205,13 +228,13 @@ std::unique_ptr SileroVad::load(const std::string& path, std::string* // Typed metadata reads: a key of the wrong type is an error, not an abort. auto u32 = [&](const std::string& k, uint32_t* out) { - const int64_t id = gguf_find_key(g, k.c_str()); + const int64_t id = gguf_find_key(g, (pfx + k).c_str()); if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_UINT32) return false; *out = gguf_get_val_u32(g, id); return true; }; auto i32_array = [&](const std::string& k, std::vector* out) { - const int64_t id = gguf_find_key(g, k.c_str()); + const int64_t id = gguf_find_key(g, (pfx + k).c_str()); if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY || gguf_get_arr_type(g, id) != GGUF_TYPE_INT32) return false; const int32_t* d = (const int32_t*)gguf_get_arr_data(g, id); @@ -219,7 +242,11 @@ std::unique_ptr SileroVad::load(const std::string& path, std::string* return true; }; - const int64_t arch = gguf_find_key(g, "general.architecture"); + if (pfx.empty() && gguf_find_key(g, "general.architecture") >= 0 && + gguf_get_kv_type(g, gguf_find_key(g, "general.architecture")) == GGUF_TYPE_STRING && + std::string(gguf_get_val_str(g, gguf_find_key(g, "general.architecture"))) == kBundleArch) + return fail("this is a bundle GGUF; load its Silero component by name"); + const int64_t arch = gguf_find_key(g, (pfx + "general.architecture").c_str()); if (arch < 0 || gguf_get_kv_type(g, arch) != GGUF_TYPE_STRING) return fail("no general.architecture; this is not a Silero VAD GGUF"); if (std::string(gguf_get_val_str(g, arch)) != "silero_vad") @@ -246,8 +273,10 @@ std::unique_ptr SileroVad::load(const std::string& path, std::string* if (!m->ctx_) return fail("ggml_init failed"); std::vector> todo; // dst, gguf tensor id for (int64_t i = 0; i < nt; ++i) { - const char* name = gguf_get_tensor_name(g, i); - ggml_tensor* src = ggml_get_tensor(meta, name); + const char* full = gguf_get_tensor_name(g, i); + if (std::strncmp(full, pfx.c_str(), pfx.size()) != 0) continue; // another component + const char* name = full + pfx.size(); + ggml_tensor* src = ggml_get_tensor(meta, full); if (!src) return fail(std::string("tensor ") + name + " has no metadata"); if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16) return fail(std::string("tensor ") + name + " has type " + ggml_type_name(src->type) + @@ -318,7 +347,7 @@ std::unique_ptr SileroVad::load(const std::string& path, std::string* std::vector raw; std::vector wide; for (auto& pr : todo) { - ggml_tensor* src = ggml_get_tensor(meta, pr.first->name); + ggml_tensor* src = ggml_get_tensor(meta, (pfx + pr.first->name).c_str()); const size_t n = (size_t)ggml_nelements(src); raw.resize(ggml_nbytes(src)); f.seekg((std::streamoff)(base + gguf_get_tensor_offset(g, pr.second))); diff --git a/src/silero_vad.hpp b/src/silero_vad.hpp index 213564a..e1af9de 100644 --- a/src/silero_vad.hpp +++ b/src/silero_vad.hpp @@ -74,7 +74,11 @@ class Framer { class SileroVad { public: // Returns nullptr on failure and writes a one-line reason to *err (when not null). - static std::unique_ptr load(const std::string& gguf_path, std::string* err = nullptr); + // With a non-empty `component`, `gguf_path` is a bundle GGUF (docs/bundle.md): + // the Silero weights are read from that component only, and no other + // component's tensor data is read. + static std::unique_ptr load(const std::string& gguf_path, std::string* err = nullptr, + const std::string& component = ""); ~SileroVad(); SileroVad(const SileroVad&) = delete; SileroVad& operator=(const SileroVad&) = delete; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0c7342b..d71a71b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -221,6 +221,9 @@ if(PARAKEET_PY) add_test(NAME check_hf_convert COMMAND ${PARAKEET_PY} ${CMAKE_SOURCE_DIR}/tests/python/check_hf_convert.py) set_tests_properties(check_hf_convert PROPERTIES SKIP_RETURN_CODE 77) + + add_test(NAME check_bundle COMMAND ${PARAKEET_PY} ${CMAKE_SOURCE_DIR}/tests/python/check_bundle.py) + set_tests_properties(check_bundle PROPERTIES SKIP_RETURN_CODE 77) endif() pk_add_test(test_speaker_profiles) @@ -236,3 +239,7 @@ pk_add_test(test_transcribe_vad_silero) target_compile_definitions(test_silero_vad PRIVATE PK_SOURCE_DIR="${CMAKE_SOURCE_DIR}") set_tests_properties(test_silero_vad test_capi_vad_silero test_transcribe_vad_silero PROPERTIES LABELS "model") set_tests_properties(test_capi_vad_silero test_transcribe_vad_silero PROPERTIES WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + +pk_add_test(test_bundle) +pk_add_test(test_bundle_models) +set_tests_properties(test_bundle_models PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) diff --git a/tests/test_bundle.cpp b/tests/test_bundle.cpp new file mode 100644 index 0000000..94fac02 --- /dev/null +++ b/tests/test_bundle.cpp @@ -0,0 +1,420 @@ +// Bundle GGUF (docs/bundle.md): header parsing, component selection, partial +// loading, the Silero component, malformed files and the C-API entry points. +// No model file is needed: the test writes small synthetic GGUF files itself. +// A synthetic "asr" component only has the keys and tensors that ModelLoader +// reads; the Silero component has the real tensor shapes with random weights. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bundle.hpp" +#include "ggml.h" +#include "gguf.h" +#include "model_loader.hpp" +#include "parakeet_capi.h" +#include "silero_vad.hpp" + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +namespace { + +// Everything one synthetic GGUF needs. Tensors live in `ctx`. +struct Builder { + gguf_context* g = gguf_init_empty(); + ggml_context* ctx = nullptr; + std::string pre; // component prefix ("" for a plain file) + Builder() { + ggml_init_params ip{64u << 20, nullptr, false}; + ctx = ggml_init(ip); + } + ~Builder() { + gguf_free(g); + ggml_free(ctx); + } + void str(const std::string& k, const std::string& v) { gguf_set_val_str(g, (pre + k).c_str(), v.c_str()); } + void u32(const std::string& k, uint32_t v) { gguf_set_val_u32(g, (pre + k).c_str(), v); } + void i32s(const std::string& k, const std::vector& v) { + gguf_set_arr_data(g, (pre + k).c_str(), GGUF_TYPE_INT32, v.data(), v.size()); + } + // F32 or F16 tensor with seeded random values in [-s, s]. + void tensor(const std::string& name, std::vector ne, std::mt19937& rng, float s, bool f16 = false) { + ggml_tensor* t = ggml_new_tensor(ctx, f16 ? GGML_TYPE_F16 : GGML_TYPE_F32, (int)ne.size(), ne.data()); + ggml_set_name(t, (pre + name).c_str()); + std::uniform_real_distribution d(-s, s); + const int64_t n = ggml_nelements(t); + for (int64_t i = 0; i < n; ++i) { + const float v = d(rng); + if (f16) ((ggml_fp16_t*)t->data)[i] = ggml_fp32_to_fp16(v); + else ((float*)t->data)[i] = v; + } + gguf_add_tensor(g, t); + } + bool write(const std::string& path) { return gguf_write_to_file(g, path.c_str(), false); } +}; + +// The keys and tensors of a Silero VAD file, under `b.pre`. +void silero_content(Builder& b, uint32_t seed) { + std::mt19937 rng(seed); + b.str("general.architecture", "silero_vad"); + b.str("general.license", "MIT"); + b.u32("silero_vad.lstm.hidden", 128); + b.u32("silero_vad.encoder.n_layers", 4); + b.u32("silero_vad.encoder.kernel", 3); + b.i32s("silero_vad.encoder.strides", {1, 2, 2, 1}); + b.i32s("silero_vad.encoder.channels", {128, 64, 64, 128}); + b.i32s("silero_vad.sample_rates", {16000, 8000}); + struct R { int sr; int chunk, ctx, n_fft, hop, rpad; const char* p; } rates[] = { + {16000, 512, 64, 256, 128, 64, "vad16k."}, {8000, 256, 32, 128, 64, 32, "vad8k."}}; + for (const R& r : rates) { + const std::string k = "silero_vad." + std::to_string(r.sr); + b.u32(k + ".chunk_samples", r.chunk); + b.u32(k + ".context_samples", r.ctx); + b.u32(k + ".stft.n_fft", r.n_fft); + b.u32(k + ".stft.hop", r.hop); + b.u32(k + ".stft.right_reflect_pad", r.rpad); + const int64_t bins = r.n_fft / 2 + 1; + const int64_t in_ch[4] = {bins, 128, 64, 64}, out_ch[4] = {128, 64, 64, 128}; + const std::string p = r.p; + b.tensor(p + "stft.forward_basis_buffer", {r.n_fft, 1, 2 * bins}, rng, 0.2f); + for (int i = 0; i < 4; ++i) { + b.tensor(p + "encoder." + std::to_string(i) + ".reparam_conv.weight", {3, in_ch[i], out_ch[i]}, rng, 0.2f, true); + b.tensor(p + "encoder." + std::to_string(i) + ".reparam_conv.bias", {out_ch[i]}, rng, 0.1f); + } + b.tensor(p + "decoder.rnn.weight_ih", {128, 512}, rng, 0.2f, true); + b.tensor(p + "decoder.rnn.weight_hh", {128, 512}, rng, 0.2f, true); + b.tensor(p + "decoder.rnn.bias_ih", {512}, rng, 0.1f); + b.tensor(p + "decoder.rnn.bias_hh", {512}, rng, 0.1f); + b.tensor(p + "decoder.decoder.2.weight", {1, 128, 1}, rng, 0.2f); + b.tensor(p + "decoder.decoder.2.bias", {1}, rng, 0.1f); + } +} + +// A stand-in ASR component: the keys ModelLoader needs and two tensors, one big. +void asr_content(Builder& b, uint32_t seed, int64_t big_floats) { + std::mt19937 rng(seed); + b.str("general.architecture", "parakeet"); + b.str("parakeet.arch", "tdt"); + b.u32("parakeet.encoder.d_model", 16); + b.u32("parakeet.encoder.n_layers", 2); + b.u32("parakeet.encoder.n_heads", 2); + b.u32("parakeet.vocab_size", 5); + b.u32("parakeet.blank_id", 5); + b.tensor("encoder.small", {16, 4}, rng, 1.0f); + b.tensor("encoder.big", {big_floats}, rng, 1.0f); +} + +struct Comp { + std::string name, kind; + bool omit_tensors = false; +}; + +// Writes a bundle. `defs` gives each component's content and kind. +bool write_bundle(const std::string& path, const std::vector& defs, uint32_t version = 1, + bool with_components_key = true, int64_t big_floats = 1 << 20) { + Builder b; + b.str("general.architecture", "parakeet-bundle"); + b.str("general.name", "test-bundle"); + b.u32("parakeet.bundle.version", version); + std::vector names; + for (const Comp& c : defs) names.push_back(c.name.c_str()); + if (with_components_key) gguf_set_arr_str(b.g, "parakeet.bundle.components", names.data(), names.size()); + uint32_t seed = 1; + for (const Comp& c : defs) { + const std::string pk = "parakeet.bundle." + c.name + "."; + b.str(pk + "kind", c.kind); + b.str(pk + "license", c.kind == "vad" ? "MIT" : "CC-BY-4.0"); + b.str(pk + "license_url", "https://example.org/license"); + b.str(pk + "source", "example/" + c.name); + b.str(pk + "attribution", "Credit " + c.name); + b.str(pk + "changes", "converted"); + if (c.omit_tensors) continue; + b.pre = c.name + "."; + if (c.kind == "vad") silero_content(b, 7); + else asr_content(b, seed++, big_floats); + b.pre.clear(); + } + return b.write(path); +} + +unsigned long long bytes_read() { + std::FILE* f = std::fopen("/proc/self/io", "r"); + if (!f) return 0; + char k[64]; + unsigned long long v = 0, r = 0; + while (std::fscanf(f, "%63s %llu", k, &v) == 2) + if (std::string(k) == "rchar:") r = v; + std::fclose(f); + return r; +} + +std::string take(char* p) { + std::string s = p ? p : ""; + if (p) parakeet_capi_free_string(p); + return s; +} + +std::vector noise(size_t n, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_real_distribution d(-0.3f, 0.3f); + std::vector v(n); + for (auto& x : v) x = d(rng); + return v; +} + +} // namespace + +int main() { + const std::string dir = "test_bundle.tmp."; + const std::string plain_sil = dir + "silero.gguf", plain_asr = dir + "asr.gguf"; + const std::string B = dir + "b.gguf", B2 = dir + "b2.gguf", Bv = dir + "bv.gguf", Bbad = dir + "bad.gguf"; + + // Single-model files, written with the same content (no prefix). + { + Builder s; silero_content(s, 7); CHECK(s.write(plain_sil)); + Builder a; asr_content(a, 1, 1 << 20); CHECK(a.write(plain_asr)); + } + CHECK(write_bundle(B, {{"asr", "asr"}, {"vad", "vad"}})); + CHECK(write_bundle(B2, {{"a", "asr"}, {"b", "asr"}, {"vad", "vad"}})); + CHECK(write_bundle(Bv, {{"vad", "vad"}})); + + // --- header --- + CHECK(pk::gguf_is_bundle(B)); + CHECK(!pk::gguf_is_bundle(plain_sil)); + CHECK(!pk::gguf_is_bundle(plain_asr)); + CHECK(!pk::gguf_is_bundle("test_bundle.does-not-exist")); + pk::BundleInfo info; + std::string err; + CHECK(pk::read_bundle_info(B, info, &err)); + CHECK(info.version == 1 && info.name == "test-bundle" && info.components.size() == 2); + if (info.components.size() == 2) { + const pk::BundleComponent* a = info.find("asr"); + const pk::BundleComponent* v = info.find("vad"); + CHECK(a && v && !info.find("nope")); + if (a && v) { + CHECK(a->kind == "asr" && a->license == "CC-BY-4.0" && a->source == "example/asr"); + CHECK(v->kind == "vad" && v->license == "MIT" && v->attribution == "Credit vad"); + CHECK(a->n_tensors == 2 && v->n_tensors == 30); + CHECK(a->n_bytes > (1u << 22)); + } + } + CHECK(!pk::read_bundle_info(plain_sil, info, &err) && err.find("not a bundle") != std::string::npos); + + // --- selection --- + std::string pick; + CHECK(pk::read_bundle_info(B, info, &err) && pk::select_default_component(info, pick, &err) && pick == "asr"); + CHECK(pk::read_bundle_info(Bv, info, &err) && pk::select_default_component(info, pick, &err) && pick == "vad"); + CHECK(pk::read_bundle_info(B2, info, &err)); + CHECK(!pk::select_default_component(info, pick, &err)); + CHECK(err.find("several asr") != std::string::npos && err.find("a (asr)") != std::string::npos && + err.find("b (asr)") != std::string::npos); + + // --- ModelLoader: a bundle is refused without a component, plain files unchanged --- + { + pk::ModelLoader ml; + CHECK(!ml.load(B)); + pk::ModelLoader pl; + CHECK(pl.load(plain_asr) && pl.config().d_model == 16 && pl.config().vocab_size == 5); + } + // --- ModelLoader::load_component: same config and tensor bytes as the plain file --- + { + pk::ModelLoader plain, bun; + CHECK(plain.load(plain_asr)); + const unsigned long long r0 = bytes_read(); + CHECK(bun.load_component(B, "asr")); + const unsigned long long r1 = bytes_read(); + CHECK(bun.config().d_model == 16 && bun.config().n_layers == 2 && bun.config().vocab_size == 5); + CHECK(bun.config().arch == "tdt"); + ggml_tensor* x = plain.tensor("encoder.big"); + ggml_tensor* y = bun.tensor("encoder.big"); + CHECK(x && y && x->type == y->type && ggml_nbytes(x) == ggml_nbytes(y)); + CHECK(x && y && std::memcmp(x->data, y->data, ggml_nbytes(x)) == 0); + ggml_tensor* x2 = plain.tensor("encoder.small"); + ggml_tensor* y2 = bun.tensor("encoder.small"); + CHECK(x2 && y2 && std::memcmp(x2->data, y2->data, ggml_nbytes(x2)) == 0); + CHECK(!bun.tensor("vad16k.stft.forward_basis_buffer") && !bun.tensor("asr.encoder.big")); + // Partial read: the Silero component's bytes (about 1.2 MB) stay on disk, and + // the other ASR component (4 MB) too. Loading "asr" reads about its own 4 MB. + if (r1 > r0) { + CHECK(r1 - r0 < 4300000ull); + std::fprintf(stderr, "bytes read for asr of B: %llu\n", r1 - r0); + } + pk::ModelLoader none; + CHECK(!none.load_component(B, "nope")); + pk::ModelLoader onplain; + CHECK(!onplain.load_component(plain_asr, "asr")); + } + { // the second ASR component of a two-ASR bundle is its own file + pk::ModelLoader bb, pa; + CHECK(bb.load_component(B2, "b")); + CHECK(bb.config().d_model == 16); + ggml_tensor* t = bb.tensor("encoder.big"); + CHECK(t != nullptr); + Builder a2; asr_content(a2, 2, 1 << 20); // component b was built with seed 2 + const std::string p2 = dir + "asr2.gguf"; + CHECK(a2.write(p2)); + pk::ModelLoader pl2; + CHECK(pl2.load(p2)); + ggml_tensor* t2 = pl2.tensor("encoder.big"); + CHECK(t && t2 && std::memcmp(t->data, t2->data, ggml_nbytes(t)) == 0); + std::remove(p2.c_str()); + } + + // --- Silero from a component equals Silero from the plain file --- + { + std::string e1, e2; + auto plain = pk::SileroVad::load(plain_sil, &e1); + const unsigned long long r0 = bytes_read(); + auto bun = pk::SileroVad::load(B, &e2, "vad"); + const unsigned long long r1 = bytes_read(); + CHECK(plain && bun); + if (plain && bun) { + for (int sr : {16000, 8000}) { + const std::vector pcm = noise((size_t)sr * 2 + 123, 99); + const std::vector a = plain->probabilities(pcm.data(), pcm.size(), sr); + const std::vector b = bun->probabilities(pcm.data(), pcm.size(), sr); + CHECK(!a.empty() && a.size() == b.size()); + CHECK(a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0); + } + } + if (r1 > r0) { + CHECK(r1 - r0 < 2500000ull); // the 4 MB ASR component is not read + std::fprintf(stderr, "bytes read for vad of B: %llu\n", r1 - r0); + } + std::string e; + CHECK(!pk::SileroVad::load(B, &e) && e.find("bundle") != std::string::npos); + CHECK(!pk::SileroVad::load(B, &e, "asr") && e.find("kind") != std::string::npos); + CHECK(!pk::SileroVad::load(B, &e, "nope") && e.find("components:") != std::string::npos); + CHECK(!pk::SileroVad::load(plain_sil, &e, "vad") && e.find("not a bundle") != std::string::npos); + } + + // --- an old reader cannot mistake a bundle for a model --- + { + gguf_init_params ip{true, nullptr}; + gguf_context* g = gguf_init_from_file(B.c_str(), ip); + CHECK(g != nullptr); + if (g) { + for (const char* k : {"parakeet.encoder.d_model", "parakeet.vocab_size", "parakeet.arch", + "silero_vad.lstm.hidden", "silero_vad.sample_rates"}) + CHECK(gguf_find_key(g, k) < 0); + const int64_t a = gguf_find_key(g, "general.architecture"); + CHECK(a >= 0 && std::string(gguf_get_val_str(g, a)) == "parakeet-bundle"); + gguf_free(g); + } + } + + // --- malformed bundles --- + { + CHECK(write_bundle(Bbad, {{"asr", "asr"}}, /*version*/ 2)); + CHECK(!pk::read_bundle_info(Bbad, info, &err) && err.find("version 2") != std::string::npos); + pk::ModelLoader ml; + CHECK(!ml.load_component(Bbad, "asr")); + CHECK(write_bundle(Bbad, {{"asr", "asr"}}, 1, /*components key*/ false)); + CHECK(!pk::read_bundle_info(Bbad, info, &err) && err.find("components") != std::string::npos); + CHECK(write_bundle(Bbad, {{"asr", "asr"}, {"asr", "asr", true}})); + CHECK(!pk::read_bundle_info(Bbad, info, &err) && err.find("twice") != std::string::npos); + Comp notensors{"asr", "asr", true}; + CHECK(write_bundle(Bbad, {notensors, {"vad", "vad"}})); + CHECK(!pk::read_bundle_info(Bbad, info, &err) && err.find("no tensors") != std::string::npos); + CHECK(write_bundle(Bbad, {{"Bad.Name", "asr"}})); + CHECK(!pk::read_bundle_info(Bbad, info, &err) && err.find("not valid") != std::string::npos); + CHECK(write_bundle(Bbad, {{"parakeet", "asr"}})); // reserved prefix + CHECK(!pk::read_bundle_info(Bbad, info, &err)); + // a file cut short: the component whose data is missing is refused, the + // earlier component still loads + CHECK(write_bundle(Bbad, {{"vad", "vad"}, {"asr", "asr"}})); + std::string all; + { + std::ifstream in(Bbad, std::ios::binary); + all.assign((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + } + { + std::ofstream out(Bbad, std::ios::binary); + out.write(all.data(), (std::streamsize)(all.size() - 100)); + } + pk::ModelLoader t1, t2; + std::string e; + CHECK(!t1.load_component(Bbad, "asr")); + CHECK(pk::SileroVad::load(Bbad, &e, "vad") != nullptr); + CHECK(write_bundle(Bbad, {{"asr", "asr"}, {"vad", "vad"}})); + { + std::ifstream in(Bbad, std::ios::binary); + all.assign((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::ofstream out(Bbad, std::ios::binary); + out.write(all.data(), (std::streamsize)(all.size() - 100)); + } + CHECK(!pk::SileroVad::load(Bbad, &e, "vad") && e.find("truncated") != std::string::npos); + { // a file that is not a GGUF at all + std::ofstream out(Bbad, std::ios::binary); + out << "not a gguf"; + } + CHECK(!pk::gguf_is_bundle(Bbad)); + CHECK(!pk::read_bundle_info(Bbad, info, &err)); + } + + // --- C-API --- + { + // vad-only bundle: parakeet_capi_load picks the Silero component. + parakeet_ctx* c = parakeet_capi_load(Bv.c_str()); + CHECK(c && parakeet_capi_model_kind(c) == PARAKEET_MODEL_KIND_VAD); + parakeet_ctx* p = parakeet_capi_load(plain_sil.c_str()); + CHECK(p && parakeet_capi_model_kind(p) == PARAKEET_MODEL_KIND_VAD); + parakeet_ctx* d = parakeet_capi_load_component(B.c_str(), "vad"); + CHECK(d && parakeet_capi_model_kind(d) == PARAKEET_MODEL_KIND_VAD); + if (c && p && d) { + const std::vector pcm = noise(16000 * 3, 5); + const char* opt = "{\"probabilities\":true}"; + const std::string jp = take(parakeet_capi_vad_pcm_json(p, pcm.data(), (int)pcm.size(), 16000, opt)); + const std::string jc = take(parakeet_capi_vad_pcm_json(c, pcm.data(), (int)pcm.size(), 16000, opt)); + const std::string jd = take(parakeet_capi_vad_pcm_json(d, pcm.data(), (int)pcm.size(), 16000, opt)); + CHECK(!jp.empty() && jp.find("\"probabilities\"") != std::string::npos); + CHECK(jp == jc && jp == jd); + } + parakeet_capi_free(c); parakeet_capi_free(p); parakeet_capi_free(d); + + // a bundle with an ASR component and a Silero one: load selects the ASR + // component (the stand-in has the keys the loader needs, so it opens). + parakeet_ctx* a = parakeet_capi_load(B.c_str()); + CHECK(a && parakeet_capi_model_kind(a) == PARAKEET_MODEL_KIND_ASR); + parakeet_capi_free(a); + // ambiguous bundle + CHECK(parakeet_capi_load(B2.c_str()) == nullptr); + const std::string amb = parakeet_capi_load_error(); + CHECK(amb.find("several asr") != std::string::npos && amb.find("parakeet_capi_load_component") != std::string::npos); + // component errors + CHECK(parakeet_capi_load_component(B.c_str(), "nope") == nullptr); + CHECK(std::string(parakeet_capi_load_error()).find("no component \"nope\"") != std::string::npos); + CHECK(parakeet_capi_load_component(plain_sil.c_str(), "vad") == nullptr); + CHECK(std::string(parakeet_capi_load_error()).find("not a bundle") != std::string::npos); + CHECK(parakeet_capi_load_component(nullptr, "vad") == nullptr); + // NULL or empty component is a plain load + parakeet_ctx* e1 = parakeet_capi_load_component(plain_sil.c_str(), nullptr); + parakeet_ctx* e2 = parakeet_capi_load_component(Bv.c_str(), ""); + CHECK(e1 && e2); + parakeet_capi_free(e1); parakeet_capi_free(e2); + // load error is cleared by a good call and set by a failure + CHECK(parakeet_capi_load("test_bundle.does-not-exist") == nullptr); + CHECK(std::string(parakeet_capi_load_error()).size() > 0); + + // components JSON + const std::string js = take(parakeet_capi_bundle_components_json(B.c_str())); + CHECK(js.find("\"name\":\"asr\"") != std::string::npos && js.find("\"kind\":\"vad\"") != std::string::npos); + CHECK(js.find("\"license\":\"MIT\"") != std::string::npos && js.find("\"license\":\"CC-BY-4.0\"") != std::string::npos); + CHECK(js.front() == '[' && js.back() == ']'); + CHECK(parakeet_capi_bundle_components_json(plain_sil.c_str()) == nullptr); + CHECK(std::string(parakeet_capi_load_error()).find("not a bundle") != std::string::npos); + CHECK(parakeet_capi_bundle_components_json(nullptr) == nullptr); + } + + for (const std::string& f : {plain_sil, plain_asr, B, B2, Bv, Bbad}) std::remove(f.c_str()); + if (failures) return 1; + std::printf("test_bundle OK\n"); + return 0; +} diff --git a/tests/test_bundle_models.cpp b/tests/test_bundle_models.cpp new file mode 100644 index 0000000..527cae0 --- /dev/null +++ b/tests/test_bundle_models.cpp @@ -0,0 +1,175 @@ +// A real bundle against its single-model files: the transcript, the VAD head +// probabilities and the Silero probabilities are identical, through the C++ API +// and through the C-API, and loading one component does not read the others. +// +// LABEL model; run from the project root (fixtures are relative). +// Env (skip 77 without the first three): +// PARAKEET_TEST_BUNDLE the bundle GGUF (an "asr" and a "vad" component) +// PARAKEET_TEST_BUNDLE_ASR the single-model ASR GGUF the "asr" component was built from +// PARAKEET_TEST_BUNDLE_SILERO the Silero GGUF the "vad" component was built from +// PARAKEET_TEST_BUNDLE_EXPECT optional: the exact transcript of tests/fixtures/speech.wav +#include +#include +#include +#include +#include + +#include + +#include "audio_io.hpp" +#include "bundle.hpp" +#include "model.hpp" +#include "parakeet_capi.h" +#include "silero_vad.hpp" +#include "vad_json.hpp" + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +static unsigned long long bytes_read() { + std::FILE* f = std::fopen("/proc/self/io", "r"); + if (!f) return 0; + char k[64]; + unsigned long long v = 0, r = 0; + while (std::fscanf(f, "%63s %llu", k, &v) == 2) + if (std::string(k) == "rchar:") r = v; + std::fclose(f); + return r; +} + +static std::string take(char* p) { + std::string s = p ? p : ""; + if (p) parakeet_capi_free_string(p); + return s; +} + +static bool same_bits(const std::vector& a, const std::vector& b) { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0; +} + +int main() { + const char* bundle = std::getenv("PARAKEET_TEST_BUNDLE"); + const char* asr_path = std::getenv("PARAKEET_TEST_BUNDLE_ASR"); + const char* sil_path = std::getenv("PARAKEET_TEST_BUNDLE_SILERO"); + if (!bundle || !asr_path || !sil_path) { + std::puts("skip: PARAKEET_TEST_BUNDLE, _ASR and _SILERO must be set"); + return 77; + } + const char* expect = std::getenv("PARAKEET_TEST_BUNDLE_EXPECT"); + const std::string wav = "tests/fixtures/speech.wav"; + pk::Audio audio; + CHECK(pk::load_audio_16k_mono(wav, audio)); + + pk::BundleInfo info; + std::string err; + CHECK(pk::read_bundle_info(bundle, info, &err)); + const pk::BundleComponent* ac = info.find("asr"); + const pk::BundleComponent* vc = info.find("vad"); + CHECK(ac && vc); + if (!ac || !vc) return 1; + CHECK(!ac->license.empty() && !vc->license.empty() && ac->license != vc->license); + CHECK(vc->license == "MIT"); + struct stat st{}; + CHECK(::stat(bundle, &st) == 0); + + // ---- ASR: transcript identical to the single-model file ---- + auto solo = pk::Model::load(asr_path); + CHECK(solo != nullptr); + CHECK(!pk::Model::load(bundle)); // a bundle without a component is refused + const unsigned long long r0 = bytes_read(); + auto bun = pk::Model::load(bundle, "asr"); + const unsigned long long r1 = bytes_read(); + CHECK(bun != nullptr); + if (!solo || !bun) return 1; + const std::string t0 = solo->transcribe_path(wav), t1 = bun->transcribe_path(wav); + std::printf("standalone: %s\nbundle : %s\n", t0.c_str(), t1.c_str()); + CHECK(!t0.empty() && t0 == t1); + if (expect) CHECK(t1 == expect); + // Only the asr component's tensors are read (plus the header). + if (r1 > r0) { + std::printf("bytes read for asr: %.1f MB (component %.1f MB, file %.1f MB)\n", (double)(r1 - r0) / 1e6, + (double)ac->n_bytes / 1e6, (double)st.st_size / 1e6); + CHECK(r1 - r0 < ac->n_bytes + (8u << 20)); + CHECK(r1 - r0 < (unsigned long long)st.st_size - vc->n_bytes / 2); + } + + // ---- VAD head (Ultra/Redux) from the bundle's ASR component ---- + if (solo->config().vad.present) { + CHECK(bun->config().vad.present && bun->config().vad.frame_sec == solo->config().vad.frame_sec); + const std::vector p0 = solo->vad_probabilities(audio.samples), p1 = bun->vad_probabilities(audio.samples); + CHECK(!p0.empty() && same_bits(p0, p1)); + pk::VadRequest req; + req.opts = pk::default_segmenter_opts(pk::VadKind::kHead); + req.probabilities = true; + CHECK(pk::vad_to_json(*solo, audio.samples, req) == pk::vad_to_json(*bun, audio.samples, req)); + } else { + std::puts("note: the ASR component has no VAD head"); + } + + // ---- Silero from the bundle ---- + std::string e0, e1; + auto s0 = pk::SileroVad::load(sil_path, &e0); + const unsigned long long q0 = bytes_read(); + auto s1 = pk::SileroVad::load(bundle, &e1, "vad"); + const unsigned long long q1 = bytes_read(); + CHECK(s0 && s1); + if (s0 && s1) { + const std::vector a = s0->probabilities(audio.samples.data(), audio.samples.size(), 16000); + const std::vector b = s1->probabilities(audio.samples.data(), audio.samples.size(), 16000); + CHECK(!a.empty() && same_bits(a, b)); + pk::VadRequest req; + req.kind = pk::VadKind::kSilero; + req.opts = pk::default_segmenter_opts(pk::VadKind::kSilero); + req.probabilities = true; + CHECK(pk::silero_vad_to_json(*s0, audio.samples, 16000, req) == pk::silero_vad_to_json(*s1, audio.samples, 16000, req)); + // ASR transcription cut by the bundle's Silero equals the one cut by the single file's Silero. + pk::SegmenterOpts opts = pk::default_segmenter_opts(pk::VadKind::kSilero); + auto fn = [](const pk::SileroVad* sv) { + return pk::Model::VadProbabilityFn([sv](const std::vector& pcm) { + return sv->probabilities(pcm.data(), pcm.size(), 16000); + }); + }; + pk::Model::VadProbabilityFn f0 = fn(s0.get()), f1 = fn(s1.get()); + const std::string v0 = solo->transcribe_pcm_vad(audio.samples, audio.sample_rate, pk::Decoder::kDefault, "", opts, &f0); + const std::string v1 = bun->transcribe_pcm_vad(audio.samples, audio.sample_rate, pk::Decoder::kDefault, "", opts, &f1); + CHECK(!v0.empty() && v0 == v1); + if (q1 > q0) { + std::printf("bytes read for vad: %.2f MB (component %.2f MB)\n", (double)(q1 - q0) / 1e6, (double)vc->n_bytes / 1e6); + CHECK(q1 - q0 < vc->n_bytes + (4u << 20)); // far below the ASR component + } + } + + // ---- C-API ---- + { + parakeet_ctx* a0 = parakeet_capi_load(asr_path); + parakeet_ctx* a1 = parakeet_capi_load(bundle); // auto-selects the ASR component + parakeet_ctx* a2 = parakeet_capi_load_component(bundle, "asr"); + parakeet_ctx* v0 = parakeet_capi_load(sil_path); + parakeet_ctx* v1 = parakeet_capi_load_component(bundle, "vad"); + CHECK(a0 && a1 && a2 && v0 && v1); + if (a0 && a1 && a2 && v0 && v1) { + CHECK(parakeet_capi_model_kind(a1) == PARAKEET_MODEL_KIND_ASR); + CHECK(parakeet_capi_model_kind(v1) == PARAKEET_MODEL_KIND_VAD); + const std::string c0 = take(parakeet_capi_transcribe_path(a0, wav.c_str(), 0)); + const std::string c1 = take(parakeet_capi_transcribe_path(a1, wav.c_str(), 0)); + const std::string c2 = take(parakeet_capi_transcribe_path(a2, wav.c_str(), 0)); + CHECK(!c0.empty() && c0 == c1 && c0 == c2 && c0 == t0); + const char* opt = "{\"probabilities\":true}"; + CHECK(take(parakeet_capi_vad_path_json(v0, wav.c_str(), opt)) == take(parakeet_capi_vad_path_json(v1, wav.c_str(), opt))); + if (solo->config().vad.present) + CHECK(take(parakeet_capi_vad_path_json(a0, wav.c_str(), opt)) == take(parakeet_capi_vad_path_json(a1, wav.c_str(), opt))); + // segmented transcription cut by the bundle's Silero + const std::string j0 = take(parakeet_capi_transcribe_path_json_vad_with(a0, v0, wav.c_str(), 0, nullptr)); + const std::string j1 = take(parakeet_capi_transcribe_path_json_vad_with(a1, v1, wav.c_str(), 0, nullptr)); + CHECK(!j0.empty() && j0 == j1); + } + parakeet_capi_free(a0); parakeet_capi_free(a1); parakeet_capi_free(a2); + parakeet_capi_free(v0); parakeet_capi_free(v1); + const std::string js = take(parakeet_capi_bundle_components_json(bundle)); + CHECK(js.find("\"name\":\"asr\"") != std::string::npos && js.find("\"name\":\"vad\"") != std::string::npos); + } + + if (failures) return 1; + std::printf("test_bundle_models OK\n"); + return 0; +} From 7762bd0ad949eee708a45cbb2d3933aa186a338b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 3 Oct 2026 22:55:23 +0000 Subject: [PATCH 3/9] feat: use bundle GGUF files in parakeet-cli info lists the components of a bundle with their kind, licence, source, credit, changes and size, read from the header only; --component NAME prints the details of one component. transcribe and vad take --component. A bundle opens its only ASR component by default, and a bundle with several candidates is refused with the list of names. For vad, and for transcribe --vad, the VAD source on a bundle is the Silero component if there is one, otherwise the VAD head of the ASR component. --component (vad) and --vad-component (transcribe) override this; --vad-model still takes precedence. Other subcommands do not take a bundle yet and refuse it with a message. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- examples/cli/main.cpp | 226 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 198 insertions(+), 28 deletions(-) diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 16e1945..c85d171 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -27,6 +27,7 @@ #include "vad_head.hpp" #include "vad_json.hpp" #include "silero_vad.hpp" +#include "bundle.hpp" #include "speaker_encoder.hpp" #include "speaker_registry.hpp" #include @@ -53,13 +54,118 @@ #include #endif -static int cmd_info(const char* path) { +// The ASR component that --model/--component name, "" for a plain single-model +// file. Set once by the commands that take --component; load_asr() uses it. +static std::string g_asr_component; + +// Resolves `component` (may be empty) against `model`. On a bundle with no +// component named, picks the default ASR component. Returns false with a message +// when the file is a plain GGUF but a component was named, or when there is no +// unique default. +static bool resolve_asr_component(const std::string& model, const std::string& component, std::string& out) { + out.clear(); + if (!pk::gguf_is_bundle(model)) { + if (!component.empty()) { + std::fprintf(stderr, "parakeet-cli: %s is not a bundle GGUF, so --component does not apply\n", model.c_str()); + return false; + } + return true; + } + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(model, info, &err)) { std::fprintf(stderr, "parakeet-cli: %s\n", err.c_str()); return false; } + if (!component.empty()) { + const pk::BundleComponent* c = info.find(component); + if (!c) { + std::fprintf(stderr, "parakeet-cli: bundle has no component '%s'; components: %s\n", component.c_str(), + pk::bundle_component_names(info).c_str()); + return false; + } + if (c->kind != pk::kBundleKindAsr) { + std::fprintf(stderr, "parakeet-cli: component '%s' has kind %s, not an ASR model\n", component.c_str(), c->kind.c_str()); + return false; + } + out = component; + return true; + } + if (!pk::select_default_component(info, out, &err)) { std::fprintf(stderr, "parakeet-cli: %s\n", err.c_str()); return false; } + return true; +} + +static std::unique_ptr load_asr(const std::string& model) { + return pk::Model::load(model, g_asr_component); +} + +static parakeet_ctx* load_asr_ctx(const std::string& model) { + parakeet_ctx* ctx = parakeet_capi_load_component(model.c_str(), g_asr_component.c_str()); + if (!ctx) std::fprintf(stderr, "parakeet-cli: %s\n", parakeet_capi_load_error()); + return ctx; +} + +static int cmd_info_bundle(const char* path, const pk::BundleInfo& info) { + std::printf("parakeet.cpp %s\n", parakeet_version()); + std::printf("bundle: %s\n", path); + std::printf(" name : %s\n", info.name.c_str()); + std::printf(" format version : %u\n", info.version); + for (const pk::BundleComponent& c : info.components) { + std::printf(" component %s\n", c.name.c_str()); + std::printf(" kind : %s\n", c.kind.c_str()); + std::printf(" licence : %s (%s)\n", c.license.c_str(), c.license_url.c_str()); + std::printf(" source : %s\n", c.source.c_str()); + std::printf(" attribution : %s\n", c.attribution.c_str()); + std::printf(" changes : %s\n", c.changes.c_str()); + std::printf(" tensors : %llu (%.1f MB)\n", (unsigned long long)c.n_tensors, (double)c.n_bytes / 1e6); + if (!c.content_sha256.empty()) std::printf(" content sha256: %s\n", c.content_sha256.c_str()); + } + std::string def, err; + if (pk::select_default_component(info, def, &err)) std::printf(" default for load: %s\n", def.c_str()); + else std::printf(" default for load: none (%s)\n", err.c_str()); + std::printf(" (use --component NAME for the details of one component)\n"); + return 0; +} + +static int cmd_info(int argc, char** argv) { + const char* path = argv[0]; + std::string component; + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--component") == 0 && i + 1 < argc) component = argv[++i]; + else { std::fprintf(stderr, "usage: parakeet-cli info [--component NAME]\n"); return 2; } + } + if (pk::gguf_is_bundle(path)) { + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(path, info, &err)) { std::fprintf(stderr, "parakeet-cli: %s\n", err.c_str()); return 1; } + if (component.empty()) return cmd_info_bundle(path, info); + const pk::BundleComponent* bc = info.find(component); + if (!bc) { + std::fprintf(stderr, "parakeet-cli: bundle has no component '%s'; components: %s\n", component.c_str(), + pk::bundle_component_names(info).c_str()); + return 1; + } + std::printf("component: %s (kind %s, licence %s)\n", bc->name.c_str(), bc->kind.c_str(), bc->license.c_str()); + if (bc->kind == pk::kBundleKindVad) { + std::string err2; + std::unique_ptr sv = pk::SileroVad::load(path, &err2, component); + if (!sv) { std::fprintf(stderr, "parakeet-cli: %s\n", err2.c_str()); return 1; } + std::printf(" silero VAD, sample rates:"); + for (int r : sv->sample_rates()) std::printf(" %d", r); + std::printf("\n"); + return 0; + } + } else if (!component.empty()) { + std::fprintf(stderr, "parakeet-cli: %s is not a bundle GGUF, so --component does not apply\n", path); + return 1; + } pk::ModelLoader ml; - if (!ml.load(path)) { std::fprintf(stderr, "failed to load %s\n", path); return 1; } + if (!(component.empty() ? ml.load(path) : ml.load_component(path, component))) { + std::fprintf(stderr, "failed to load %s\n", path); + return 1; + } const pk::ParakeetConfig& c = ml.config(); std::printf("parakeet.cpp %s\n", parakeet_version()); std::printf("model: %s\n", path); std::printf(" arch : %s\n", c.arch.c_str()); + if (c.vad.present) std::printf(" vad head : yes (%.3f s frames)\n", (double)c.vad.frame_sec); std::printf(" d_model/layers/heads: %u / %u / %u\n", c.d_model, c.n_layers, c.n_heads); std::printf(" conv_kernel/norm: %u / %s\n", c.conv_kernel, c.conv_norm_type.c_str()); std::printf(" xscaling : %s\n", c.xscaling ? "true" : "false"); @@ -222,23 +328,43 @@ struct VadOverrides { static int cmd_transcribe_vad(const std::string& model, const std::string& input, pk::Decoder dec, const std::string& lang, bool timestamps, bool json, - const VadOverrides& ov, const std::string& vad_model) { + const VadOverrides& ov, const std::string& vad_model, + const std::string& vad_component) { pk::Audio audio; if (!load_audio_arg_16k_mono(input, audio)) { std::fprintf(stderr, "parakeet-cli: failed to load audio %s\n", input.c_str()); return 1; } try { - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = load_asr(model); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; } - pk::SegmenterOpts opts = pk::default_segmenter_opts(vad_model.empty() ? pk::VadKind::kHead : pk::VadKind::kSilero); + // VAD source: --vad-model (a Silero GGUF, or a bundle with --vad-component), + // else the Silero component of the --model bundle, else the ASR head. + std::string silero_path = vad_model, silero_comp; + if (vad_model.empty() && pk::gguf_is_bundle(model)) { + pk::BundleInfo info; + std::string berr; + if (!pk::read_bundle_info(model, info, &berr)) { std::fprintf(stderr, "parakeet-cli: %s\n", berr.c_str()); return 1; } + if (!vad_component.empty()) { + const pk::BundleComponent* c = info.find(vad_component); + if (!c) { std::fprintf(stderr, "parakeet-cli: bundle has no component '%s'; components: %s\n", vad_component.c_str(), pk::bundle_component_names(info).c_str()); return 1; } + if (c->kind == pk::kBundleKindVad) { silero_path = model; silero_comp = vad_component; } + else if (vad_component != g_asr_component) { std::fprintf(stderr, "parakeet-cli: --vad-component %s is not a VAD component\n", vad_component.c_str()); return 1; } + } else { + for (const pk::BundleComponent& c : info.components) + if (c.kind == pk::kBundleKindVad) { silero_path = model; silero_comp = c.name; break; } + } + } else if (!vad_model.empty() && !vad_component.empty()) { + silero_comp = vad_component; + } + pk::SegmenterOpts opts = pk::default_segmenter_opts(silero_path.empty() ? pk::VadKind::kHead : pk::VadKind::kSilero); ov.apply(opts); std::unique_ptr silero; pk::Model::VadProbabilityFn fn; - if (!vad_model.empty()) { + if (!silero_path.empty()) { std::string err; - silero = pk::SileroVad::load(vad_model, &err); - if (!silero) { std::fprintf(stderr, "parakeet-cli: failed to load VAD model %s: %s\n", vad_model.c_str(), err.c_str()); return 1; } + silero = pk::SileroVad::load(silero_path, &err, silero_comp); + if (!silero) { std::fprintf(stderr, "parakeet-cli: failed to load VAD model %s: %s\n", silero_path.c_str(), err.c_str()); return 1; } const pk::SileroVad* sv = silero.get(); fn = [sv](const std::vector& pcm16k) { std::vector p = sv->probabilities(pcm16k.data(), pcm16k.size(), 16000); @@ -277,7 +403,7 @@ static int cmd_transcribe(int argc, char** argv) { bool timestamps = false; bool json = false; bool vad = false; - std::string vad_model; + std::string vad_model, vad_component, component; VadOverrides vad_ov; double d = 0.0; auto parse_pos = [](const char* str, double& out) { @@ -314,6 +440,11 @@ static int cmd_transcribe(int argc, char** argv) { score_norm = false; } else if (std::strcmp(argv[i], "--vad") == 0) { vad = true; + } else if (std::strcmp(argv[i], "--component") == 0 && i + 1 < argc) { + component = argv[++i]; + } else if (std::strcmp(argv[i], "--vad-component") == 0 && i + 1 < argc) { + vad_component = argv[++i]; + vad = true; } else if (std::strcmp(argv[i], "--vad-model") == 0 && i + 1 < argc) { vad_model = argv[++i]; vad = true; @@ -336,11 +467,14 @@ static int cmd_transcribe(int argc, char** argv) { "usage: parakeet-cli transcribe --model --input " "[--decoder ctc|tdt] [--lang ] [--stream] [--timestamps] " "[--threads N] [--json] " - "[--vad [--vad-model ] [--vad-threshold F=0.5] [--vad-min-pause SEC] " + "[--component NAME] " + "[--vad [--vad-model ] [--vad-component NAME] [--vad-threshold F=0.5] [--vad-min-pause SEC] " "[--vad-min-speech SEC] [--vad-max-seg SEC=30]] " "[--beam-size N [--nbest N] [--no-score-norm]]\n"); return 2; } + // A bundle GGUF: pick the ASR component (the only "asr" one unless --component). + if (!resolve_asr_component(model, component, g_asr_component)) return 2; // Apply the thread override (offline + streaming graph compute). When unset // the persistent-backend default (kDefaultThreads) is used. if (threads > 0) pk::set_num_threads(threads); @@ -389,7 +523,7 @@ static int cmd_transcribe(int argc, char** argv) { std::fprintf(stderr, "parakeet-cli: --vad works with greedy decoding only\n"); return 2; } - return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model); + return cmd_transcribe_vad(model, input, dec, lang, timestamps, json, vad_ov, vad_model, vad_component); } if (nbest != 0 && beam_size == 0) { std::fprintf(stderr, @@ -431,7 +565,7 @@ static int cmd_transcribe(int argc, char** argv) { return 1; } try { - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = load_asr(model); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); @@ -452,7 +586,7 @@ static int cmd_transcribe(int argc, char** argv) { return 0; } - parakeet_ctx* ctx = parakeet_capi_load(model.c_str()); + parakeet_ctx* ctx = load_asr_ctx(model); if (!ctx) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); @@ -483,7 +617,7 @@ static int cmd_transcribe(int argc, char** argv) { return 1; } try { - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = load_asr(model); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; @@ -499,7 +633,7 @@ static int cmd_transcribe(int argc, char** argv) { return 0; } - parakeet_ctx* ctx = parakeet_capi_load(model.c_str()); + parakeet_ctx* ctx = load_asr_ctx(model); if (!ctx) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; @@ -525,7 +659,7 @@ static int cmd_transcribe(int argc, char** argv) { return 1; } try { - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = load_asr(model); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; @@ -550,7 +684,7 @@ static int cmd_transcribe(int argc, char** argv) { // surfaces as a clean error). With no --lang keep the existing free-function // path so behavior for every other model is byte-for-byte unchanged. if (!lang.empty() && !is_stdin_input(input)) { - parakeet_ctx* ctx = parakeet_capi_load(model.c_str()); + parakeet_ctx* ctx = load_asr_ctx(model); if (!ctx) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; @@ -576,14 +710,17 @@ static int cmd_transcribe(int argc, char** argv) { std::fprintf(stderr, "parakeet-cli: failed to load audio stdin\n"); return 1; } - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = load_asr(model); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; } text = m->transcribe_pcm(audio.samples, audio.sample_rate, dec, lang); } else { - text = pk::transcribe(model, input, dec); + // Same as pk::transcribe(), with the bundle component applied. + std::unique_ptr m = load_asr(model); + if (!m) throw std::runtime_error("parakeet: failed to load model: " + model); + text = m->transcribe_path(input, dec); } } catch (const std::exception& e) { std::fprintf(stderr, "transcribe failed: %s\n", e.what()); @@ -1904,7 +2041,7 @@ static int cmd_vad_probe(int argc, char** argv) { // with a VAD head (Ultra, Redux) or a Silero VAD GGUF. Unset options keep the // defaults of that model kind. static int cmd_vad(int argc, char** argv) { - std::string model, input; + std::string model, input, component; VadOverrides ov; std::optional mode; bool want_probs = false; @@ -1922,6 +2059,7 @@ static int cmd_vad(int argc, char** argv) { double d = 0.0; if (std::strcmp(argv[i], "--model") == 0 && i + 1 < argc) model = argv[++i]; else if (std::strcmp(argv[i], "--input") == 0 && i + 1 < argc) input = argv[++i]; + else if (std::strcmp(argv[i], "--component") == 0 && i + 1 < argc) component = argv[++i]; else if (std::strcmp(argv[i], "--threshold") == 0 && i + 1 < argc) { if (!num(argv[++i], d, false) || d > 1.0) return bad("--threshold must be in (0,1]"); ov.threshold = d; @@ -1948,8 +2086,8 @@ static int cmd_vad(int argc, char** argv) { } if (model.empty() || input.empty()) { std::fprintf(stderr, - "usage: parakeet-cli vad --model --input " - "[--threshold F=0.5] [--min-pause SEC] [--min-speech SEC] [--speech-pad SEC] " + "usage: parakeet-cli vad --model --input " + "[--component NAME] [--threshold F=0.5] [--min-pause SEC] [--min-speech SEC] [--speech-pad SEC] " "[--max-segment SEC=30] [--mode speech|segments] [--probabilities] [--threads N]\n"); return 2; } @@ -1960,7 +2098,38 @@ static int cmd_vad(int argc, char** argv) { return 1; } try { - const bool is_silero = pk::gguf_is_silero(model); + // A bundle: --component names the VAD source (a Silero component, or the + // ASR component whose head is used). Without it, the Silero component + // if there is one, else the default ASR component. + std::string comp; + bool is_silero = pk::gguf_is_silero(model); + if (pk::gguf_is_bundle(model)) { + pk::BundleInfo info; + std::string berr; + if (!pk::read_bundle_info(model, info, &berr)) { std::fprintf(stderr, "parakeet-cli: %s\n", berr.c_str()); return 1; } + const pk::BundleComponent* pick = component.empty() ? nullptr : info.find(component); + if (!component.empty() && !pick) { + std::fprintf(stderr, "parakeet-cli: bundle has no component '%s'; components: %s\n", component.c_str(), pk::bundle_component_names(info).c_str()); + return 1; + } + if (component.empty()) + for (const pk::BundleComponent& c : info.components) + if (c.kind == pk::kBundleKindVad) { pick = &c; break; } + if (!pick) { + if (!pk::select_default_component(info, comp, &berr)) { std::fprintf(stderr, "parakeet-cli: %s\n", berr.c_str()); return 1; } + } else { + comp = pick->name; + } + const pk::BundleComponent* chosen = info.find(comp); + is_silero = chosen && chosen->kind == pk::kBundleKindVad; + if (chosen && chosen->kind != pk::kBundleKindVad && chosen->kind != pk::kBundleKindAsr) { + std::fprintf(stderr, "parakeet-cli: component '%s' (kind %s) is not a VAD source\n", comp.c_str(), chosen->kind.c_str()); + return 1; + } + } else if (!component.empty()) { + std::fprintf(stderr, "parakeet-cli: %s is not a bundle GGUF, so --component does not apply\n", model.c_str()); + return 2; + } pk::VadRequest req; req.kind = is_silero ? pk::VadKind::kSilero : pk::VadKind::kHead; req.opts = pk::default_segmenter_opts(req.kind); @@ -1969,12 +2138,12 @@ static int cmd_vad(int argc, char** argv) { req.probabilities = want_probs; if (is_silero) { std::string err; - std::unique_ptr sv = pk::SileroVad::load(model, &err); + std::unique_ptr sv = pk::SileroVad::load(model, &err, comp); if (!sv) { std::fprintf(stderr, "parakeet-cli: failed to load model %s: %s\n", model.c_str(), err.c_str()); return 1; } std::printf("%s\n", pk::silero_vad_to_json(*sv, audio.samples, 16000, req).c_str()); return 0; } - std::unique_ptr m = pk::Model::load(model); + std::unique_ptr m = pk::Model::load(model, comp); if (!m) { std::fprintf(stderr, "parakeet-cli: failed to load model %s\n", model.c_str()); return 1; } std::printf("%s\n", pk::vad_to_json(*m, audio.samples, req).c_str()); } catch (const std::exception& e) { @@ -1991,7 +2160,7 @@ int main(int argc, char** argv) { return 0; } if (argc >= 3 && std::strcmp(argv[1], "info") == 0) - return run_and_shutdown([](int, char** a) { return cmd_info(a[0]); }, 1, argv + 2); + return run_and_shutdown(cmd_info, argc - 2, argv + 2); if (argc >= 2 && std::strcmp(argv[1], "transcribe") == 0) return run_and_shutdown(cmd_transcribe, argc - 2, argv + 2); if (argc >= 2 && std::strcmp(argv[1], "quantize") == 0) @@ -2016,11 +2185,12 @@ int main(int argc, char** argv) { "[--threshold F=0.5] [--min-pause SEC] [--min-speech SEC] [--speech-pad SEC] " "[--max-segment SEC=30] [--mode speech|segments] [--probabilities] [--threads N]\n" " parakeet-cli vad-probe --model --input [--variant N]\n" - " parakeet-cli info \n" + " parakeet-cli info [--component NAME]\n" " parakeet-cli transcribe --model --input " "[--decoder ctc|tdt] [--lang ] [--stream] [--timestamps] " "[--threads N] [--json] " - "[--vad [--vad-model ] [--vad-threshold F=0.5] [--vad-min-pause SEC] " + "[--component NAME] " + "[--vad [--vad-model ] [--vad-component NAME] [--vad-threshold F=0.5] [--vad-min-pause SEC] " "[--vad-min-speech SEC] [--vad-max-seg SEC=30]] " "[--beam-size N [--nbest N] [--no-score-norm]]\n" " parakeet-cli quantize " From 118714073da0244002558d1234d2b952fbacdd0f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 07:14:07 +0000 Subject: [PATCH 4/9] feat: load a diarization component from a bundle A bundle component of kind "diar" holds a Nemotron diarization model. DiarizationModel::load takes an optional component and reads it through the same prefixed ModelLoader as an ASR component, so only that component's tensors are read. parakeet_capi_load_component gives a diarization context for it, which every diarization, speaker-attributed ASR, scene and named-diarization call accepts as for a single-model file. The kinds ced and voice are known to the header reader and to the default-component choice; their loaders follow. The diarize tool takes a bundle and uses its only diar component, or the one named with --component. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- examples/cli/diarize.cpp | 33 +++++++++++++++++++++++++++++---- src/bundle.cpp | 7 ++++++- src/bundle.hpp | 8 +++++++- src/diarization.cpp | 4 ++-- src/diarization.hpp | 5 ++++- src/parakeet_capi.cpp | 6 ++++++ 6 files changed, 54 insertions(+), 9 deletions(-) diff --git a/examples/cli/diarize.cpp b/examples/cli/diarize.cpp index a82301f..2b9a327 100644 --- a/examples/cli/diarize.cpp +++ b/examples/cli/diarize.cpp @@ -1,15 +1,18 @@ // Standalone diarize tool: loads a diarization GGUF and diarizes a WAV. -// Usage: diarize [--stream [model|low|very_low|ultra_low]] +// Usage: diarize [--stream [model|low|very_low|ultra_low]] [--component NAME] +// may be a bundle GGUF: its only "diar" component is used, or the one named by --component. // Prints {"speakers":N,"segments":[{"speaker","start","end"}, ...]}. // --stream feeds the audio through the streaming C-API in 100 ms pieces (NeMo // cache-aware streaming) instead of the whole-file path, optionally in one of // the model card's latency modes (default: the checkpoint's configuration). #include "parakeet_capi.h" #include "audio_io.hpp" +#include "bundle.hpp" #include #include #include +#include #include static int diarize_stream(parakeet_ctx* ctx, const char* wav, int latency) { @@ -56,9 +59,12 @@ int main(int argc, char** argv) { std::fprintf(stderr, "usage: %s [--stream [model|low|very_low|ultra_low]]\n", argv[0]); return 1; } + std::string component; + for (int i = 3; i + 1 < argc; ++i) + if (std::strcmp(argv[i], "--component") == 0) { component = argv[i + 1]; break; } const bool stream = argc > 3 && std::strcmp(argv[3], "--stream") == 0; int latency = PARAKEET_DIAR_LATENCY_MODEL; - if (stream && argc > 4) { + if (stream && argc > 4 && std::strcmp(argv[4], "--component") != 0) { const char* names[] = {"model", "low", "very_low", "ultra_low"}; latency = -1; for (int i = 0; i < 4; ++i) @@ -68,9 +74,28 @@ int main(int argc, char** argv) { return 1; } } - parakeet_ctx* ctx = parakeet_capi_load(argv[1]); + if (component.empty() && pk::gguf_is_bundle(argv[1])) { + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(argv[1], info, &err)) { std::fprintf(stderr, "%s\n", err.c_str()); return 1; } + int n = 0; + for (const pk::BundleComponent& c : info.components) + if (c.kind == pk::kBundleKindDiar) { component = c.name; ++n; } + if (n != 1) { + std::fprintf(stderr, "bundle has %s diarization component; name one with --component (components: %s)\n", + n == 0 ? "no" : "several", pk::bundle_component_names(info).c_str()); + return 1; + } + } + parakeet_ctx* ctx = component.empty() ? parakeet_capi_load(argv[1]) + : parakeet_capi_load_component(argv[1], component.c_str()); if (!ctx) { - std::fprintf(stderr, "failed to load %s\n", argv[1]); + std::fprintf(stderr, "failed to load %s: %s\n", argv[1], parakeet_capi_load_error()); + return 1; + } + if (parakeet_capi_model_kind(ctx) != PARAKEET_MODEL_KIND_DIARIZATION) { + std::fprintf(stderr, "%s is not a diarization model\n", argv[1]); + parakeet_capi_free(ctx); return 1; } int rc = 0; diff --git a/src/bundle.cpp b/src/bundle.cpp index cb51f25..1cdadb2 100644 --- a/src/bundle.cpp +++ b/src/bundle.cpp @@ -65,6 +65,11 @@ const BundleComponent* BundleInfo::find(const std::string& n) const { return nullptr; } +bool bundle_kind_known(const std::string& k) { + return k == kBundleKindAsr || k == kBundleKindVad || k == kBundleKindDiar || k == kBundleKindCed || + k == kBundleKindVoice; +} + bool gguf_is_bundle(const std::string& path) { gguf_init_params ip{/*no_alloc=*/true, /*ctx=*/nullptr}; Gguf f; @@ -180,7 +185,7 @@ bool select_default_component(const BundleInfo& info, std::string& name, std::st std::vector asr, known; for (const auto& c : info.components) { if (c.kind == kBundleKindAsr) asr.push_back(&c); - if (c.kind == kBundleKindAsr || c.kind == kBundleKindVad) known.push_back(&c); + if (bundle_kind_known(c.kind)) known.push_back(&c); } const std::vector& pick = !asr.empty() ? asr : known; if (pick.size() == 1) { diff --git a/src/bundle.hpp b/src/bundle.hpp index 5169836..1c0a69b 100644 --- a/src/bundle.hpp +++ b/src/bundle.hpp @@ -14,10 +14,13 @@ namespace pk { constexpr const char* kBundleArch = "parakeet-bundle"; constexpr uint32_t kBundleVersion = 1; // highest format version this reader knows -// Component kinds understood in phase 1. A bundle may hold other kinds (for +// Component kinds this reader understands. A bundle may hold other kinds (for // example from a newer writer); a reader skips them. constexpr const char* kBundleKindAsr = "asr"; constexpr const char* kBundleKindVad = "vad"; // Silero VAD, or a VAD-only slice of an ASR model (scripts/slice_vad_gguf.py) +constexpr const char* kBundleKindDiar = "diar"; // Nemotron diarization +constexpr const char* kBundleKindCed = "ced"; // ced.cpp sound-event tagger +constexpr const char* kBundleKindVoice = "voice"; // voice-detect.cpp speaker encoder struct BundleComponent { std::string name; @@ -40,6 +43,9 @@ struct BundleInfo { const BundleComponent* find(const std::string& name) const; }; +// True for a kind this reader can describe (asr, vad, diar, ced, voice). +bool bundle_kind_known(const std::string& kind); + // True when the file is a GGUF whose general.architecture is "parakeet-bundle". // Reads the header only. bool gguf_is_bundle(const std::string& path); diff --git a/src/diarization.cpp b/src/diarization.cpp index 3358522..27d1c29 100644 --- a/src/diarization.cpp +++ b/src/diarization.cpp @@ -13,11 +13,11 @@ namespace pk { -std::unique_ptr DiarizationModel::load(const std::string& path) { +std::unique_ptr DiarizationModel::load(const std::string& path, const std::string& component) { // unique_ptr via private ctor: construct then load. std::unique_ptr m(new (std::nothrow) DiarizationModel()); if (!m) return nullptr; - if (!m->loader_.load(path)) return nullptr; + if (!(component.empty() ? m->loader_.load(path) : m->loader_.load_component(path, component))) return nullptr; const auto& cfg = m->loader_.config(); if (cfg.arch != "diarization") { diff --git a/src/diarization.hpp b/src/diarization.hpp index 4d39b75..2e0b5ec 100644 --- a/src/diarization.hpp +++ b/src/diarization.hpp @@ -35,7 +35,10 @@ struct DiarizationResult { class DiarizationModel { public: // Load a diarization GGUF. Returns nullptr on failure. - static std::unique_ptr load(const std::string& path); + // With a non-empty `component`, `path` is a bundle GGUF (docs/bundle.md) and + // the model is that component of kind "diar", read natively through the + // same prefixed loader as an ASR component; only its tensors are read. + static std::unique_ptr load(const std::string& path, const std::string& component = ""); // Diarize an audio file (any format audio_io supports). Returns segments. DiarizationResult diarize_path(const std::string& wav_path); diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index c32ed37..290d620 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -278,6 +278,12 @@ parakeet_ctx* load_bundle_component(const char* path, const std::string& name) { g_load_error = "cannot load ASR component \"" + name + "\" (see the log for the reason)"; return nullptr; } + } else if (c->kind == pk::kBundleKindDiar) { + ctx->diar = pk::DiarizationModel::load(path, name); + if (!ctx->diar) { + g_load_error = "cannot load diarization component \"" + name + "\" (see the log for the reason)"; + return nullptr; + } } else if (c->kind == pk::kBundleKindVad && pk::bundle_vad_is_slice(path, name)) { // A VAD-only slice: same context as a standalone slice file. ctx->vad_model = pk::Model::load_vad_only(path, name); From fe9fe9c7fc6502b1724fa0cf47d3e8cd7e6b186d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 07:14:19 +0000 Subject: [PATCH 5/9] feat: load sound-event and speaker components from a bundle ced.cpp and voice-detect.cpp open a model by file path only, and their sources stay unchanged. A component of kind "ced" or "voice" is therefore written as a standalone single-model GGUF (keys and tensors without the component prefix, data streamed from the bundle) and the path of that copy is given to the loader. On Linux the copy is an anonymous in-memory file (memfd_create, opened as /proc/self/fd/N); elsewhere, or with PARAKEET_BUNDLE_NO_MEMFD set, it is an owner-only temporary file that is removed after the load. CedTagger::load and SpeakerEncoder::load take an optional component and an error string. parakeet_capi_load_component handles the kinds ced and voice; the speaker identity of a voice component is the sha256 of the single-model file it was built from, so profiles made with the standalone file match. parakeet-cli: scene takes a bundle for each of --model, --diar, --sound and --speakers (the only component of the right kind, or one named with --asr-component, --diar-component, --sound-component or --speakers-component); enroll takes --component; info describes ced and voice components. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- CMakeLists.txt | 1 + examples/cli/main.cpp | 104 +++++++++++++++-- include/parakeet_capi.h | 7 +- src/bundle_extract.cpp | 252 ++++++++++++++++++++++++++++++++++++++++ src/bundle_extract.hpp | 53 +++++++++ src/ced_tagger.cpp | 24 +++- src/ced_tagger.hpp | 10 +- src/parakeet_capi.cpp | 20 ++++ src/speaker_encoder.cpp | 25 +++- src/speaker_encoder.hpp | 10 +- 10 files changed, 480 insertions(+), 26 deletions(-) create mode 100644 src/bundle_extract.cpp create mode 100644 src/bundle_extract.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6507c33..27dc107 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,7 @@ set(PARAKEET_SRC src/vad_segmenter.cpp src/vad_json.cpp src/bundle.cpp + src/bundle_extract.cpp src/parakeet_capi.cpp src/common.cpp src/audio_io.cpp diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 64d5706..95ef737 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -92,6 +92,49 @@ static bool resolve_asr_component(const std::string& model, const std::string& c return true; } +// Component of `path` to open for a command that needs a model of `kind` (diar, ced, +// voice or asr). A plain GGUF gives an empty `out` (and refuses a component name). On a +// bundle, `component` names it (its kind must match); without a name the only component +// of that kind is used. `what` and `flag` are for messages (e.g. "diarization", "--diar-component"). +static bool resolve_kind_component(const std::string& path, const std::string& kind, const std::string& component, + const char* what, const char* flag, std::string& out) { + out.clear(); + if (!pk::gguf_is_bundle(path)) { + if (!component.empty()) { + std::fprintf(stderr, "parakeet-cli: %s is not a bundle GGUF, so %s does not apply\n", path.c_str(), flag); + return false; + } + return true; + } + pk::BundleInfo info; + std::string err; + if (!pk::read_bundle_info(path, info, &err)) { std::fprintf(stderr, "parakeet-cli: %s\n", err.c_str()); return false; } + if (!component.empty()) { + const pk::BundleComponent* c = info.find(component); + if (!c) { + std::fprintf(stderr, "parakeet-cli: bundle has no component '%s'; components: %s\n", component.c_str(), + pk::bundle_component_names(info).c_str()); + return false; + } + if (c->kind != kind) { + std::fprintf(stderr, "parakeet-cli: component '%s' has kind %s, not a %s model (kind %s)\n", component.c_str(), + c->kind.c_str(), what, kind.c_str()); + return false; + } + out = component; + return true; + } + int n = 0; + for (const pk::BundleComponent& c : info.components) + if (c.kind == kind) { out = c.name; ++n; } + if (n == 1) return true; + out.clear(); + std::fprintf(stderr, "parakeet-cli: bundle %s has %s component of kind %s; components: %s%s\n", path.c_str(), + n == 0 ? "no" : "several", kind.c_str(), pk::bundle_component_names(info).c_str(), + n == 0 ? "" : (std::string(" (pick one with ") + flag + ")").c_str()); + return false; +} + static std::unique_ptr load_asr(const std::string& model) { return pk::Model::load(model, g_asr_component); } @@ -143,6 +186,20 @@ static int cmd_info(int argc, char** argv) { return 1; } std::printf("component: %s (kind %s, licence %s)\n", bc->name.c_str(), bc->kind.c_str(), bc->license.c_str()); + if (bc->kind == pk::kBundleKindCed) { + std::string err2; + std::unique_ptr t = pk::CedTagger::load(path, component, &err2); + if (!t) { std::fprintf(stderr, "parakeet-cli: %s\n", err2.c_str()); return 1; } + std::printf(" sound-event tagger (ced.cpp), %d classes\n", t->n_classes()); + return 0; + } + if (bc->kind == pk::kBundleKindVoice) { + std::string err2; + std::unique_ptr e = pk::SpeakerEncoder::load(path, component, &err2); + if (!e) { std::fprintf(stderr, "parakeet-cli: %s\n", err2.c_str()); return 1; } + std::printf(" speaker encoder (voice-detect.cpp), %d-dim embedding\n", e->dim()); + return 0; + } if (bc->kind == pk::kBundleKindVad && !pk::bundle_vad_is_slice(path, component)) { std::string err2; std::unique_ptr sv = pk::SileroVad::load(path, &err2, component); @@ -1674,7 +1731,7 @@ static std::string registry_read_error(const std::string& path, int e) { } static const char* kEnrollUsage = - "usage: parakeet-cli enroll --model --name " + "usage: parakeet-cli enroll --model [--component NAME] --name " "--input [--input ...] --registry \n"; // parakeet-cli enroll --model --name --input [--input ...] @@ -1682,10 +1739,11 @@ static const char* kEnrollUsage = // Embeds each input as one clip of and adds it to the registry file // (created when missing). The file is written only after every clip embedded. static int cmd_enroll(int argc, char** argv) { - std::string model, name, registry_path; + std::string model, name, registry_path, component; std::vector inputs; for (int i = 0; i < argc; ++i) { if (std::strcmp(argv[i], "--model") == 0 && i + 1 < argc) model = argv[++i]; + else if (std::strcmp(argv[i], "--component") == 0 && i + 1 < argc) component = argv[++i]; else if (std::strcmp(argv[i], "--name") == 0 && i + 1 < argc) name = argv[++i]; else if (std::strcmp(argv[i], "--input") == 0 && i + 1 < argc) inputs.push_back(argv[++i]); else if (std::strcmp(argv[i], "--registry") == 0 && i + 1 < argc) registry_path = argv[++i]; @@ -1699,9 +1757,12 @@ static int cmd_enroll(int argc, char** argv) { std::fprintf(stderr, "parakeet-cli: built without speaker identification (PARAKEET_WITH_VOICEDETECT=OFF)\n"); return 2; } - auto enc = pk::SpeakerEncoder::load(model); + std::string comp, lerr; + if (!resolve_kind_component(model, pk::kBundleKindVoice, component, "speaker", "--component", comp)) return 2; + auto enc = pk::SpeakerEncoder::load(model, comp, &lerr); if (!enc) { - std::fprintf(stderr, "parakeet-cli enroll: failed to load speaker model %s\n", model.c_str()); + std::fprintf(stderr, "parakeet-cli enroll: failed to load speaker model %s%s%s\n", model.c_str(), + lerr.empty() ? "" : ": ", lerr.c_str()); return 1; } pk::SpeakerRegistry reg; @@ -1758,6 +1819,8 @@ static const char* kSceneUsage = "[--speaker-threshold F]] --input " "[--latency model|low|very_low|ultra_low] [--chunk-ms N] " "[--show-speech] [--json]\n" + " each model may be a bundle GGUF (the only component of the right kind is used); name another with\n" + " --asr-component, --diar-component, --sound-component or --speakers-component\n" " --speaker-threshold: default 0.5; ECAPA needs about 0.7, see docs/speaker.md\n"; // parakeet-cli scene [--model ] [--diar ] [--sound ] @@ -1773,6 +1836,7 @@ static const char* kSceneUsage = static int cmd_scene(int argc, char** argv) { std::string model, diar, sound, input, latency_str; std::string speakers, registry_path; + std::string asr_comp_arg, diar_comp_arg, sound_comp_arg, speakers_comp_arg; bool have_threshold = false; float speaker_threshold = 0.0f; bool json = false; @@ -1781,6 +1845,14 @@ static int cmd_scene(int argc, char** argv) { for (int i = 0; i < argc; ++i) { if (std::strcmp(argv[i], "--model") == 0 && i + 1 < argc) { model = argv[++i]; + } else if (std::strcmp(argv[i], "--asr-component") == 0 && i + 1 < argc) { + asr_comp_arg = argv[++i]; + } else if (std::strcmp(argv[i], "--diar-component") == 0 && i + 1 < argc) { + diar_comp_arg = argv[++i]; + } else if (std::strcmp(argv[i], "--sound-component") == 0 && i + 1 < argc) { + sound_comp_arg = argv[++i]; + } else if (std::strcmp(argv[i], "--speakers-component") == 0 && i + 1 < argc) { + speakers_comp_arg = argv[++i]; } else if (std::strcmp(argv[i], "--diar") == 0 && i + 1 < argc) { diar = argv[++i]; } else if (std::strcmp(argv[i], "--sound") == 0 && i + 1 < argc) { @@ -1875,9 +1947,15 @@ static int cmd_scene(int argc, char** argv) { } } + std::string asr_comp, diar_comp, sound_comp, speakers_comp; + if (!model.empty() && !resolve_kind_component(model, pk::kBundleKindAsr, asr_comp_arg, "ASR", "--asr-component", asr_comp)) return 2; + if (!diar.empty() && !resolve_kind_component(diar, pk::kBundleKindDiar, diar_comp_arg, "diarization", "--diar-component", diar_comp)) return 2; + if (!sound.empty() && !resolve_kind_component(sound, pk::kBundleKindCed, sound_comp_arg, "sound", "--sound-component", sound_comp)) return 2; + if (!speakers.empty() && !resolve_kind_component(speakers, pk::kBundleKindVoice, speakers_comp_arg, "speaker", "--speakers-component", speakers_comp)) return 2; + std::string lerr; std::unique_ptr asr_model; if (!model.empty()) { - asr_model = pk::Model::load(model); + asr_model = pk::Model::load(model, asr_comp); if (!asr_model) { std::fprintf(stderr, "parakeet-cli scene: failed to load model %s\n", model.c_str()); return 1; @@ -1885,7 +1963,7 @@ static int cmd_scene(int argc, char** argv) { } std::unique_ptr diar_model; if (!diar.empty()) { - diar_model = pk::DiarizationModel::load(diar); + diar_model = pk::DiarizationModel::load(diar, diar_comp); if (!diar_model) { std::fprintf(stderr, "parakeet-cli scene: failed to load diarization model %s\n", diar.c_str()); @@ -1894,9 +1972,10 @@ static int cmd_scene(int argc, char** argv) { } std::unique_ptr tagger; if (!sound.empty()) { - tagger = pk::CedTagger::load(sound); + tagger = pk::CedTagger::load(sound, sound_comp, &lerr); if (!tagger) { - std::fprintf(stderr, "parakeet-cli scene: failed to load sound model %s\n", sound.c_str()); + std::fprintf(stderr, "parakeet-cli scene: failed to load sound model %s%s%s\n", sound.c_str(), + lerr.empty() ? "" : ": ", lerr.c_str()); return 1; } } @@ -1904,10 +1983,10 @@ static int cmd_scene(int argc, char** argv) { std::unique_ptr speaker_enc; pk::SpeakerRegistry registry; if (!speakers.empty()) { - speaker_enc = pk::SpeakerEncoder::load(speakers); + speaker_enc = pk::SpeakerEncoder::load(speakers, speakers_comp, &lerr); if (!speaker_enc) { - std::fprintf(stderr, "parakeet-cli scene: failed to load speaker model %s\n", - speakers.c_str()); + std::fprintf(stderr, "parakeet-cli scene: failed to load speaker model %s%s%s\n", + speakers.c_str(), lerr.empty() ? "" : ": ", lerr.c_str()); return 1; } std::string blob; @@ -2211,7 +2290,8 @@ int main(int argc, char** argv) { "[--latency model|low|very_low|ultra_low] [--chunk-ms N] " "[--show-speech] [--json]\n" " --speaker-threshold: default 0.5; ECAPA needs about 0.7, see docs/speaker.md\n" - " parakeet-cli enroll --model --name " + " each model may be a bundle (--asr-component, --diar-component, --sound-component, --speakers-component)\n" + " parakeet-cli enroll --model [--component NAME] --name " "--input [--input ...] --registry \n"); return 2; } diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index 31f5e4b..53b9c81 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -87,8 +87,11 @@ int parakeet_capi_abi_version(void); parakeet_ctx* parakeet_capi_load(const char* gguf_path); // Additive (no ABI bump): open one named component of a bundle GGUF. The -// context is of the kind the component declares: "asr" gives an ASR context, -// "vad" a Silero VAD context (parakeet_capi_model_kind PARAKEET_MODEL_KIND_VAD). +// context is of the kind the component declares: "asr" gives an ASR context +// (PARAKEET_MODEL_KIND_ASR), "diar" a diarization context, "ced" a sound tagger, +// "voice" a speaker encoder, "vad" a Silero VAD context (PARAKEET_MODEL_KIND_VAD) +// or, for a VAD-only slice, a VAD-only context. The "ced" and "voice" components are +// handed to ced.cpp and voice-detect.cpp as a standalone copy (docs/bundle.md). // Returns NULL, with a reason in parakeet_capi_load_error, when `gguf_path` is // not a bundle, has no such component, or has a component kind this build // cannot load. A NULL or empty `component` behaves like parakeet_capi_load. diff --git a/src/bundle_extract.cpp b/src/bundle_extract.cpp new file mode 100644 index 0000000..d360449 --- /dev/null +++ b/src/bundle_extract.cpp @@ -0,0 +1,252 @@ +#include "bundle_extract.hpp" + +#include +#include +#include +#include +#include +#include + +#include "ggml.h" +#include "gguf.h" + +#if defined(__linux__) +#include +#include +#include +#include +#elif !defined(_WIN32) +#include +#include +#include +#endif + +namespace pk { + +namespace { + +#if defined(_WIN32) +typedef long long off_t_compat; +#define PK_FSEEK _fseeki64 +#define PK_FTELL _ftelli64 +#else +typedef off_t off_t_compat; +#define PK_FSEEK fseeko +#define PK_FTELL ftello +#endif + +void set_err(std::string* err, const std::string& m) { + if (err) *err = m; +} + +// Copies one key of `g` to `out` under `name`. Unknown value types are skipped. +void copy_kv(gguf_context* out, gguf_context* g, int64_t i, const std::string& name) { + const char* k = name.c_str(); + switch (gguf_get_kv_type(g, i)) { + case GGUF_TYPE_UINT8: gguf_set_val_u8(out, k, gguf_get_val_u8(g, i)); break; + case GGUF_TYPE_INT8: gguf_set_val_i8(out, k, gguf_get_val_i8(g, i)); break; + case GGUF_TYPE_UINT16: gguf_set_val_u16(out, k, gguf_get_val_u16(g, i)); break; + case GGUF_TYPE_INT16: gguf_set_val_i16(out, k, gguf_get_val_i16(g, i)); break; + case GGUF_TYPE_UINT32: gguf_set_val_u32(out, k, gguf_get_val_u32(g, i)); break; + case GGUF_TYPE_INT32: gguf_set_val_i32(out, k, gguf_get_val_i32(g, i)); break; + case GGUF_TYPE_FLOAT32: gguf_set_val_f32(out, k, gguf_get_val_f32(g, i)); break; + case GGUF_TYPE_UINT64: gguf_set_val_u64(out, k, gguf_get_val_u64(g, i)); break; + case GGUF_TYPE_INT64: gguf_set_val_i64(out, k, gguf_get_val_i64(g, i)); break; + case GGUF_TYPE_FLOAT64: gguf_set_val_f64(out, k, gguf_get_val_f64(g, i)); break; + case GGUF_TYPE_BOOL: gguf_set_val_bool(out, k, gguf_get_val_bool(g, i)); break; + case GGUF_TYPE_STRING: gguf_set_val_str(out, k, gguf_get_val_str(g, i)); break; + case GGUF_TYPE_ARRAY: { + const gguf_type at = gguf_get_arr_type(g, i); + const size_t n = gguf_get_arr_n(g, i); + if (at == GGUF_TYPE_STRING) { + std::vector v(n); + for (size_t j = 0; j < n; ++j) v[j] = gguf_get_arr_str(g, i, j); + gguf_set_arr_str(out, k, v.data(), n); + } else if (at != GGUF_TYPE_ARRAY) { + gguf_set_arr_data(out, k, at, gguf_get_arr_data(g, i), n); + } + break; + } + default: break; + } +} + +bool no_memfd_requested() { + const char* e = std::getenv("PARAKEET_BUNDLE_NO_MEMFD"); + return e && *e && std::strcmp(e, "0") != 0; +} + +// Writes the standalone GGUF of `comp` to `f` (a freshly opened, empty file). +// Returns the number of bytes written, or 0 with *err set. +uint64_t write_component(const std::string& bundle, const std::string& comp, FILE* f, std::string* err) { + ggml_context* meta = nullptr; + gguf_init_params ip{/*no_alloc*/ true, &meta}; + gguf_context* g = gguf_init_from_file(bundle.c_str(), ip); + if (!g) { + set_err(err, "cannot read " + bundle + " as a GGUF file"); + return 0; + } + const std::string pre = comp + "."; + gguf_context* out = gguf_init_empty(); + ggml_context* tctx = nullptr; + FILE* in = nullptr; + uint64_t written = 0; + bool ok = false; + do { + for (int64_t i = 0; i < gguf_get_n_kv(g); ++i) { + const char* k = gguf_get_key(g, i); + if (std::strncmp(k, pre.c_str(), pre.size()) != 0) continue; + copy_kv(out, g, i, std::string(k + pre.size())); + } + const int64_t nt = gguf_get_n_tensors(g); + std::vector ids; + for (int64_t i = 0; i < nt; ++i) + if (std::strncmp(gguf_get_tensor_name(g, i), pre.c_str(), pre.size()) == 0) ids.push_back(i); + if (ids.empty()) { + set_err(err, "bundle " + bundle + ": component '" + comp + "' has no tensors"); + break; + } + // Tensor descriptors only (no data): the writer needs type, shape and name. + ggml_init_params tp{ggml_tensor_overhead() * ids.size() + (1u << 16), nullptr, /*no_alloc*/ true}; + tctx = ggml_init(tp); + if (!tctx) { set_err(err, "out of memory"); break; } + bool bad = false; + for (int64_t i : ids) { + const char* nm = gguf_get_tensor_name(g, i); + ggml_tensor* s = ggml_get_tensor(meta, nm); + ggml_tensor* t = s ? ggml_new_tensor(tctx, s->type, GGML_MAX_DIMS, s->ne) : nullptr; + if (!t) { set_err(err, std::string("cannot describe tensor ") + nm); bad = true; break; } + ggml_set_name(t, nm + pre.size()); + gguf_add_tensor(out, t); + } + if (bad) break; + if (!gguf_write_to_file_ptr(out, f, /*only_meta*/ true)) { set_err(err, "cannot write the component file"); break; } + written = (uint64_t)gguf_get_data_offset(out); // header + tensor table, padded + in = std::fopen(bundle.c_str(), "rb"); + if (!in) { set_err(err, "cannot open " + bundle); break; } + PK_FSEEK(in, 0, SEEK_END); + const uint64_t file_size = (uint64_t)PK_FTELL(in); + const uint64_t base = (uint64_t)gguf_get_data_offset(g); + std::vector buf(1u << 20); + bool failed = false; + for (size_t n = 0; n < ids.size() && !failed; ++n) { + const int64_t i = ids[n]; + const uint64_t size = gguf_get_tensor_size(g, i); + const uint64_t off = base + gguf_get_tensor_offset(g, i); + if (off + size > file_size) { + set_err(err, std::string("bundle ") + bundle + " is truncated (tensor " + gguf_get_tensor_name(g, i) + ")"); + failed = true; + break; + } + // Pad up to this tensor's aligned offset in the output. + const uint64_t want = (uint64_t)gguf_get_data_offset(out) + gguf_get_tensor_offset(out, (int64_t)n); + while (written < want) { + const char zero[64] = {0}; + const size_t k = (size_t)std::min(sizeof zero, want - written); + if (std::fwrite(zero, 1, k, f) != k) { failed = true; break; } + written += k; + } + if (failed || PK_FSEEK(in, (off_t_compat)off, SEEK_SET) != 0) { failed = true; break; } + for (uint64_t done = 0; done < size && !failed;) { + const size_t k = (size_t)std::min(buf.size(), size - done); + if (std::fread(buf.data(), 1, k, in) != k || std::fwrite(buf.data(), 1, k, f) != k) failed = true; + done += k; + written += k; + } + } + if (failed) { + if (!err || err->empty()) set_err(err, "cannot copy the component data (disk or memory full?)"); + break; + } + // Pad the end to the alignment like the single-pass writer does. + const size_t al = gguf_get_alignment(out); + while (written % al) { + if (std::fputc(0, f) == EOF) { failed = true; break; } + ++written; + } + if (failed || std::fflush(f) != 0) { set_err(err, "cannot write the component file"); break; } + ok = true; + } while (false); + if (in) std::fclose(in); + if (tctx) ggml_free(tctx); + gguf_free(out); + gguf_free(g); + if (meta) ggml_free(meta); + return ok ? written : 0; +} + +} // namespace + +ComponentFile::~ComponentFile() { +#if !defined(_WIN32) + if (fd_ >= 0) ::close(fd_); +#endif + if (remove_) { + std::error_code ec; + std::filesystem::remove(path_, ec); + } +} + +std::unique_ptr ComponentFile::create(const std::string& bundle, const std::string& comp, + std::string* err) { + std::unique_ptr cf(new ComponentFile()); +#if defined(__linux__) + if (!no_memfd_requested()) { + const int fd = (int)memfd_create("parakeet-component", MFD_CLOEXEC); + if (fd >= 0) { + const std::string p = "/proc/self/fd/" + std::to_string(fd); + FILE* f = ::fdopen(::dup(fd), "wb"); + std::string e; + const uint64_t n = f ? write_component(bundle, comp, f, &e) : 0; + if (f) std::fclose(f); + struct stat st {}; + if (n > 0 && ::stat(p.c_str(), &st) == 0) { // /proc must be mounted for the path to work + cf->fd_ = fd; + cf->path_ = p; + cf->in_memory_ = true; + cf->size_ = n; + return cf; + } + ::close(fd); + // A real failure (missing component, truncated bundle) is final; only fall back + // when the in-memory file itself could not be used. + if (n == 0 && !e.empty() && e.find("cannot write") == std::string::npos && + e.find("cannot copy") == std::string::npos) { + set_err(err, e); + return nullptr; + } + } + } +#endif + // Temporary file. + std::error_code ec; + const std::filesystem::path dir = std::filesystem::temp_directory_path(ec); + if (ec) { set_err(err, "no temporary directory for the component file"); return nullptr; } + FILE* f = nullptr; + std::string path; +#if defined(_WIN32) + for (int attempt = 0; attempt < 100 && !f; ++attempt) { + path = (dir / ("parakeet-component-" + std::to_string(std::rand()) + "-" + std::to_string(attempt) + ".gguf")).string(); + if (std::filesystem::exists(path)) continue; + f = std::fopen(path.c_str(), "wb"); + } +#else + for (int attempt = 0; attempt < 100 && !f; ++attempt) { + path = (dir / ("parakeet-component-" + std::to_string((long)::getpid()) + "-" + std::to_string(std::rand()) + + "-" + std::to_string(attempt) + ".gguf")).string(); + const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); // never follow or reuse a file + if (fd >= 0) f = ::fdopen(fd, "wb"); + } +#endif + if (!f) { set_err(err, "cannot create a temporary file in " + dir.string()); return nullptr; } + cf->path_ = path; + cf->remove_ = true; // from here the destructor deletes the file, also on failure + std::string e; + const uint64_t n = write_component(bundle, comp, f, &e); + std::fclose(f); + if (n == 0) { set_err(err, e.empty() ? "cannot write the temporary file " + path : e); return nullptr; } + cf->size_ = n; + return cf; +} + +} // namespace pk diff --git a/src/bundle_extract.hpp b/src/bundle_extract.hpp new file mode 100644 index 0000000..829885b --- /dev/null +++ b/src/bundle_extract.hpp @@ -0,0 +1,53 @@ +#pragma once +// A standalone single-model GGUF made from one component of a bundle, for a +// third-party loader that only opens a file by path (ced.cpp, voice-detect.cpp). +// +// The component's keys and tensors are written without the "." prefix, +// so the result is a valid single-model GGUF that the loader reads as if it had +// been published on its own. Tensor data is streamed from the bundle in blocks; +// only the requested component's bytes are read. +// +// Where the file lives: +// * Linux: an anonymous in-memory file (memfd_create), opened through +// /proc/self/fd/N. Nothing is written to disk. +// * Elsewhere, or when memfd_create or /proc is not available, or when the +// environment variable PARAKEET_BUNDLE_NO_MEMFD is set to a value other than +// "0": a temporary file in the system temporary directory (TMPDIR, TEMP, +// else /tmp), created owner-only and removed when the ComponentFile is +// destroyed. A crash between creation and removal leaves the file behind. +// +// Keep the object alive until the loader has finished opening the file, then +// let it go out of scope. +#include +#include +#include + +namespace pk { + +class ComponentFile { +public: + // Returns nullptr and writes the reason to *err on failure (bundle unreadable, + // component missing, truncated bundle, no space for the file). + static std::unique_ptr create(const std::string& bundle, const std::string& component, + std::string* err); + ~ComponentFile(); + ComponentFile(const ComponentFile&) = delete; + ComponentFile& operator=(const ComponentFile&) = delete; + + // Path to pass to a path-only loader. Valid until the object is destroyed. + const std::string& path() const { return path_; } + // True when the file is an in-memory file, false for a temporary file on disk. + bool in_memory() const { return in_memory_; } + // Size of the standalone GGUF in bytes. + uint64_t size() const { return size_; } + +private: + ComponentFile() = default; + std::string path_; + bool in_memory_ = false; + int fd_ = -1; // memfd, when in_memory_ + bool remove_ = false; // temporary file to delete + uint64_t size_ = 0; +}; + +} // namespace pk diff --git a/src/ced_tagger.cpp b/src/ced_tagger.cpp index 122d1fd..b902e8f 100644 --- a/src/ced_tagger.cpp +++ b/src/ced_tagger.cpp @@ -1,5 +1,6 @@ #include "ced_tagger.hpp" +#include "bundle_extract.hpp" #include "gguf.h" #ifdef PARAKEET_WITH_CED @@ -23,9 +24,21 @@ bool gguf_is_ced(const std::string& path) { bool CedTagger::available() { return true; } -std::unique_ptr CedTagger::load(const std::string& path) { - ced_ctx* c = ced_capi_load(path.c_str()); - if (!c) return nullptr; +std::unique_ptr CedTagger::load(const std::string& path, const std::string& component, + std::string* err) { + ced_ctx* c = nullptr; + if (component.empty()) { + c = ced_capi_load(path.c_str()); + } else { + std::string e; + std::unique_ptr cf = ComponentFile::create(path, component, &e); + if (!cf) { if (err) *err = e; return nullptr; } + c = ced_capi_load(cf->path().c_str()); // the file is removed when cf goes out of scope + } + if (!c) { + if (err && err->empty()) *err = "cannot load the sound model " + (component.empty() ? path : component); + return nullptr; + } std::unique_ptr t(new CedTagger()); t->ctx_ = c; return t; @@ -55,7 +68,10 @@ SoundScorer CedTagger::scorer() { #else // PARAKEET_WITH_CED bool CedTagger::available() { return false; } -std::unique_ptr CedTagger::load(const std::string&) { return nullptr; } +std::unique_ptr CedTagger::load(const std::string&, const std::string&, std::string* err) { + if (err) *err = "this build has no sound-event support (PARAKEET_WITH_CED=OFF)"; + return nullptr; +} CedTagger::~CedTagger() = default; int CedTagger::n_classes() const { return 0; } const char* CedTagger::label(int) const { return nullptr; } diff --git a/src/ced_tagger.hpp b/src/ced_tagger.hpp index b932916..9054232 100644 --- a/src/ced_tagger.hpp +++ b/src/ced_tagger.hpp @@ -17,8 +17,14 @@ class CedTagger { public: // False when parakeet was built with PARAKEET_WITH_CED=OFF. static bool available(); - // nullptr on failure (or when unavailable). - static std::unique_ptr load(const std::string& gguf_path); + // nullptr on failure (or when unavailable). With a non-empty `component`, + // `gguf_path` is a bundle GGUF (docs/bundle.md) and the model is its component + // of kind "ced". ced.cpp opens models by path only, so the component is first + // written as a standalone GGUF (an in-memory file on Linux, else a temporary + // file removed after the load; see bundle_extract.hpp). `err`, when given, + // receives the reason for a failure. + static std::unique_ptr load(const std::string& gguf_path, const std::string& component = "", + std::string* err = nullptr); ~CedTagger(); CedTagger(const CedTagger&) = delete; CedTagger& operator=(const CedTagger&) = delete; diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index 290d620..9d8a93a 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -284,6 +284,26 @@ parakeet_ctx* load_bundle_component(const char* path, const std::string& name) { g_load_error = "cannot load diarization component \"" + name + "\" (see the log for the reason)"; return nullptr; } + } else if (c->kind == pk::kBundleKindCed) { + ctx->tagger = pk::CedTagger::load(path, name, &err); + if (!ctx->tagger) { g_load_error = "cannot load sound component \"" + name + "\": " + err; return nullptr; } + } else if (c->kind == pk::kBundleKindVoice) { + try { + ctx->speaker = pk::SpeakerEncoder::load(path, name, &err); + } catch (...) { ctx->speaker.reset(); } + if (!ctx->speaker) { g_load_error = "cannot load voice component \"" + name + "\": " + err; return nullptr; } + // Profiles are tied to the exact model. For a component the identity is the sha256 + // of the single-model file it was built from, recorded in the bundle header, so a + // voice enrolled with the standalone file matches the same model inside a bundle. + const std::string& h = c->source_sha256; + bool hex = h.size() == 64; + for (char ch : h) hex = hex && ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f')); + if (!hex) { + g_load_error = "voice component \"" + name + "\" has no valid source_sha256 in the bundle header, " + "which is its speaker model identity"; + return nullptr; + } + ctx->speaker_identity = "sha256:" + h; } else if (c->kind == pk::kBundleKindVad && pk::bundle_vad_is_slice(path, name)) { // A VAD-only slice: same context as a standalone slice file. ctx->vad_model = pk::Model::load_vad_only(path, name); diff --git a/src/speaker_encoder.cpp b/src/speaker_encoder.cpp index 6962b86..cd74f47 100644 --- a/src/speaker_encoder.cpp +++ b/src/speaker_encoder.cpp @@ -1,5 +1,6 @@ #include "speaker_encoder.hpp" +#include "bundle_extract.hpp" #include "gguf.h" #ifdef PARAKEET_WITH_VOICEDETECT @@ -23,12 +24,25 @@ bool gguf_is_voicedetect(const std::string& path) { bool SpeakerEncoder::available() { return true; } -std::unique_ptr SpeakerEncoder::load(const std::string& path) { - voicedetect_ctx* c = voicedetect_capi_load(path.c_str()); - if (!c) return nullptr; +std::unique_ptr SpeakerEncoder::load(const std::string& path, const std::string& component, + std::string* err) { + voicedetect_ctx* c = nullptr; + if (component.empty()) { + c = voicedetect_capi_load(path.c_str()); + } else { + std::string e; + std::unique_ptr cf = ComponentFile::create(path, component, &e); + if (!cf) { if (err) *err = e; return nullptr; } + c = voicedetect_capi_load(cf->path().c_str()); // the file is removed when cf goes out of scope + } + if (!c) { + if (err && err->empty()) *err = "cannot load the speaker model " + (component.empty() ? path : component); + return nullptr; + } const int dim = voicedetect_capi_embedding_dim(c); if (dim <= 0) { // an analyze-only model has no speaker embedding voicedetect_capi_free(c); + if (err) *err = "the speaker model has no speaker embedding (an analysis model?)"; return nullptr; } std::unique_ptr e(new SpeakerEncoder()); @@ -62,7 +76,10 @@ bool SpeakerEncoder::embed(const float* pcm, int n, std::vector& emb) { #else // PARAKEET_WITH_VOICEDETECT bool SpeakerEncoder::available() { return false; } -std::unique_ptr SpeakerEncoder::load(const std::string&) { return nullptr; } +std::unique_ptr SpeakerEncoder::load(const std::string&, const std::string&, std::string* err) { + if (err) *err = "this build has no speaker identification (PARAKEET_WITH_VOICEDETECT=OFF)"; + return nullptr; +} SpeakerEncoder::~SpeakerEncoder() = default; bool SpeakerEncoder::embed(const float*, int, std::vector&) { last_error_ = "built without speaker identification (PARAKEET_WITH_VOICEDETECT=OFF)"; diff --git a/src/speaker_encoder.hpp b/src/speaker_encoder.hpp index 165127f..b5e2b75 100644 --- a/src/speaker_encoder.hpp +++ b/src/speaker_encoder.hpp @@ -16,8 +16,14 @@ class SpeakerEncoder { // False when parakeet was built with PARAKEET_WITH_VOICEDETECT=OFF. static bool available(); // nullptr on failure, when unavailable, or when the GGUF has no speaker - // embedding (for example an age/gender/emotion model). - static std::unique_ptr load(const std::string& gguf_path); + // embedding (for example an age/gender/emotion model). With a non-empty + // `component`, `gguf_path` is a bundle GGUF (docs/bundle.md) and the encoder is + // its component of kind "voice". voice-detect.cpp opens models by path only, so + // the component is first written as a standalone GGUF (an in-memory file on + // Linux, else a temporary file removed after the load; see bundle_extract.hpp). + // `err`, when given, receives the reason for a failure. + static std::unique_ptr load(const std::string& gguf_path, const std::string& component = "", + std::string* err = nullptr); ~SpeakerEncoder(); SpeakerEncoder(const SpeakerEncoder&) = delete; SpeakerEncoder& operator=(const SpeakerEncoder&) = delete; From e9fc8cf49135793600f3aaabf0f453dfb23baedb Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 07:14:19 +0000 Subject: [PATCH 6/9] feat: per-component licences and a NOTICE with the licence texts bundle_gguf.py builds the kinds diar, ced and voice and checks each input against its kind (a diarization GGUF, a ced.cpp GGUF, a speaker encoder with an embedding). Known sources must carry the agreed licence: Nemotron diarization OpenMDW-1.1, CED, ECAPA and ERes2Net Apache-2.0, WeSpeaker CC-BY-4.0 (the stricter of the two readings upstream), Silero MIT. The build refuses models that must not be bundled: the audeering heads (CC-BY-NC-SA) and the end-of-utterance model (NVIDIA Open Model License), by source, licence or input name. --notice prints each component's credit and then the full text of every licence used, taken from scripts/bundle_licenses/. It fails when a licence has no text there. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- scripts/bundle_gguf.py | 77 ++++- scripts/bundle_licenses/Apache-2.0.txt | 202 ++++++++++++ scripts/bundle_licenses/CC-BY-4.0.txt | 396 ++++++++++++++++++++++++ scripts/bundle_licenses/MIT.txt | 21 ++ scripts/bundle_licenses/OpenMDW-1.1.txt | 19 ++ tests/python/check_bundle.py | 106 ++++++- 6 files changed, 814 insertions(+), 7 deletions(-) create mode 100644 scripts/bundle_licenses/Apache-2.0.txt create mode 100644 scripts/bundle_licenses/CC-BY-4.0.txt create mode 100644 scripts/bundle_licenses/MIT.txt create mode 100644 scripts/bundle_licenses/OpenMDW-1.1.txt diff --git a/scripts/bundle_gguf.py b/scripts/bundle_gguf.py index 032070e..3b9316c 100644 --- a/scripts/bundle_gguf.py +++ b/scripts/bundle_gguf.py @@ -22,7 +22,10 @@ bundle_gguf.py --list bundle.gguf [--json] bundle_gguf.py --verify bundle.gguf [--source asr=asr.gguf --source vad=silero.gguf] - bundle_gguf.py --notice bundle.gguf + bundle_gguf.py --notice bundle.gguf # credits plus the full licence texts to ship + +Kinds: asr (parakeet ASR), vad (Silero, or a VAD-only slice), diar (Nemotron +diarization), ced (ced.cpp sound events), voice (voice-detect.cpp speaker encoder). The output is deterministic: the same inputs and manifest give the same bytes. The script refuses a component without full licence metadata, a licence that @@ -40,7 +43,8 @@ ARCH = "parakeet-bundle" VERSION = 1 -KINDS = ("asr", "vad") # kinds this tool can build (docs/bundle.md lists the reserved ones) +KINDS = ("asr", "vad", "diar", "ced", "voice") # kinds this tool can build (see docs/bundle.md) +LICENSE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bundle_licenses") RESERVED_NAMES = {"general", "parakeet", "bundle"} NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$") LICENSE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+-]*$") @@ -49,6 +53,27 @@ MAX_TENSOR_NAME = 63 # ggml keeps names in 64 bytes including the NUL SKIP_KEYS = {"general.alignment"} +# Licence of each known source, as agreed for bundling. A manifest that names one of these +# sources with another licence is refused, so a wrong or silent change cannot slip in. +# WeSpeaker: Hugging Face says Apache-2.0 for the plain model and CC-BY-4.0 for the -LM +# variant, and the voice-detect GGUF card says CC-BY-4.0. The stricter reading is used. +KNOWN_SOURCES = ( + ("nemotron-3-diarization", "OpenMDW-1.1"), + ("mispeech/ced", "Apache-2.0"), + ("spkrec-ecapa", "Apache-2.0"), + ("eres2net", "Apache-2.0"), + ("wespeaker", "CC-BY-4.0"), + ("silero", "MIT"), +) +# Models that must never be bundled, with the reason. Matched against the manifest's +# source and licence and against the input's general.name. +FORBIDDEN = ( + (re.compile(r"audeering"), "audeering wav2vec2 heads are CC-BY-NC-SA-4.0 (non-commercial)"), + (re.compile(r"(^|[^a-z])eou([^a-z]|$)"), "the end-of-utterance model is under the NVIDIA Open Model License " + "(notice duty and a revocation clause)"), + (re.compile(r"nvidia-open-model"), "the NVIDIA Open Model License has a revocation clause"), +) + class BundleError(Exception): pass @@ -137,6 +162,14 @@ def check_license_metadata(where, meta): f"such models must not be bundled") if not re.match(r"^https?://", meta["license_url"]): raise BundleError(f"{where}: license_url must be an http(s) URL") + for rx, why in FORBIDDEN: + if rx.search(norm_license(meta["source"])) or rx.search(norm_license(meta["license"])): + raise BundleError(f"{where}: must not be bundled: {why}") + src = meta["source"].lower() + for needle, lic in KNOWN_SOURCES: + if needle in src and norm_license(meta["license"]) != norm_license(lic): + raise BundleError(f"{where}: source '{meta['source']}' is licensed {lic}, the manifest says " + f"'{meta['license']}'; fix the manifest (or the KNOWN_SOURCES table if upstream changed)") # --------------------------------------------------------------------------- build @@ -193,15 +226,34 @@ def check_input(name, kind, reader, meta): if str_field(reader, "parakeet.arch") == "vad": raise BundleError(f"component {name}: input is a VAD-only slice, not an ASR model; use kind vad") if str_field(reader, "parakeet.arch") == "diarization": - raise BundleError(f"component {name}: input is a diarization model, not ASR (a later phase)") + raise BundleError(f"component {name}: input is a diarization model, not ASR; use kind diar") for k in ("parakeet.encoder.d_model", "parakeet.vocab_size"): if k not in reader.fields: raise BundleError(f"component {name}: input lacks the key {k}") + elif kind == "diar": + if arch != "parakeet" or str_field(reader, "parakeet.arch") != "diarization": + raise BundleError(f"component {name}: kind diar needs a Nemotron diarization GGUF " + f"(parakeet.arch 'diarization'; general.architecture is '{arch}')") + for k in ("parakeet.diar.n_speakers", "parakeet.encoder.d_model"): + if k not in reader.fields: + raise BundleError(f"component {name}: input lacks the key {k}") + elif kind == "ced": + if arch != "ced" or "ced.embed_dim" not in reader.fields: + raise BundleError(f"component {name}: kind ced needs a ced.cpp GGUF (general.architecture is '{arch}')") + elif kind == "voice": + if arch != "voicedetect" or "voicedetect.embedding_dim" not in reader.fields: + raise BundleError(f"component {name}: kind voice needs a voice-detect.cpp speaker encoder GGUF " + f"(general.architecture 'voicedetect' with an embedding; analysis heads are not " + f"bundled); general.architecture is '{arch}'") elif kind == "vad": is_slice = arch == "parakeet" and str_field(reader, "parakeet.arch") == "vad" if not is_slice and (arch != "silero_vad" or "silero_vad.sample_rates" not in reader.fields): raise BundleError(f"component {name}: kind vad needs a Silero VAD GGUF or a VAD-only slice " f"(scripts/slice_vad_gguf.py); general.architecture is '{arch}'") + gname = (str_field(reader, "general.name") or "").lower() + for rx, why in FORBIDDEN: + if rx.search(gname): + raise BundleError(f"component {name}: input '{gname}' must not be bundled: {why}") declared = str_field(reader, "general.license") if declared is not None and norm_license(declared) != norm_license(meta["license"]): raise BundleError(f"component {name}: the input file declares general.license '{declared}' but the manifest " @@ -401,6 +453,18 @@ def compare_with_source(r, n, src): return problems +def license_text(spdx): + """Full text of a licence from scripts/bundle_licenses/, matched on the normalised id.""" + want = norm_license(spdx) + if os.path.isdir(LICENSE_DIR): + for f in sorted(os.listdir(LICENSE_DIR)): + if f.endswith(".txt") and norm_license(f[:-4]) == want: + with open(os.path.join(LICENSE_DIR, f), encoding="utf-8") as fh: + return fh.read().strip("\n") + raise BundleError(f"no licence text for '{spdx}': add scripts/bundle_licenses/{spdx}.txt " + f"(the text must ship with the bundle)") + + def notice(path): r = load_bundle(path) lines = [ @@ -410,11 +474,14 @@ def notice(path): "Each model keeps its own licence and attribution, listed below. They are also stored in the", "file header under parakeet.bundle..*. The models were converted to GGUF and", "merged without further changes to their weights, except where a component says otherwise.", - "The full text of each licence is at the URL given for it.", + "The full text of every licence used follows the component list. Ship this file with the bundle.", "", ] + used = [] for n in components_of(r): i = component_info(r, n) + if i["license"] not in used: + used.append(i["license"]) lines += [ f"Component {n} ({i['kind']})", f" Model: {i['source']}", @@ -424,6 +491,8 @@ def notice(path): f" Input file sha256: {i['source_sha256']}", "", ] + for lic in used: + lines += ["=" * 78, f"Licence text: {lic}", "=" * 78, "", license_text(lic), ""] sys.stdout.write("\n".join(lines)) diff --git a/scripts/bundle_licenses/Apache-2.0.txt b/scripts/bundle_licenses/Apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/scripts/bundle_licenses/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/bundle_licenses/CC-BY-4.0.txt b/scripts/bundle_licenses/CC-BY-4.0.txt new file mode 100644 index 0000000..da6ab6c --- /dev/null +++ b/scripts/bundle_licenses/CC-BY-4.0.txt @@ -0,0 +1,396 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/scripts/bundle_licenses/MIT.txt b/scripts/bundle_licenses/MIT.txt new file mode 100644 index 0000000..0bf5e90 --- /dev/null +++ b/scripts/bundle_licenses/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020-present Silero Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/scripts/bundle_licenses/OpenMDW-1.1.txt b/scripts/bundle_licenses/OpenMDW-1.1.txt new file mode 100644 index 0000000..23a55d5 --- /dev/null +++ b/scripts/bundle_licenses/OpenMDW-1.1.txt @@ -0,0 +1,19 @@ +OpenMDW License Agreement, version 1.1 (OpenMDW-1.1) + +By exercising rights granted to you under this agreement, you accept and agree to its terms. + +As used in this agreement, “Model Materials” means the materials provided to you under this agreement, consisting of: (1) one or more machine learning models (including architecture and parameters); and (2) all related artifacts (including associated data, documentation and software) that are provided to you hereunder. + +Subject to your compliance with this agreement, permission is hereby granted, free of charge, to deal in the Model Materials without restriction, including under all copyright, patent, database, and trade secret rights included or embodied therein. + +If you distribute any portion of the Model Materials, you shall retain in your distribution (1) a copy of this agreement, and (2) all copyright notices and other notices of origin included in the Model Materials that are applicable to your distribution. + +If you file, maintain, or voluntarily participate in a lawsuit against any person or entity asserting that the Model Materials directly or indirectly infringe any patent or copyright, then all rights and grants made to you hereunder are terminated, unless that lawsuit was in response to a corresponding lawsuit first brought against you. + +This agreement does not impose any restrictions or obligations with respect to any use, modification, or sharing of any outputs generated by using the Model Materials. + +THE MODEL MATERIALS ARE PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, NONINFRINGEMENT, ACCURACY, OR THE ABSENCE OF LATENT OR OTHER DEFECTS OR ERRORS, WHETHER OR NOT DISCOVERABLE, ALL TO THE GREATEST EXTENT PERMISSIBLE UNDER APPLICABLE LAW. + +YOU ARE SOLELY RESPONSIBLE FOR (1) CLEARING RIGHTS OF OTHER PERSONS THAT MAY APPLY TO THE MODEL MATERIALS OR ANY USE THEREOF, INCLUDING WITHOUT LIMITATION ANY PERSON’S COPYRIGHTS OR OTHER RIGHTS INCLUDED OR EMBODIED IN THE MODEL MATERIALS; (2) OBTAINING ANY NECESSARY CONSENTS, PERMISSIONS OR OTHER RIGHTS REQUIRED FOR ANY USE OF THE MODEL MATERIALS; OR (3) PERFORMING ANY DUE DILIGENCE OR UNDERTAKING ANY OTHER INVESTIGATIONS INTO THE MODEL MATERIALS OR ANYTHING INCORPORATED OR EMBODIED THEREIN. + +IN NO EVENT SHALL THE PROVIDERS OF THE MODEL MATERIALS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE MODEL MATERIALS, THE USE THEREOF OR OTHER DEALINGS THEREIN. diff --git a/tests/python/check_bundle.py b/tests/python/check_bundle.py index c80116e..8ee6583 100644 --- a/tests/python/check_bundle.py +++ b/tests/python/check_bundle.py @@ -71,6 +71,41 @@ def write_slice(path, license="CC-BY-4.0"): w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() +def write_diar(path, name="nvidia/Nemotron-3-Diarization"): + w = GGUFWriter(path, "parakeet") + w.add_string("general.name", name) + w.add_string("parakeet.arch", "diarization") + w.add_uint32("parakeet.encoder.d_model", 16) + w.add_uint32("parakeet.diar.n_speakers", 4) + rng = np.random.default_rng(4) + w.add_tensor("encoder.w", rng.standard_normal((4, 16)).astype(np.float32)) + q = rng.integers(0, 255, size=(4, 34), dtype=np.uint8) + w.add_tensor("encoder.q", q, raw_shape=q.shape, raw_dtype=GGMLQuantizationType.Q8_0) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + + +def write_ced(path): + w = GGUFWriter(path, "ced") + w.add_string("general.name", "mispeech/ced-tiny") + w.add_string("ced.arch", "ced") + w.add_uint32("ced.embed_dim", 8) + w.add_array("ced.labels", ["Speech", "Music"]) + rng = np.random.default_rng(5) + w.add_tensor("patch_embed.proj.weight", rng.standard_normal((8, 4)).astype(np.float32)) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + + +def write_voice(path, arch="voicedetect", embedding=True, name="Wespeaker/wespeaker-voxceleb-resnet34"): + w = GGUFWriter(path, arch) + w.add_string("general.name", name) + w.add_string("voicedetect.arch", "wespeaker_resnet34") + if embedding: + w.add_uint32("voicedetect.embedding_dim", 256) + rng = np.random.default_rng(6) + w.add_tensor("model.seg_1.weight", rng.standard_normal((4, 8)).astype(np.float32)) + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + + def write_vad(path, license="MIT"): w = GGUFWriter(path, "silero_vad") w.add_string("general.name", "silero-vad") @@ -84,6 +119,14 @@ def write_vad(path, license="MIT"): w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() +DIAR_META = {"kind": "diar", "license": "OpenMDW-1.1", "license_url": "https://openmdw.ai/license/1-1/", + "source": "nvidia/Nemotron-3-Diarization", "attribution": "Nemotron 3 Diarization by NVIDIA", + "changes": "Converted to GGUF and quantised to Q8_0"} +CED_META = {"kind": "ced", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", + "source": "mispeech/ced-tiny", "attribution": "CED by Xiaomi (mispeech)", "changes": "Converted to GGUF"} +VOICE_META = {"kind": "voice", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", + "source": "Wespeaker/wespeaker-voxceleb-resnet34", "attribution": "WeSpeaker ResNet34", + "changes": "Converted from ONNX to GGUF"} ASR_META = {"kind": "asr", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "source": "nvidia/stand-in", "attribution": "Stand-in model by NVIDIA", "changes": "Converted to GGUF"} VAD_META = {"kind": "vad", "license": "MIT", "license_url": "https://github.com/snakers4/silero-vad/blob/master/LICENSE", @@ -153,6 +196,64 @@ def main(): rc, out, err = run("--out", "bs2.gguf", "--manifest", "ms2.json") check(rc != 0 and "VAD-only slice" in err, "a slice as kind asr must be refused") + # --- the other kinds: diar, ced, voice; a full bundle with every kind --- + write_diar("diar.gguf"); write_ced("ced.gguf"); write_voice("voice.gguf") + manifest("mf.json", [comp("asr", "asr.gguf", ASR_META), comp("diar", "diar.gguf", DIAR_META), + comp("ced", "ced.gguf", CED_META), comp("voice", "voice.gguf", VOICE_META), + comp("vad", "vad.gguf", VAD_META)], name="full-test") + rc, out, err = run("--out", "bf.gguf", "--manifest", "mf.json") + check(rc == 0, f"full bundle failed to build: {err}") + rc, out, err = run("--out", "bf2.gguf", "--manifest", "mf.json") + check(rc == 0 and sha("bf.gguf") == sha("bf2.gguf"), "the full bundle is not deterministic") + rc, out, err = run("--verify", "bf.gguf", "--source", "asr=asr.gguf", "--source", "diar=diar.gguf", + "--source", "ced=ced.gguf", "--source", "voice=voice.gguf", "--source", "vad=vad.gguf") + check(rc == 0, f"verify of the full bundle failed: {err}") + rc, out, err = run("--list", "bf.gguf", "--json") + j2 = json.loads(out) if rc == 0 else {} + check(sorted(j2.get("components", {})) == ["asr", "ced", "diar", "vad", "voice"], "full list components") + check({c: v["kind"] for c, v in j2.get("components", {}).items()} == + {"asr": "asr", "diar": "diar", "ced": "ced", "voice": "voice", "vad": "vad"}, "full list kinds") + # NOTICE: every component's credit and the full text of each distinct licence + rc, out, err = run("--notice", "bf.gguf") + check(rc == 0, f"notice failed: {err}") + for needle in ("Nemotron 3 Diarization by NVIDIA", "CED by Xiaomi (mispeech)", "WeSpeaker ResNet34", + "Copyright (c) 2020-present Silero Team", "NVIDIA", "OpenMDW License Agreement, version 1.1", + "Apache License", "Attribution 4.0 International", "Permission is hereby granted", + "you shall retain in your distribution"): + check(needle in out, f"notice lacks: {needle}") + check(out.count("Licence text: ") == 4, "one text per distinct licence (CC-BY-4.0 is used twice)") + # a licence without a shipped text is an error, not a silent gap + manifest("mx.json", [comp("asr", "asr.gguf", dict(ASR_META, license="LicenseRef-Other"))]) + run("--out", "bx.gguf", "--manifest", "mx.json") + rc, out, err = run("--notice", "bx.gguf") + check(rc == 1 and "no licence text" in err, "a licence without a text must make --notice fail") + + # --- kinds must match the file, and the forbidden models are refused --- + def refuse2(label, comps, needle): + manifest("bad2.json", comps) + rc, out, err = run("--out", "bad2.gguf", "--manifest", "bad2.json") + check(rc == 1 and needle in err, f"{label}: want a refusal with '{needle}', got rc={rc} {err!r}") + check(not os.path.exists("bad2.gguf"), f"{label}: left a file behind") + refuse2("diar kind on an ASR file", [comp("diar", "asr.gguf", DIAR_META)], "kind diar") + refuse2("ced kind on a voice file", [comp("ced", "voice.gguf", CED_META)], "kind ced") + refuse2("voice kind on a ced file", [comp("voice", "ced.gguf", VOICE_META)], "kind voice") + write_voice("analyze.gguf", embedding=False, name="audeering/wav2vec2-large-robust-24-ft-age-gender") + refuse2("analysis head as voice (no embedding)", [comp("voice", "analyze.gguf", VOICE_META)], "kind voice") + write_voice("analyze2.gguf", name="audeering/wav2vec2-large-robust-24-ft-age-gender") + refuse2("audeering by file name", [comp("voice", "analyze2.gguf", VOICE_META)], "must not be bundled") + refuse2("audeering by source and licence", + [comp("voice", "voice.gguf", dict(VOICE_META, source="audeering/wav2vec2-large-robust", license="CC-BY-NC-SA-4.0"))], + "must not be bundled") + refuse2("NVIDIA Open Model License (EOU)", + [comp("asr", "asr.gguf", dict(ASR_META, source="nvidia/parakeet_realtime_eou_120m-v1", + license="LicenseRef-NVIDIA-Open-Model-License"))], "must not be bundled") + refuse2("EOU by licence text", [comp("asr", "asr.gguf", dict(ASR_META, license="NVIDIA-Open-Model-License"))], + "must not be bundled") + refuse2("diar with a wrong licence", [comp("diar", "diar.gguf", dict(DIAR_META, license="CC-BY-4.0"))], "is licensed OpenMDW-1.1") + refuse2("ced with a wrong licence", [comp("ced", "ced.gguf", dict(CED_META, license="MIT"))], "is licensed Apache-2.0") + refuse2("wespeaker as Apache-2.0 (the stricter reading is required)", + [comp("voice", "voice.gguf", dict(VOICE_META, license="Apache-2.0"))], "is licensed CC-BY-4.0") + # --- a tampered bundle fails verify --- data = bytearray(open("b1.gguf", "rb").read()) data[-5] ^= 0xFF @@ -193,9 +294,8 @@ def refuse(label, comps, needle, **kw): refuse("silero licence conflict", [comp("vad", "vad_bad_lic.gguf", VAD_META)], "declares general.license") refuse("kind does not match the file", [comp("asr", "vad.gguf", ASR_META)], "kind asr") refuse("vad kind on an ASR file", [comp("vad", "asr.gguf", VAD_META)], "kind vad") - write_asr("diar.gguf", parakeet_arch="diarization") - refuse("diarization is a later phase", [comp("asr", "diar.gguf", ASR_META)], "diarization") - refuse("unknown kind", [comp("x", "asr.gguf", ASR_META, kind="diarization")], "kind must be") + refuse("a diarization file is not kind asr", [comp("asr", "diar.gguf", ASR_META)], "use kind diar") + refuse("unknown kind", [comp("x", "asr.gguf", ASR_META, kind="sound")], "kind must be") refuse("duplicate names", [comp("asr", "asr.gguf", ASR_META), comp("asr", "asr.gguf", ASR_META)], "duplicate") for bad in ("Asr", "a.b", "general", "parakeet", "bundle", "1a", ""): refuse(f"bad name {bad!r}", [comp(bad, "asr.gguf", ASR_META)], "component name") From d4be85248c3bfc35bbc67fe6bf99f7a31eceba7f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 07:14:19 +0000 Subject: [PATCH 7/9] docs: describe the diar, ced and voice bundle components Kinds, selection rules for commands that need one kind, the standalone copy used for the ced.cpp and voice-detect.cpp loaders (memfd and the temporary-file fallback), the follow-up of a load_from_buffer hook in those projects, the licence table with the WeSpeaker discrepancy, and what is not verified. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- AGENTS.md | 6 +- docs/bundle.md | 154 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 134 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 808c6e3..413423b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,6 +82,7 @@ src/ libparakeet implementation audio_io.hpp/cpp , dr_wav load + linear resample to 16k model_loader.hpp/cpp, GGUF -> ParakeetConfig + name->tensor bundle.hpp/cpp , bundle GGUF header: components, licences, default component (docs/bundle.md) + bundle_extract.hpp/cpp, one bundle component as a standalone GGUF (memfd or temporary file) for the path-only ced.cpp and voice-detect.cpp loaders mel.cpp , log-mel frontend encoder.cpp / conformer.cpp / relpos_attention.cpp ctc_decoder.cpp , CTC head + greedy decode @@ -113,7 +114,7 @@ examples/cli/ parakeet-cli binary diarize binary: diarize [--stream] scripts/ Python tooling convert_parakeet_to_gguf.py, .nemo/.hf -> GGUF (--dtype f32|f16|q8_0) - bundle_gguf.py , merge GGUFs into one bundle; --list/--verify/--notice (docs/bundle.md) + bundle_gguf.py , merge GGUFs into one bundle; --list/--verify/--notice with the licence texts of scripts/bundle_licenses/ (docs/bundle.md) convert_hf_parakeet_to_gguf.py, HF safetensors (moondream/parakeet-ultra, -redux) to GGUF (--template, --ternary keep|dequant, --vad keep|drop) slice_vad_gguf.py , cut a VAD-only GGUF (parakeet.arch "vad") out of an Ultra or Redux GGUF gen_nemo_baseline.py , NeMo intermediates -> baseline.gguf @@ -133,7 +134,8 @@ tests/ ctest targets test_model_loader.cpp , config + tensor map (model-dependent) test_bundle.cpp , bundle header, selection, partial read, Silero component, C-API (model-independent, synthetic files) test_bundle_models.cpp , real bundle == single-model files: transcript, VAD head, Silero (PARAKEET_TEST_BUNDLE, _ASR, _SILERO) - python/check_bundle.py , bundle_gguf.py build/verify/refusals (model-independent) + test_bundle_full.cpp , full bundle (asr, diar, ced, voice, vad) == single-model files, memfd and temporary-file paths (PARAKEET_TEST_BUNDLE_FULL, PARAKEET_TEST_FULL_*) + python/check_bundle.py , bundle_gguf.py build/verify/refusals, licence policy, NOTICE (model-independent) test_capi.cpp , C-API load -> transcribe -> free (model-dependent) test_transcribe_speech.cpp, end-to-end CTC transcript (model-dependent) test_transcribe_tdt.cpp , TDT transcript on speech fixture (model-dependent) diff --git a/docs/bundle.md b/docs/bundle.md index b00a754..512080c 100644 --- a/docs/bundle.md +++ b/docs/bundle.md @@ -1,8 +1,10 @@ # Bundle GGUF A bundle is one GGUF file that holds several single-model GGUF files, so a user -passes one file instead of several. Phase 1 supports two component kinds: an -ASR model (with its own VAD head, if it has one) and a Silero VAD model. +passes one file instead of several. A bundle can hold five kinds of component: +an ASR model (with its own VAD head, if it has one), a VAD model (Silero, or a +VAD-only slice), speaker diarization, sound-event tagging (ced.cpp) and speaker +identification (voice-detect.cpp). A bundle is a packaging format. It does not change how any model runs, and it does not convert or re-quantise any weight: each tensor is copied byte for byte @@ -35,8 +37,13 @@ A component name is 1 to 32 characters from `a-z`, `0-9` and `_`, and starts wit a letter. The names `general`, `parakeet` and `bundle` are reserved. A name never contains a dot, so `.` is an unambiguous prefix. -By convention the ASR component is named `asr` and the Silero component is -named `vad`. The name has no meaning to the loader: the `kind` key does. +By convention the components are named after their role: `asr`, `vad`, `diar`, +`ced` and `voice`. The name has no meaning to the loader: the `kind` key does. + +The prefixed tensor name must stay within 63 bytes (see "Namespacing"). A few +speaker encoders have long tensor names: the ECAPA GGUF has names of 61 bytes, so +a bundle can only hold it under a one-letter component name (for example `v`). +The build script reports the limit. ### Namespacing @@ -70,7 +77,7 @@ are required. | Key | Meaning | |---|---| -| `parakeet.bundle..kind` | `asr` or `vad` in version 1 (see "Kinds") | +| `parakeet.bundle..kind` | `asr`, `vad`, `diar`, `ced` or `voice` in version 1 (see "Kinds") | | `parakeet.bundle..license` | SPDX identifier, or `LicenseRef-` | | `parakeet.bundle..license_url` | `http(s)` URL of the licence text | | `parakeet.bundle..source` | Upstream model id or URL | @@ -111,9 +118,47 @@ worth knowing: of the ASR component. It does not use a slice component (the ASR component already has the head), and `--vad-component` that names a slice is an error. -The names `diarization`, `sound` and `voice` are reserved for later phases. A -reader skips components of a kind it does not know: they are listed, but it does -not load them and does not fail because they exist. +| `diar` | A Nemotron diarization GGUF (`parakeet.arch` `diarization`) | `DiarizationModel::load(path, name)`, through the same prefixed loader as `asr` | +| `ced` | A ced.cpp GGUF (`general.architecture` `ced`, CED-tiny, -mini, -small or -base) | `CedTagger::load(path, name)`, through a standalone copy (see "Components for third-party loaders") | +| `voice` | A voice-detect.cpp speaker encoder GGUF (`general.architecture` `voicedetect` with an embedding: WeSpeaker ResNet34, ECAPA, ERes2Net, CAM++) | `SpeakerEncoder::load(path, name)`, through a standalone copy | + +A reader skips components of a kind it does not know: they are listed, but it does +not load them and does not fail because they exist. The wav2vec2 analysis heads of +voice-detect.cpp (age, gender, emotion) are not a `voice` component: their licence +is non-commercial, and the build script refuses them. + +## Components for third-party loaders + +`ced.cpp` and `voice-detect.cpp` open a model by file path only, and parakeet.cpp +does not change their sources. To load a `ced` or `voice` component, parakeet.cpp +writes it as a standalone single-model GGUF (keys and tensors without the prefix, +data streamed from the bundle, only that component's bytes read) and gives the +loader the path of that copy: + +* **Linux:** an anonymous in-memory file (`memfd_create`), opened as + `/proc/self/fd/N`. Nothing is written to disk. The loader reads it, then the + file is closed. The copy briefly uses memory equal to the component size. +* **Other systems, or if `memfd_create` or `/proc` is not usable, or when the + environment variable `PARAKEET_BUNDLE_NO_MEMFD` is set to a value other than + `0`:** a temporary file in the system temporary directory (`TMPDIR`, `TEMP`, + else `/tmp`), created owner-only with an exclusive create, and removed as soon as + the load has finished. A crash during the load can leave the file behind. The + path is only used on macOS and Windows by this fallback; it is not tested there. + +The component loaded this way gives the same output as the single-model file: +the CED class scores and the speaker embeddings are bitwise equal in the tests. + +The speaker identity of a `voice` component (`parakeet_capi_speaker_identity`) is +`sha256:` plus the `source_sha256` of its header, which is the sha256 of the +single-model file. A voice enrolled with the standalone file therefore matches the +same model inside a bundle. The header is trusted for this: load bundles you +trust. + +Cleaner alternative, a follow-up outside this repository: a `load_from_buffer` +(or `load_from_reader`) entry point in ced.cpp and voice-detect.cpp, so a +component could be handed over as a memory range with no copy and no temporary +file. Both projects are owned by the same maintainer. When it exists, +`bundle_extract.cpp` can be replaced by a call that passes the mapped range. ## Compatibility rules @@ -151,6 +196,17 @@ A component is chosen by name or by default. choice. * **By name** (`parakeet_capi_load_component`, `--component`): that component, whatever its kind. The context has the kind the component declares. +* **Other kinds** (`diar`, `ced`, `voice`) are never the default when an `asr` + component exists: `parakeet_capi_load` opens the ASR component, and the others + are opened by name with `parakeet_capi_load_component`. A bundle without an + `asr` component but with exactly one loadable component opens that one; with + several it is refused with a message that lists them. +* **Commands that need one kind** (`parakeet-cli scene`, `enroll`, `info`, the + `diarize` tool): given a bundle, they use the only component of the kind they + need (ASR, diarization, sound or speaker) and refuse when there is none or + several; `--component` (`enroll`, `diarize`) or `--asr-component`, + `--diar-component`, `--sound-component`, `--speakers-component` (`scene`) name + one. Naming a component of the wrong kind is an error. * **VAD source** for `parakeet-cli vad` and for segmented transcription (`transcribe --vad`) on a bundle: the Silero component if the bundle has one, otherwise the VAD head of the ASR component. `--component` (for `vad`) and @@ -161,9 +217,11 @@ A component is chosen by name or by default. ## Partial loading A loader reads the header and the tensor table, then reads only the tensors of -the component it was asked for. The ASR loader reads them into one private -memory block, and the Silero loader seeks to each of its tensors. The bytes of -the other components are never read. A test checks the number of bytes read from +the component it was asked for. The ASR and +diarization loaders read them into one private memory block, and the Silero loader +seeks to each of its tensors. The `ced` and `voice` loaders read them while they +write the standalone copy, then read that copy. The bytes of the other components +are never read. A test checks the number of bytes read from the file for each component. Memory: a loaded component uses the same memory as the same single-model file. @@ -178,7 +236,7 @@ and is not memory mapped. bundle_gguf.py --out bundle.gguf --manifest manifest.json bundle_gguf.py --list bundle.gguf [--json] bundle_gguf.py --verify bundle.gguf [--source asr=asr.gguf --source vad=silero.gguf] -bundle_gguf.py --notice bundle.gguf +bundle_gguf.py --notice bundle.gguf # credits and the full licence texts ``` The manifest is a JSON file. `file` is relative to the manifest. @@ -207,6 +265,11 @@ The build: `source`, `attribution` or `changes`; * refuses a licence that is not an SPDX-style identifier, and refuses licences that restrict commercial use or derivatives (`-NC-`, `-ND-`); +* refuses models that must not be bundled: the audeering wav2vec2 heads + (CC-BY-NC-SA-4.0) and the end-of-utterance ASR model (NVIDIA Open Model + License: notice duty and a revocation clause), by source, licence or input name; +* refuses a known source with another licence than the table `KNOWN_SOURCES` in + the script (see "Licences"); * refuses a component whose input file declares a `general.license` that differs from the manifest (the same licence in another spelling is accepted); * records the sha256 of each input file and the content digest of each @@ -222,8 +285,9 @@ bytes) and the keys with the single-model file and its recorded sha256. It exits 1 on any problem. `--notice` prints a NOTICE text from the licence keys: for each component the -model, licence and URL, credit line, changes and input hash. Ship this text and -the licence texts with the file. +model, licence and URL, credit line, changes and input hash, followed by the full +text of every licence used (from `scripts/bundle_licenses/`). It fails when a +licence has no text there. Ship this text with the file. ## Licences @@ -238,11 +302,30 @@ is a collection, so it does not have one licence. Rules the tools apply: the licences require (CC-BY-4.0 asks for credit, a licence link and a note of changes; MIT asks that the copyright notice and licence text stay with copies). +### Licences of the supported components + +| Component | Source | Licence | What the bundle must carry | +|---|---|---|---| +| ASR | NVIDIA Parakeet (NeMo) and Moondream Ultra/Redux | CC-BY-4.0 | credit, licence link, note of changes | +| diar | `nvidia/Nemotron-3-Diarization` | OpenMDW-1.1 | a copy of the agreement and the origin notices | +| ced | `mispeech/ced-*` | Apache-2.0 | licence text and copyright notices | +| voice, ECAPA | `speechbrain/spkrec-ecapa-voxceleb` | Apache-2.0 | licence text and notices | +| voice, ERes2Net | 3D-Speaker | Apache-2.0 | licence text and notices | +| voice, WeSpeaker ResNet34 | `Wespeaker/wespeaker-voxceleb-resnet34` | CC-BY-4.0 (see below) | credit, licence link, note of changes | +| vad | `snakers4/silero-vad` | MIT | copyright notice and licence text | + +WeSpeaker: Hugging Face lists Apache-2.0 for the plain model and CC-BY-4.0 for the +`-LM` variant, and the voice-detect GGUF card says CC-BY-4.0. The table in the +script uses CC-BY-4.0, the stricter reading, until the upstream licence is +confirmed. Not bundled: the audeering heads (CC-BY-NC-SA-4.0) and the EOU model. + ## C-API and CLI ```c parakeet_ctx* parakeet_capi_load(const char* path); // bundle: default ASR component parakeet_ctx* parakeet_capi_load_component(const char* path, const char* name); +// kind asr -> ASR context, diar -> diarization, ced -> sound tagger, voice -> speaker +// encoder, vad -> Silero (or a VAD-only slice). See parakeet_capi_model_kind. char* parakeet_capi_bundle_components_json(const char* path); // free with parakeet_capi_free_string const char* parakeet_capi_load_error(void); // reason of the last failed load on this thread ``` @@ -252,17 +335,40 @@ parakeet-cli info bundle.gguf [--component NAME] parakeet-cli transcribe --model bundle.gguf --input a.wav [--component NAME] parakeet-cli transcribe --model bundle.gguf --input a.wav --vad [--vad-component NAME] parakeet-cli vad --model bundle.gguf --input a.wav [--component NAME] +parakeet-cli scene --model bundle.gguf --diar bundle.gguf --sound bundle.gguf --input a.wav +parakeet-cli scene ... --speakers bundle.gguf --registry reg.bin +parakeet-cli enroll --model bundle.gguf [--component NAME] --name N --input a.wav --registry reg.bin +diarize bundle.gguf a.wav [--stream] [--component NAME] ``` +Every function that takes a context works with the contexts of a bundle: the +diarization, sound and speaker contexts from `parakeet_capi_load_component` +feed `parakeet_capi_diarize_*`, `parakeet_capi_transcribe_and_diarize_*`, the +scene stream and the named-diarization calls the same way as contexts from +single-model files. + `parakeet-cli info` without `--component` lists the components, kinds, licences, sources, credits and sizes from the header. `--component` prints the details of -one component. Other subcommands (bench, scene, streaming) do not take a bundle -yet: they refuse it with a message. - -## Not covered in phase 1 - -* diarization, sound-event and speaker-identification components (the kinds are - reserved; their loaders are separate libraries); -* cache-aware streaming from a bundle component; -* GPU backends were not run with bundles (the loader and the compute path are - the same as for single-model files, but there is no test). +one component. `bench` and the streaming ASR modes do not take a bundle yet: they +refuse it with a message. + +### A VAD-only slice in a bundle + +A `vad` component can also hold a VAD-only slice of an Ultra or Redux model +(`scripts/slice_vad_gguf.py`, `parakeet.arch` `vad`). It is told apart from a +Silero model by that key. `parakeet_capi_load_component` gives a VAD-only context +(only the `parakeet_capi_vad_*` calls work), the same as `parakeet_capi_load` on the +standalone slice file. The build accepts a slice only as kind `vad`. An `asr` +component with its own VAD head needs no slice. `parakeet-cli vad` uses the first +`vad` component by default; `transcribe --vad` looks for a Silero component, else +uses the head of the ASR component, and does not use a slice. + +## Not verified + +* GPU backends (the loaders and the compute path are the same as for + single-model files, but no bundle was run on a GPU); +* cache-aware streaming ASR from a bundle component, and the other ASR families + (EOU, CTC, Nemotron ASR, K-quants) through a bundle; +* macOS and Windows: the temporary-file fallback for `ced` and `voice` is written + for them but not run there; +* a bundle with a K-quant or Q4 component. From ae031a234c38603577345673cd6261dfd751afe2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 07:14:19 +0000 Subject: [PATCH 8/9] test: cover the full bundle (asr, diar, ced, voice, vad) test_bundle checks the standalone copy of a component against the single-model file (memfd and temporary file, no file left behind). test_bundle_full compares a real bundle with its source files: ASR transcript, diarization JSON, CED scores and speaker embedding bitwise, speaker identity, Silero probabilities, named diarization with bundle contexts, partial reads, selection rules and the refusal by the unmodified third-party loaders. check_bundle.py covers the new kinds, the forbidden models, wrong licences and the NOTICE texts. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- tests/CMakeLists.txt | 4 + tests/test_bundle.cpp | 52 ++++++++ tests/test_bundle_full.cpp | 235 +++++++++++++++++++++++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 tests/test_bundle_full.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 97a8714..d015ced 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -248,3 +248,7 @@ set_tests_properties(test_vad_only PROPERTIES LABELS "model") pk_add_test(test_bundle) pk_add_test(test_bundle_models) set_tests_properties(test_bundle_models PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +if(PARAKEET_WITH_CED AND PARAKEET_WITH_VOICEDETECT) + pk_add_test(test_bundle_full) + set_tests_properties(test_bundle_full PROPERTIES LABELS "model" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +endif() diff --git a/tests/test_bundle.cpp b/tests/test_bundle.cpp index 8c721e0..eafb6f2 100644 --- a/tests/test_bundle.cpp +++ b/tests/test_bundle.cpp @@ -14,8 +14,10 @@ #include #include #include +#include #include "bundle.hpp" +#include "bundle_extract.hpp" #include "ggml.h" #include "gguf.h" #include "model.hpp" @@ -382,6 +384,56 @@ int main() { CHECK(!pk::read_bundle_info(Bbad, info, &err)); } + // --- ComponentFile: a component as a standalone GGUF (for path-only third-party loaders) --- + { + char tmpl[] = "test_bundle_tmpdir_XXXXXX"; + const std::string tmpdir = ::mkdtemp(tmpl) ? tmpl : ""; + CHECK(!tmpdir.empty()); + for (int pass = 0; pass < 2; ++pass) { + if (pass == 1) { ::setenv("PARAKEET_BUNDLE_NO_MEMFD", "1", 1); ::setenv("TMPDIR", tmpdir.c_str(), 1); } + for (const char* comp : {"asr", "vad"}) { + const std::string& plain = std::string(comp) == "asr" ? plain_asr : plain_sil; + std::string cerr; + std::string path; + { + std::unique_ptr cf = pk::ComponentFile::create(B, comp, &cerr); + CHECK(cf != nullptr); + if (!cf) continue; + CHECK(cf->in_memory() == (pass == 0)); + path = cf->path(); + // Same keys, same types and the same tensor bytes as the single-model file. + ggml_context *c1 = nullptr, *c2 = nullptr; + gguf_context* g1 = gguf_init_from_file(plain.c_str(), {false, &c1}); + gguf_context* g2 = gguf_init_from_file(path.c_str(), {false, &c2}); + CHECK(g1 && g2); + if (g1 && g2) { + CHECK(gguf_get_n_kv(g1) == gguf_get_n_kv(g2) && gguf_get_n_tensors(g1) == gguf_get_n_tensors(g2)); + for (int64_t i = 0; i < gguf_get_n_kv(g1); ++i) { + const int64_t j = gguf_find_key(g2, gguf_get_key(g1, i)); + CHECK(j >= 0 && gguf_get_kv_type(g1, i) == gguf_get_kv_type(g2, j)); + } + for (ggml_tensor* t = ggml_get_first_tensor(c1); t; t = ggml_get_next_tensor(c1, t)) { + ggml_tensor* u = ggml_get_tensor(c2, t->name); + CHECK(u && u->type == t->type && ggml_nbytes(u) == ggml_nbytes(t) && + std::memcmp(u->data, t->data, ggml_nbytes(t)) == 0); + } + } + if (g1) gguf_free(g1); + if (g2) gguf_free(g2); + if (c1) ggml_free(c1); + if (c2) ggml_free(c2); + } + if (pass == 1) CHECK(access(path.c_str(), F_OK) != 0); // the temporary file is removed + } + std::string cerr; + CHECK(pk::ComponentFile::create(B, "nope", &cerr) == nullptr && !cerr.empty()); + CHECK(pk::ComponentFile::create("test_bundle.does-not-exist", "asr", &cerr) == nullptr); + } + ::unsetenv("PARAKEET_BUNDLE_NO_MEMFD"); + ::rmdir(tmpdir.c_str()); // fails if a temporary file was left behind + CHECK(access(tmpdir.c_str(), F_OK) != 0); + } + // --- a VAD-only slice as a "vad" component (interplay with the standalone slice file) --- { const std::string plain_slice = dir + "slice.gguf", Bs = dir + "bs.gguf", Bs2 = dir + "bs2.gguf"; diff --git a/tests/test_bundle_full.cpp b/tests/test_bundle_full.cpp new file mode 100644 index 0000000..ed4b05e --- /dev/null +++ b/tests/test_bundle_full.cpp @@ -0,0 +1,235 @@ +// A full bundle (asr, diar, ced, voice, vad) against the single-model files it was +// built from: every component gives the same output as its standalone file. +// ASR transcript, diarization JSON, CED class scores (bitwise), speaker embedding +// (bitwise) and identity, Silero probabilities (bitwise), named diarization with +// bundle components. Also: partial loading, selection rules, and the temporary-file +// fallback of the ced and voice loaders (no file is left behind). +// +// LABEL model; run from the project root (fixtures are relative). +// Env (skip 77 unless all are set): +// PARAKEET_TEST_BUNDLE_FULL the bundle (components asr, diar, ced, voice, vad) +// PARAKEET_TEST_FULL_ASR / _DIAR / _CED / _VOICE / _SILERO the single-model files +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "audio_io.hpp" +#include "bundle.hpp" +#include "bundle_extract.hpp" +#include "ced_tagger.hpp" +#include "model.hpp" +#include "parakeet_capi.h" +#include "silero_vad.hpp" +#include "speaker_encoder.hpp" + +static int failures = 0; +#define CHECK(c) do { if (!(c)) { std::fprintf(stderr, "FAIL: %s (line %d)\n", #c, __LINE__); ++failures; } } while (0) + +static unsigned long long bytes_read() { + std::FILE* f = std::fopen("/proc/self/io", "r"); + if (!f) return 0; + char k[64]; + unsigned long long v = 0, r = 0; + while (std::fscanf(f, "%63s %llu", k, &v) == 2) + if (std::string(k) == "rchar:") r = v; + std::fclose(f); + return r; +} +static std::string take(char* p) { + std::string s = p ? p : ""; + if (p) parakeet_capi_free_string(p); + return s; +} +static bool same_bits(const std::vector& a, const std::vector& b) { + return !a.empty() && a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0; +} +static int count_entries(const std::string& dir) { + int n = 0; + if (DIR* d = opendir(dir.c_str())) { + while (dirent* e = readdir(d)) if (e->d_name[0] != '.') ++n; + closedir(d); + } + return n; +} + +// Results of the component-by-component comparison, with the loaders told how to open. +struct Results { + std::string transcript, diar_json, named_json, id; + std::vector ced, emb, sil; +}; + +int main() { + const char* bundle = std::getenv("PARAKEET_TEST_BUNDLE_FULL"); + const char *fa = std::getenv("PARAKEET_TEST_FULL_ASR"), *fd = std::getenv("PARAKEET_TEST_FULL_DIAR"), + *fc = std::getenv("PARAKEET_TEST_FULL_CED"), *fv = std::getenv("PARAKEET_TEST_FULL_VOICE"), + *fs = std::getenv("PARAKEET_TEST_FULL_SILERO"); + if (!bundle || !fa || !fd || !fc || !fv || !fs) { + std::puts("skip: PARAKEET_TEST_BUNDLE_FULL and PARAKEET_TEST_FULL_{ASR,DIAR,CED,VOICE,SILERO} must be set"); + return 77; + } + if (!pk::CedTagger::available() || !pk::SpeakerEncoder::available()) { + std::puts("skip: built without CED or voice-detect"); + return 77; + } + const std::string speech = "tests/fixtures/speech.wav", two = "tests/fixtures/two_speakers.wav"; + pk::Audio a_speech, a_two; + CHECK(pk::load_audio_16k_mono(speech, a_speech) && pk::load_audio_16k_mono(two, a_two)); + struct stat st{}; + CHECK(::stat(bundle, &st) == 0); + + // ---- header: five components, the licences of record ---- + pk::BundleInfo info; + std::string err; + CHECK(pk::read_bundle_info(bundle, info, &err)); + const char* want[][3] = {{"asr", "asr", "CC-BY-4.0"}, {"diar", "diar", "OpenMDW-1.1"}, {"ced", "ced", "Apache-2.0"}, + {"voice", "voice", ""}, {"vad", "vad", "MIT"}}; + for (auto& w : want) { + const pk::BundleComponent* c = info.find(w[0]); + CHECK(c && c->kind == w[1] && !c->license.empty() && !c->attribution.empty() && !c->source_sha256.empty()); + if (c && w[2][0]) CHECK(c->license == w[2]); + } + const pk::BundleComponent* vc = info.find("voice"); + CHECK(vc && (vc->license == "CC-BY-4.0" || vc->license == "Apache-2.0")); + + // ---- selection ---- + { + parakeet_ctx* d = parakeet_capi_load(bundle); + CHECK(d && parakeet_capi_model_kind(d) == PARAKEET_MODEL_KIND_ASR); // plain load picks the ASR component + parakeet_capi_free(d); + struct { const char* n; int kind; } k[] = {{"asr", PARAKEET_MODEL_KIND_ASR}, {"diar", PARAKEET_MODEL_KIND_DIARIZATION}, + {"ced", PARAKEET_MODEL_KIND_SOUND}, {"voice", PARAKEET_MODEL_KIND_SPEAKER}, {"vad", PARAKEET_MODEL_KIND_VAD}}; + for (auto& e : k) { + parakeet_ctx* c = parakeet_capi_load_component(bundle, e.n); + CHECK(c && parakeet_capi_model_kind(c) == e.kind); + parakeet_capi_free(c); + } + CHECK(!pk::Model::load(bundle)); // a bundle needs a component + CHECK(!pk::Model::load(bundle, "diar")); // wrong kind for the ASR loader + CHECK(!pk::DiarizationModel::load(bundle, "asr")); // and for the diarization loader + // The unmodified third-party loaders (what an older parakeet build calls) refuse the bundle. + CHECK(!pk::CedTagger::load(bundle)); + CHECK(!pk::SpeakerEncoder::load(bundle)); + CHECK(!pk::CedTagger::load(bundle, "voice")); + CHECK(!pk::SpeakerEncoder::load(bundle, "ced")); + CHECK(!pk::SpeakerEncoder::load(bundle, "nope")); + const std::string js = take(parakeet_capi_bundle_components_json(bundle)); + for (auto& w : want) CHECK(js.find(std::string("\"kind\":\"") + w[1] + "\"") != std::string::npos); + } + + // ---- standalone results ---- + Results solo; + { + auto m = pk::Model::load(fa); + CHECK(m != nullptr); + if (m) solo.transcript = m->transcribe_path(speech); + parakeet_ctx* d = parakeet_capi_load(fd); + CHECK(d != nullptr); + if (d) solo.diar_json = take(parakeet_capi_diarize_path(d, two.c_str())); + auto t = pk::CedTagger::load(fc); + CHECK(t != nullptr); + if (t) { pk::SoundScorer sc = t->scorer(); CHECK(sc(a_speech.samples.data(), (int)a_speech.samples.size(), solo.ced)); } + auto e = pk::SpeakerEncoder::load(fv); + CHECK(e != nullptr); + if (e) CHECK(e->embed(a_speech.samples.data(), (int)a_speech.samples.size(), solo.emb)); + parakeet_ctx* sp = parakeet_capi_load(fv); + CHECK(sp != nullptr); + if (sp && d) { + parakeet_speaker_registry* reg = parakeet_capi_speaker_registry_new(); + solo.named_json = take(parakeet_capi_diarize_named_pcm_json(d, sp, reg, a_two.samples.data(), + (int)a_two.samples.size(), 16000, 0.5f, 0.05f)); + parakeet_capi_speaker_registry_free(reg); + solo.id = parakeet_capi_speaker_identity(sp) ? parakeet_capi_speaker_identity(sp) : ""; + } + std::string se; + auto s = pk::SileroVad::load(fs, &se); + CHECK(s != nullptr); + if (s) solo.sil = s->probabilities(a_speech.samples.data(), a_speech.samples.size(), 16000); + parakeet_capi_free(d); parakeet_capi_free(sp); + } + CHECK(!solo.transcript.empty() && !solo.diar_json.empty() && !solo.ced.empty() && !solo.emb.empty() && !solo.sil.empty()); + CHECK(!solo.named_json.empty() && solo.id.rfind("sha256:", 0) == 0); + std::printf("transcript: %s\ndiarization: %s\n", solo.transcript.c_str(), solo.diar_json.c_str()); + + // ---- the same through the bundle, with the in-memory file and with the temporary-file fallback ---- + char tmpl[] = "/tmp/pk_bundle_full_XXXXXX"; + const std::string tmpdir = ::mkdtemp(tmpl) ? tmpl : ""; + CHECK(!tmpdir.empty()); + for (int pass = 0; pass < 2; ++pass) { + if (pass == 1) { + ::setenv("PARAKEET_BUNDLE_NO_MEMFD", "1", 1); + ::setenv("TMPDIR", tmpdir.c_str(), 1); + } + std::printf("-- pass %d (%s)\n", pass, pass ? "temporary file" : "memfd"); + { + auto cf = pk::ComponentFile::create(bundle, "ced", &err); + CHECK(cf && cf->in_memory() == (pass == 0)); + if (cf && pass == 1) CHECK(count_entries(tmpdir) == 1); + } + CHECK(count_entries(tmpdir) == 0); // the temporary file is gone once the object is + + auto m = pk::Model::load(bundle, "asr"); + CHECK(m != nullptr); + if (m) CHECK(m->transcribe_path(speech) == solo.transcript); + + parakeet_ctx* d = parakeet_capi_load_component(bundle, "diar"); + CHECK(d != nullptr); + if (d) CHECK(take(parakeet_capi_diarize_path(d, two.c_str())) == solo.diar_json); + + const unsigned long long r0 = bytes_read(); + std::string le; + auto t = pk::CedTagger::load(bundle, "ced", &le); + const unsigned long long r1 = bytes_read(); + CHECK(t != nullptr); + const pk::BundleComponent* cc = info.find("ced"); + if (t) { + std::vector p; + pk::SoundScorer sc = t->scorer(); + CHECK(sc(a_speech.samples.data(), (int)a_speech.samples.size(), p) && same_bits(p, solo.ced)); + if (r1 > r0 && cc) { + std::printf("ced: read %.1f MB (component %.1f MB, bundle %.1f MB)\n", (double)(r1 - r0) / 1e6, + (double)cc->n_bytes / 1e6, (double)st.st_size / 1e6); + CHECK(r1 - r0 < 3 * cc->n_bytes + (8u << 20)); // copy into the file, then the loader reads it + CHECK(r1 - r0 < (unsigned long long)st.st_size / 2); + } + } + const unsigned long long q0 = bytes_read(); + auto e = pk::SpeakerEncoder::load(bundle, "voice", &le); + const unsigned long long q1 = bytes_read(); + CHECK(e != nullptr); + if (e) { + std::vector v; + CHECK(e->embed(a_speech.samples.data(), (int)a_speech.samples.size(), v) && same_bits(v, solo.emb)); + if (q1 > q0 && vc) CHECK(q1 - q0 < 3 * vc->n_bytes + (8u << 20)); + } + parakeet_ctx* sp = parakeet_capi_load_component(bundle, "voice"); + CHECK(sp != nullptr); + if (sp && d) { + CHECK(std::string(parakeet_capi_speaker_identity(sp)) == solo.id); // same identity as the standalone file + parakeet_speaker_registry* reg = parakeet_capi_speaker_registry_new(); + CHECK(take(parakeet_capi_diarize_named_pcm_json(d, sp, reg, a_two.samples.data(), (int)a_two.samples.size(), + 16000, 0.5f, 0.05f)) == solo.named_json); + parakeet_capi_speaker_registry_free(reg); + } + std::string se; + auto s = pk::SileroVad::load(bundle, &se, "vad"); + CHECK(s != nullptr); + if (s) CHECK(same_bits(s->probabilities(a_speech.samples.data(), a_speech.samples.size(), 16000), solo.sil)); + // C-API sound context from the bundle + parakeet_ctx* tg = parakeet_capi_load_component(bundle, "ced"); + CHECK(tg && parakeet_capi_num_classes(tg) > 0 && parakeet_capi_class_label(tg, 0)); + parakeet_capi_free(tg); parakeet_capi_free(d); parakeet_capi_free(sp); + CHECK(count_entries(tmpdir) == 0); + } + ::unsetenv("PARAKEET_BUNDLE_NO_MEMFD"); + ::rmdir(tmpdir.c_str()); + + if (failures) return 1; + std::printf("test_bundle_full OK\n"); + return 0; +} From 18494fa01988f548002b2d2c0f247c7629d2b6d8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 4 Oct 2026 11:30:33 +0000 Subject: [PATCH 9/9] docs: correct bundle licence sources and the NOTICE preamble The voice component is converted from the WeSpeaker ResNet34-LM model, so name that source, credit the WeSpeaker project and keep CC-BY-4.0. Name the ModelScope source of ERes2Net. Record that the licence of the CED weights is not consistent upstream (model cards say Apache-2.0, the code repository is GPL-3.0, the checkpoint records say CC-BY-4.0) and should be confirmed with the authors. Add a note on VoxCeleb training data and suggested credit strings for the voice encoders. The NOTICE preamble no longer says the weights are unchanged. It says each component is copied byte for byte from its converted file and that the Changes line states any conversion or quantisation. Assisted-by: Claude:claude-sonnet-5-5 [Claude Code] --- docs/bundle.md | 37 ++++++++++++++++++++++++++++++------ scripts/bundle_gguf.py | 10 ++++++---- tests/python/check_bundle.py | 13 +++++++------ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/docs/bundle.md b/docs/bundle.md index 512080c..a2dba41 100644 --- a/docs/bundle.md +++ b/docs/bundle.md @@ -310,14 +310,39 @@ is a collection, so it does not have one licence. Rules the tools apply: | diar | `nvidia/Nemotron-3-Diarization` | OpenMDW-1.1 | a copy of the agreement and the origin notices | | ced | `mispeech/ced-*` | Apache-2.0 | licence text and copyright notices | | voice, ECAPA | `speechbrain/spkrec-ecapa-voxceleb` | Apache-2.0 | licence text and notices | -| voice, ERes2Net | 3D-Speaker | Apache-2.0 | licence text and notices | -| voice, WeSpeaker ResNet34 | `Wespeaker/wespeaker-voxceleb-resnet34` | CC-BY-4.0 (see below) | credit, licence link, note of changes | +| voice, ERes2Net | `iic/speech_eres2net_base_200k_sv_zh-cn_16k-common` (3D-Speaker, ModelScope) | Apache-2.0 | licence text and notices | +| voice, WeSpeaker ResNet34 | `Wespeaker/wespeaker-voxceleb-resnet34-LM` | CC-BY-4.0 (see below) | credit to the WeSpeaker project, licence link, note of changes | | vad | `snakers4/silero-vad` | MIT | copyright notice and licence text | -WeSpeaker: Hugging Face lists Apache-2.0 for the plain model and CC-BY-4.0 for the -`-LM` variant, and the voice-detect GGUF card says CC-BY-4.0. The table in the -script uses CC-BY-4.0, the stricter reading, until the upstream licence is -confirmed. Not bundled: the audeering heads (CC-BY-NC-SA-4.0) and the EOU model. +WeSpeaker: the voice-detect.cpp file is converted from `voxceleb_resnet34_LM.onnx` of +`Wespeaker/wespeaker-voxceleb-resnet34-LM`, and that card says CC-BY-4.0. The card of the +plain `Wespeaker/wespeaker-voxceleb-resnet34` says Apache-2.0, but the WeSpeaker project +states in its `docs/pretrained.md` that its VoxCeleb-trained models follow CC-BY-4.0. +The bundle uses CC-BY-4.0 and credits the WeSpeaker project. + +CED: the `mispeech/ced-*` cards say Apache-2.0, and the bundle follows them. The upstream +GitHub code repository is GPL-3.0, and the original checkpoint records on Zenodo say +CC-BY-4.0. The licence of the weights is therefore not consistent upstream. Confirm it with +the authors before a public release. The CC-BY credit to Heinrich Dinkel and Xiaomi +(mispeech) is kept in the meantime. + +Not bundled: the audeering heads (CC-BY-NC-SA-4.0) and the EOU model. + +### Training data + +The speaker models ECAPA and WeSpeaker are trained on VoxCeleb. They carry the terms that +their authors state for the model (WeSpeaker: CC-BY-4.0). Whether a trained model is +derived from its training data is a legal question that this project does not settle. + +### Credit strings + +Suggested `source`, `attribution` and `changes` values for the voice encoders: + +| Model | Source | Credit | Changes | +|---|---|---|---| +| WeSpeaker ResNet34-LM | `https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet34-LM` | WeSpeaker ResNet34-LM (large-margin fine-tuned, trained on VoxCeleb2 dev) by the WeSpeaker project (https://github.com/wenet-e2e/wespeaker), licensed under CC-BY-4.0 | Converted from the ONNX export to GGUF (voice-detect.cpp, F32); no other change | +| ECAPA-TDNN | `https://huggingface.co/speechbrain/spkrec-ecapa-voxceleb` | ECAPA-TDNN speaker encoder by the SpeechBrain authors (speechbrain/spkrec-ecapa-voxceleb), trained on VoxCeleb 1 and 2, licensed under Apache-2.0 | Converted from the SpeechBrain embedding model checkpoint to GGUF (voice-detect.cpp); no other change | +| ERes2Net | `https://www.modelscope.cn/models/iic/speech_eres2net_base_200k_sv_zh-cn_16k-common` | ERes2Net-base (200k speakers, zh-cn) by the 3D-Speaker authors (https://github.com/modelscope/3D-Speaker), licensed under Apache-2.0 | Converted from the sherpa-onnx ONNX export to GGUF (voice-detect.cpp, F32); no other change | ## C-API and CLI diff --git a/scripts/bundle_gguf.py b/scripts/bundle_gguf.py index 3b9316c..123db34 100644 --- a/scripts/bundle_gguf.py +++ b/scripts/bundle_gguf.py @@ -55,8 +55,9 @@ # Licence of each known source, as agreed for bundling. A manifest that names one of these # sources with another licence is refused, so a wrong or silent change cannot slip in. -# WeSpeaker: Hugging Face says Apache-2.0 for the plain model and CC-BY-4.0 for the -LM -# variant, and the voice-detect GGUF card says CC-BY-4.0. The stricter reading is used. +# WeSpeaker: the converted file is voxceleb_resnet34_LM.onnx (Wespeaker/wespeaker-voxceleb-resnet34-LM, +# CC-BY-4.0). The WeSpeaker project says its VoxCeleb models follow CC-BY-4.0 (docs/pretrained.md), although +# the card of the plain model says Apache-2.0. KNOWN_SOURCES = ( ("nemotron-3-diarization", "OpenMDW-1.1"), ("mispeech/ced", "Apache-2.0"), @@ -472,8 +473,9 @@ def notice(path): "", f"{str_field(r, 'general.name')} is a single file that holds several machine-learning models.", "Each model keeps its own licence and attribution, listed below. They are also stored in the", - "file header under parakeet.bundle..*. The models were converted to GGUF and", - "merged without further changes to their weights, except where a component says otherwise.", + "file header under parakeet.bundle..*. Each component is copied byte for byte from", + "its converted single-model GGUF file. The Changes line of each component states any conversion", + "or quantisation of the weights.", "The full text of every licence used follows the component list. Ship this file with the bundle.", "", ] diff --git a/tests/python/check_bundle.py b/tests/python/check_bundle.py index 8ee6583..a5317f8 100644 --- a/tests/python/check_bundle.py +++ b/tests/python/check_bundle.py @@ -95,7 +95,7 @@ def write_ced(path): w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() -def write_voice(path, arch="voicedetect", embedding=True, name="Wespeaker/wespeaker-voxceleb-resnet34"): +def write_voice(path, arch="voicedetect", embedding=True, name="Wespeaker/wespeaker-voxceleb-resnet34-LM"): w = GGUFWriter(path, arch) w.add_string("general.name", name) w.add_string("voicedetect.arch", "wespeaker_resnet34") @@ -123,10 +123,10 @@ def write_vad(path, license="MIT"): "source": "nvidia/Nemotron-3-Diarization", "attribution": "Nemotron 3 Diarization by NVIDIA", "changes": "Converted to GGUF and quantised to Q8_0"} CED_META = {"kind": "ced", "license": "Apache-2.0", "license_url": "https://www.apache.org/licenses/LICENSE-2.0", - "source": "mispeech/ced-tiny", "attribution": "CED by Xiaomi (mispeech)", "changes": "Converted to GGUF"} + "source": "mispeech/ced-tiny", "attribution": "CED-small by Heinrich Dinkel et al., Xiaomi (mispeech)", "changes": "Converted to GGUF"} VOICE_META = {"kind": "voice", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", - "source": "Wespeaker/wespeaker-voxceleb-resnet34", "attribution": "WeSpeaker ResNet34", - "changes": "Converted from ONNX to GGUF"} + "source": "Wespeaker/wespeaker-voxceleb-resnet34-LM", + "attribution": "WeSpeaker ResNet34-LM by the WeSpeaker project", "changes": "Converted from ONNX to GGUF"} ASR_META = {"kind": "asr", "license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "source": "nvidia/stand-in", "attribution": "Stand-in model by NVIDIA", "changes": "Converted to GGUF"} VAD_META = {"kind": "vad", "license": "MIT", "license_url": "https://github.com/snakers4/silero-vad/blob/master/LICENSE", @@ -216,7 +216,8 @@ def main(): # NOTICE: every component's credit and the full text of each distinct licence rc, out, err = run("--notice", "bf.gguf") check(rc == 0, f"notice failed: {err}") - for needle in ("Nemotron 3 Diarization by NVIDIA", "CED by Xiaomi (mispeech)", "WeSpeaker ResNet34", + for needle in ("Nemotron 3 Diarization by NVIDIA", "CED-small by Heinrich Dinkel et al., Xiaomi (mispeech)", + "WeSpeaker ResNet34-LM by the WeSpeaker project", "Copyright (c) 2020-present Silero Team", "NVIDIA", "OpenMDW License Agreement, version 1.1", "Apache License", "Attribution 4.0 International", "Permission is hereby granted", "you shall retain in your distribution"): @@ -251,7 +252,7 @@ def refuse2(label, comps, needle): "must not be bundled") refuse2("diar with a wrong licence", [comp("diar", "diar.gguf", dict(DIAR_META, license="CC-BY-4.0"))], "is licensed OpenMDW-1.1") refuse2("ced with a wrong licence", [comp("ced", "ced.gguf", dict(CED_META, license="MIT"))], "is licensed Apache-2.0") - refuse2("wespeaker as Apache-2.0 (the stricter reading is required)", + refuse2("wespeaker as Apache-2.0 (CC-BY-4.0 is required)", [comp("voice", "voice.gguf", dict(VOICE_META, license="Apache-2.0"))], "is licensed CC-BY-4.0") # --- a tampered bundle fails verify ---