From a8f66e7ac980e1c6e82ca36b020df02b6a425b7b Mon Sep 17 00:00:00 2001 From: Mathieu Gosbee Date: Tue, 20 Oct 2026 01:13:55 -0400 Subject: [PATCH 01/59] Enable Bevy experimental_pbr_pcss for soft directional shadows The avatar editor's studio key light uses PCSS (percentage-closer soft shadows) to match Character Creator 5's soft shadowing. Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4ca45ab..124aef0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,8 @@ tracing = "0.1" # explicitly carry that default set here so the standalone-workspace # build doesn't drop `x11`, `multi_threaded`, etc. and break under # winit / wgpu. -bevy = { version = "0.15", features = ["jpeg", "tonemapping_luts", "ktx2", "zstd"] } +# experimental_pbr_pcss: soft (PCSS) directional shadows for the avatar editor's studio key light. +bevy = { version = "0.15", features = ["jpeg", "tonemapping_luts", "ktx2", "zstd", "experimental_pbr_pcss"] } # bevy_rapier3d shipped unconditionally — no nested cargo features. bevy_rapier3d = { version = "=0.28.0", default-features = false, features = ["dim3", "async-collider"] } # wgpu pinned to the version Bevy 0.15.3 bundles. Aliased to `wgpu_pin` From 0fb2df55e450af3320b29fd03f73290e438f9cf4 Mon Sep 17 00:00:00 2001 From: Mathieu Gosbee Date: Tue, 20 Oct 2026 01:30:08 -0400 Subject: [PATCH 02/59] Add CC5 avatar editor, cc5-shape crate and CC5 shader materials - crates/cc5-shape: Character Creator 5 shaping sliders outside CC5 (Rust + CPython module): slider pack evaluation, shaped GLB writing, CPU facial expressions (all ~480 CC5 morphs + ARKit mapping), and HD (subdivided) character support via a level-0 -> HD stencil map (SubdivMap, shape_glb_hd); Python fitting helpers in cc5_shape.fit. - examples/avatar_editor.rs: live CC5 slider editor in Bevy with recipes, style layers, expressions, --hd for CC5's HD mesh, snapshot mode, and a studio look calibrated against CC5's viewport. - examples/shape_avatar.rs: headless weights JSON -> shaped GLB. - src/cc_render/cc5_materials: Cc5MaterialsPlugin builds Bevy extended materials for CC5's RLHead/RLSkin, RLHair, RLEye, eye occlusion/tearline and teeth/tongue shaders from the maps and parameters glb_cc5_materials.py records in glTF material extras. - docs/design/avatar-editor.md: pipeline, pack format and accuracy notes. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + Cargo.toml | 8 + crates/cc5-shape/Cargo.toml | 25 + crates/cc5-shape/README.md | 98 ++ crates/cc5-shape/pyproject.toml | 14 + crates/cc5-shape/python/cc5_shape/__init__.py | 5 + .../cc5-shape/python/cc5_shape/__init__.pyi | 70 + crates/cc5-shape/python/cc5_shape/fit.py | 426 +++++++ crates/cc5-shape/python/cc5_shape/py.typed | 0 crates/cc5-shape/src/expr.rs | 300 +++++ crates/cc5-shape/src/glb.rs | 336 +++++ crates/cc5-shape/src/lib.rs | 24 + crates/cc5-shape/src/pack.rs | 396 ++++++ crates/cc5-shape/src/python.rs | 359 ++++++ crates/cc5-shape/src/subdiv.rs | 212 ++++ docs/design/avatar-editor.md | 148 +++ examples/avatar_editor.rs | 1123 +++++++++++++++++ examples/shape_avatar.rs | 22 + src/cc_render/cc5_materials/common.wgsl | 70 + src/cc_render/cc5_materials/eye.wgsl | 224 ++++ src/cc_render/cc5_materials/eye_overlay.wgsl | 122 ++ src/cc_render/cc5_materials/hair.wgsl | 218 ++++ src/cc_render/cc5_materials/hair_prepass.wgsl | 69 + src/cc_render/cc5_materials/hair_types.wgsl | 91 ++ src/cc_render/cc5_materials/mod.rs | 1044 +++++++++++++++ src/cc_render/cc5_materials/mouth.wgsl | 103 ++ src/cc_render/cc5_materials/skin.wgsl | 257 ++++ src/cc_render/mod.rs | 3 + 28 files changed, 5768 insertions(+) create mode 100644 crates/cc5-shape/Cargo.toml create mode 100644 crates/cc5-shape/README.md create mode 100644 crates/cc5-shape/pyproject.toml create mode 100644 crates/cc5-shape/python/cc5_shape/__init__.py create mode 100644 crates/cc5-shape/python/cc5_shape/__init__.pyi create mode 100644 crates/cc5-shape/python/cc5_shape/fit.py create mode 100644 crates/cc5-shape/python/cc5_shape/py.typed create mode 100644 crates/cc5-shape/src/expr.rs create mode 100644 crates/cc5-shape/src/glb.rs create mode 100644 crates/cc5-shape/src/lib.rs create mode 100644 crates/cc5-shape/src/pack.rs create mode 100644 crates/cc5-shape/src/python.rs create mode 100644 crates/cc5-shape/src/subdiv.rs create mode 100644 docs/design/avatar-editor.md create mode 100644 examples/avatar_editor.rs create mode 100644 examples/shape_avatar.rs create mode 100644 src/cc_render/cc5_materials/common.wgsl create mode 100644 src/cc_render/cc5_materials/eye.wgsl create mode 100644 src/cc_render/cc5_materials/eye_overlay.wgsl create mode 100644 src/cc_render/cc5_materials/hair.wgsl create mode 100644 src/cc_render/cc5_materials/hair_prepass.wgsl create mode 100644 src/cc_render/cc5_materials/hair_types.wgsl create mode 100644 src/cc_render/cc5_materials/mod.rs create mode 100644 src/cc_render/cc5_materials/mouth.wgsl create mode 100644 src/cc_render/cc5_materials/skin.wgsl diff --git a/.gitignore b/.gitignore index 7c765e7..479ca98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target/ Cargo.lock *.bk +target-*/ diff --git a/Cargo.toml b/Cargo.toml index 124aef0..df94727 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ publish = false # bevy / wgpu / bevy_rapier3d deps don't unify against the host's, and # heavy renderer stack compiles in isolation. [workspace] +# `cc5-shape` (CC5 shaping sliders; Rust + Python) is the only member crate. +members = ["crates/cc5-shape"] [profile.release] opt-level = 3 @@ -67,3 +69,9 @@ crossbeam-channel = "0.5" # a few regexes — bring in `regex` directly. regex = "1" tempfile = "3.27.0" +# CC5 shaping sliders (avatar editor); also a standalone crate + Python module. +cc5-shape = { path = "crates/cc5-shape" } + +[dev-dependencies] +# `examples/avatar_editor.rs` UI; 0.32 is the Bevy 0.15 line. +bevy_egui = "0.32" diff --git a/crates/cc5-shape/Cargo.toml b/crates/cc5-shape/Cargo.toml new file mode 100644 index 0000000..1382698 --- /dev/null +++ b/crates/cc5-shape/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "cc5-shape" +version = "0.1.0" +edition = "2021" +publish = false +description = "Apply Character Creator 5 shaping sliders to a character GLB without CC5 (Rust core + optional CPython bindings)." + +[lib] +name = "cc5_shape" +crate-type = ["rlib", "cdylib"] + +[features] +# CPython extension module `cc5_shape` (build with maturin; see pyproject.toml). +python = ["dep:pyo3"] + +[dependencies] +anyhow = "1" +# 0.29 = the glam Bevy 0.15 uses, so cc-render's types unify with ours. +glam = "0.29" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +pyo3 = { version = "0.26", features = ["abi3-py310", "extension-module"], optional = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/cc5-shape/README.md b/crates/cc5-shape/README.md new file mode 100644 index 0000000..2e8bcef --- /dev/null +++ b/crates/cc5-shape/README.md @@ -0,0 +1,98 @@ +# cc5-shape + +Apply Character Creator 5 shaping sliders (body type, height, face features — the +Morphs panel) to a character GLB without CC5. One Rust implementation, usable from +Rust and from CPython. + +Inputs, produced once per character by +`remotemedia-sdk/scripts/avatars/cc5_editor/build_cc5_editor_pack.ps1`: + +- `character.glb` — the character's reference export; every vertex carries a + `_CCIDX` attribute (its CC5 vertex index); +- `pack/` — `pack.json` + `pack.bin`, what each slider does to that character. + +See `../../docs/design/avatar-editor.md` for how the pack is captured and how +accurate it is. + +## Rust + +```toml +# from another repository +cc5-shape = { git = "https://github.com/RemoteMedia-SDK/cc-render", package = "cc5-shape" } +``` + +```rust +let pack = cc5_shape::ShapePack::load("pack".as_ref())?; +let weights = pack.weights_from([("Character Height", 0.5), ("Body Fat A", 0.3)])?; +let delta = pack.evaluate(&weights); // per-vertex / per-joint deltas, glTF frame (m) +let shaped = cc5_shape::shape_glb(&std::fs::read("character.glb")?, &pack, &delta)?; +std::fs::write("shaped.glb", shaped)?; +``` + +Sliders are found by id or by display name (an error if a name is ambiguous — +use the id then); values are range-checked. + +Facial expressions (CC5's ~400 morph targets; Bevy loads at most 64 per mesh) +come as an expression pack (`expressions.json` + `expressions.bin`, built by +`build_expression_pack.py` from the reference FBX) in the same vertex space, for +CPU morphing on top of the shape: + +```rust +let expr = cc5_shape::ExpressionPack::load("expressions".as_ref())?; +let mut w = expr.weights_from([("Eye_Blink_L", 1.0), ("Jaw_Open", 0.3)])?; +let arkit = cc5_shape::ArkitMap::load("character.arkit_map.json".as_ref(), &expr)?; +w.extend(arkit.weights([("mouthSmileLeft", 0.6)])?); +expr.add_to(&w, &mut delta.verts); // layer on a ShapeDelta (or expr.evaluate(&w)) +``` + +HD characters: CC5 renders HD characters subdivided (level 1 = 4x faces, level 2 = +16x) with sculpted detail. `build_hd_map.py` (or `build_cc5_editor_pack.ps1 -HdLevel 1`) +writes a map from the pack's level-0 space to CC5's HD FBX export, so the same pack +shapes the HD mesh: + +```rust +let hd = cc5_shape::SubdivMap::load("hd/map".as_ref())?; +hd.check(&pack)?; +let hd_delta = hd.expand(&pack.evaluate(&weights)); +let shaped = cc5_shape::shape_glb_hd(&std::fs::read("hd/character_hd.glb")?, &pack, &hd, &hd_delta)?; +``` + +Exact for sliders that only reshape the base mesh (p99 within 0.02 mm of CC5's own +HD export). HD presets and large body sliders also change the HD level directly, +which a level-0 pack cannot see: p99 up to ~0.8 mm, max ~3 mm. + +## Python + +```sh +cd crates/cc5-shape +maturin build --release # wheel: target/wheels/cc5_shape-*-cp310-abi3-*.whl +pip install ../../target/wheels/cc5_shape-*.whl +``` + +```python +import cc5_shape + +pack = cc5_shape.Pack("pack") +w = pack.weights({"Character Height": 0.5, "Body Fat A": 0.3}) +shaped = pack.shape_glb(open("character.glb", "rb").read(), w) + +verts, joints = pack.evaluate(w) # bytes: f32 xyz per vertex; {joint: (x, y, z)} +# numpy.frombuffer(verts, "=1.5,<2"] +build-backend = "maturin" + +[tool.maturin] +features = ["python"] +module-name = "cc5_shape" +python-source = "python" diff --git a/crates/cc5-shape/python/cc5_shape/__init__.py b/crates/cc5-shape/python/cc5_shape/__init__.py new file mode 100644 index 0000000..b728350 --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/__init__.py @@ -0,0 +1,5 @@ +"""CC5 shaping sliders outside CC5 (Rust core; see Pack), CC5 facial expressions (Expressions) +and CC5 HD (subdivided) meshes (SubdivMap).""" +from .cc5_shape import ArkitMap, Expressions, Pack, SubdivMap + +__all__ = ["ArkitMap", "Expressions", "Pack", "SubdivMap"] diff --git a/crates/cc5-shape/python/cc5_shape/__init__.pyi b/crates/cc5-shape/python/cc5_shape/__init__.pyi new file mode 100644 index 0000000..5b2aed4 --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/__init__.pyi @@ -0,0 +1,70 @@ +from os import PathLike +from typing import TypedDict + +class Slider(TypedDict): + id: str + name: str + category: str + min: float + max: float + base: float + +class Pack: + """A CC5 slider pack (`pack.json` + `pack.bin`).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def vertex_count(self) -> int: ... + @property + def joints(self) -> list[str]: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: ... + @property + def sliders(self) -> list[Slider]: ... + def categories(self) -> list[tuple[str, list[int]]]: ... + def find(self, key: str) -> int: ... + def base_weights(self) -> list[float]: ... + def weights(self, set: dict[str, float] | None = None) -> list[float]: ... + def evaluate(self, weights: list[float]) -> tuple[bytes, dict[str, tuple[float, float, float]]]: ... + def recipe(self, weights: list[float]) -> str: ... + def weights_from_recipe(self, recipe: str) -> list[float]: ... + def state(self, i: int, which: str) -> tuple[float, bytes, bytes, bytes, bytes] | None: ... + def shape_glb(self, glb: bytes, weights: list[float], hd: SubdivMap | None = None) -> bytes: + """Shaped copy of a GLB carrying `_CCIDX`; with `hd`, of the CC5 HD export.""" + +class Expressions: + """A CC5 expression pack (`expressions.json` + `expressions.bin`): facial morphs + as sparse vertex deltas in the slider pack's global vertex space (glTF frame, m).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def vertex_count(self) -> int: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: ... + @property + def names(self) -> list[str]: ... + def find(self, name: str) -> int: ... + def meshes_of(self, name: str) -> list[str]: ... + def deltas(self, name: str) -> tuple[bytes, bytes]: ... + def evaluate(self, weights: dict[str, float]) -> bytes: ... + def __len__(self) -> int: ... + +class ArkitMap: + """ARKit-52 blendshape weights -> expression weights (runtime ARKit map JSON).""" + def __init__(self, path: str | PathLike[str], expressions: Expressions) -> None: ... + @property + def arkit_names(self) -> list[str]: ... + @property + def missing(self) -> list[str]: ... + def weights(self, arkit: dict[str, float]) -> dict[str, float]: ... + +class SubdivMap: + """Slider pack (CC5 subdivision 0) -> CC5 HD mesh map (`hd_map.json` + `hd_map.bin`).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def level(self) -> int: ... + @property + def vertex_count(self) -> int: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: + """`[(mesh name, HD offset, HD count)]`.""" + def expand(self, verts: bytes) -> bytes: + """Base-space f32 xyz per vertex -> HD-space f32 xyz per vertex.""" diff --git a/crates/cc5-shape/python/cc5_shape/fit.py b/crates/cc5-shape/python/cc5_shape/fit.py new file mode 100644 index 0000000..cfd106b --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/fit.py @@ -0,0 +1,426 @@ +"""Conform a character to a target shape with its CC5 sliders (needs numpy + scipy). + +The pack's linear model, shape(w) = base + sum_i t_i * delta_i, turns "make our +avatar look like this mesh" into bounded least squares, iterated with closest-point +matching because the target (a scan, another character, an SMPL body) has its own +topology: + + for each iteration: + pair our vertices with the target surface (and target samples with our + nearest vertices, so the shape cannot shrink into part of the target) + solve min |A t - r|^2 + reg |t|^2 with every t_i in [0, 1] + +Each slider has up to two captured sides (towards max / towards min), so it starts as +two columns; afterwards every slider keeps its dominant side and the fit is solved +again with one column per slider, so the result is a valid slider weight vector that +`Pack.shape_glb` and the editor accept as is. + + import cc5_shape + from cc5_shape import fit + pack = cc5_shape.Pack("pack") + base = fit.base_mesh(pack, open("character.glb", "rb").read()) + target_v, target_f = fit.load_mesh("target.glb") # or .obj + result = fit.fit(pack, base, target_v, target_f, meshes=["CC_Base_Body"], + categories=["Actor/Body/", "Actor/HD Body/"]) + shaped = pack.shape_glb(open("character.glb", "rb").read(), result.weights) + +Positions are in the glTF frame (Y-up, meters). The target should stand in roughly the +same pose as the character's bind pose (CC5 A-pose). +""" +from __future__ import annotations + +import json +import struct +from dataclasses import dataclass, field + +import numpy as np + +_DT = {5120: np.int8, 5121: np.uint8, 5122: np.int16, 5123: np.uint16, 5125: np.uint32, 5126: np.float32} +_N = {"SCALAR": 1, "VEC2": 2, "VEC3": 3, "VEC4": 4, "MAT4": 16} + + +def _glb(data: bytes): + if data[:4] != b"glTF": + raise ValueError("not a GLB file") + n = struct.unpack_from(" 1 else out + return g, acc + + +def _node_worlds(g): + nodes = g.get("nodes", []) + parent = {c: i for i, n in enumerate(nodes) for c in n.get("children", [])} + + def local(n): + if "matrix" in n: + return np.array(n["matrix"], float).reshape(4, 4).T + x, y, z, w = n.get("rotation", [0, 0, 0, 1]) + R = np.array([[1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)], + [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)], + [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)]]) + M = np.eye(4) + M[:3, :3] = R * np.array(n.get("scale", [1, 1, 1])) + M[:3, 3] = n.get("translation", [0, 0, 0]) + return M + W = {} + + def world(i): + if i not in W: + W[i] = (world(parent[i]) if i in parent else np.eye(4)) @ local(nodes[i]) + return W[i] + return [world(i) for i in range(len(nodes))] + + +@dataclass +class Base: + """Our character's bind mesh in the pack's global vertex space.""" + verts: np.ndarray # (vertex_count, 3) + faces: np.ndarray # (F, 3) global vertex indices + mesh_of: np.ndarray # (vertex_count,) index into names + names: list[str] + + def mask(self, meshes=None) -> np.ndarray: + if not meshes: + return np.ones(len(self.verts), bool) + ids = [self.names.index(m) for m in meshes] + return np.isin(self.mesh_of, ids) + + +def base_mesh(pack, glb: bytes) -> Base: + """Bind mesh of the character GLB (with `_CCIDX`), in the pack's vertex space.""" + g, acc = _glb(glb) + offsets = {name: off for name, off, _ in pack.meshes} + names = [name for name, _, _ in pack.meshes] + verts = np.full((pack.vertex_count, 3), np.nan) + mesh_of = np.zeros(pack.vertex_count, int) + for name, off, cnt in pack.meshes: + mesh_of[off:off + cnt] = names.index(name) + faces = [] + for m in g["meshes"]: + if m.get("name") not in offsets: + continue + off = offsets[m["name"]] + for p in m["primitives"]: + at = p["attributes"] + if "_CCIDX" not in at: + raise ValueError("mesh %s has no _CCIDX attribute" % m["name"]) + gi = off + acc(at["_CCIDX"]).astype(np.int64) + verts[gi] = acc(at["POSITION"]) + if "indices" in p: + faces.append(gi[acc(p["indices"]).astype(np.int64)].reshape(-1, 3)) + if np.isnan(verts).any(): + raise ValueError("%d pack vertices are not in the GLB" % int(np.isnan(verts[:, 0]).sum())) + return Base(verts, np.concatenate(faces) if faces else np.zeros((0, 3), int), mesh_of, names) + + +def load_mesh(path: str, meshes=None): + """(vertices (V,3), faces (F,3)) of a .glb (all mesh primitives, node transforms + applied, Y-up meters) or .obj file. `meshes`: only glTF meshes with these names.""" + if path.lower().endswith(".obj"): + v, f = [], [] + for line in open(path, "rb"): + if line.startswith(b"v "): + v.append([float(x) for x in line.split()[1:4]]) + elif line.startswith(b"f "): + idx = [int(t.split(b"/")[0]) for t in line.split()[1:]] + idx = [i - 1 if i > 0 else len(v) + i for i in idx] + f.extend([idx[0], idx[k], idx[k + 1]] for k in range(1, len(idx) - 1)) + return np.array(v, float), np.array(f, int).reshape(-1, 3) + g, acc = _glb(open(path, "rb").read()) + worlds = _node_worlds(g) + vs, fs, n = [], [], 0 + for i, node in enumerate(g.get("nodes", [])): + if "mesh" not in node: + continue + m = g["meshes"][node["mesh"]] + if meshes and m.get("name") not in meshes: + continue + # skinned meshes are drawn in bind space; others through their node + W = np.eye(4) if "skin" in node else worlds[i] + for p in m["primitives"]: + pos = acc(p["attributes"]["POSITION"]).astype(float) + vs.append(pos @ W[:3, :3].T + W[:3, 3]) + if "indices" in p: + fs.append(acc(p["indices"]).astype(int).reshape(-1, 3) + n) + n += len(pos) + return np.concatenate(vs), (np.concatenate(fs) if fs else np.zeros((0, 3), int)) + + +def sample_surface(v, f, n, seed=0, normals=False): + """n points spread over the triangles' area (the vertices if there are no faces); + with normals=True also their unit normals (from the triangles, or estimated from + the 16 nearest points for a bare point cloud).""" + if len(f) == 0: + pts = np.asarray(v, float) + if not normals: + return pts + from scipy.spatial import cKDTree + _, nb = cKDTree(pts).query(pts, 16) + nrm = np.empty_like(pts) + for i, idx in enumerate(nb): + q = pts[idx] - pts[idx].mean(0) + nrm[i] = np.linalg.svd(q, full_matrices=False)[2][2] + return pts, nrm + rng = np.random.default_rng(seed) + a, b, c = v[f[:, 0]], v[f[:, 1]], v[f[:, 2]] + cr = np.cross(b - a, c - a) + area = np.linalg.norm(cr, axis=1) + tri = rng.choice(len(f), n, p=area / area.sum()) + u, w = rng.random((2, n)) + flip = u + w > 1 + u[flip], w[flip] = 1 - u[flip], 1 - w[flip] + pts = a[tri] + u[:, None] * (b[tri] - a[tri]) + w[:, None] * (c[tri] - a[tri]) + if not normals: + return pts + return pts, cr[tri] / np.maximum(area[tri], 1e-20)[:, None] + + +def vertex_normals(v, f): + """Area-weighted unit vertex normals.""" + nrm = np.zeros_like(v, dtype=float) + cr = np.cross(v[f[:, 1]] - v[f[:, 0]], v[f[:, 2]] - v[f[:, 0]]) + for k in range(3): + np.add.at(nrm, f[:, k], cr) + return nrm / np.maximum(np.linalg.norm(nrm, axis=1), 1e-20)[:, None] + + +@dataclass +class FitResult: + weights: list[float] # full slider weight vector (pack order) + changed: dict[str, float] # slider id -> weight, for sliders that moved + transform: np.ndarray # 4x4 applied to the target (target -> our frame) + history: list[dict] = field(default_factory=list) # per iteration: mean/p95 distance (m) + + def save_recipe(self, pack, path): + """Write the fit as a recipe (open it in the editor, or pack.weights_from_recipe).""" + with open(path, "w", encoding="utf-8") as f: + f.write(pack.recipe(self.weights)) + + +def _columns(pack, sliders, mask): + """Sparse columns over masked vertices: list of (slider index, side, rows, deltas).""" + gidx = np.flatnonzero(mask) + pos = np.full(len(mask), -1) + pos[gidx] = np.arange(len(gidx)) + cols = [] + for i in sliders: + s = pack.sliders[i] + for which in ("max", "min"): + st = pack.state(i, which) + if st is None or abs(st[0] - s["base"]) < 1e-9: + continue + vi = np.frombuffer(st[1], "= 0 + if keep.any(): + cols.append((i, which, st[0], pos[vi[keep]], vd[keep])) + return gidx, cols + + +def _dense(cols, rows, n): + A = np.zeros((n, 3, len(cols))) + for k, (_, _, _, r, d) in enumerate(cols): + A[r, :, k] = d + return A[rows] + + +def _solve(A, r, reg, lo, hi): + """min |A t - r|^2 + reg |t|^2, lo <= t <= hi, via the normal equations (fast for + many rows: the bounded problem is solved on the n x n Cholesky factor).""" + from scipy.optimize import lsq_linear + G = A.T @ A + reg * np.eye(A.shape[1]) + h = A.T @ r + L = np.linalg.cholesky(G) + y = np.linalg.solve(L, h) + return lsq_linear(L.T, y, bounds=(lo, hi), method="bvls").x + + +def target_from_cc_glb(pack, glb: bytes, meshes=("CC_Base_Body",)) -> np.ndarray: + """Per-vertex target positions (pack vertex space, NaN where absent) from another + CC character's GLB with `_CCIDX` (same CC base topology), for `fit(..., matched=...)`.""" + g, acc = _glb(glb) + offsets = {name: off for name, off, _ in pack.meshes} + out = np.full((pack.vertex_count, 3), np.nan) + for m in g["meshes"]: + if m.get("name") in offsets and m.get("name") in meshes: + for p in m["primitives"]: + at = p["attributes"] + if "_CCIDX" in at: + out[offsets[m["name"]] + acc(at["_CCIDX"]).astype(np.int64)] = acc(at["POSITION"]) + return out + + +def fit(pack, base: Base, target_v=None, target_f=None, *, meshes=("CC_Base_Body",), categories=None, + sliders=None, iterations=8, reg=1e-4, align="translate", samples=80000, stride=1, + reverse_weight=1.0, max_dist=0.05, min_normal_dot=0.5, matched=None, landmarks=None, + landmark_weight=100.0, verbose=True) -> FitResult: + """Fit slider weights so `base` matches the target surface. + + meshes our meshes that must match (default: the body skin only) + categories slider category prefixes to use (e.g. ["Actor/Body/"]); or `sliders` + as explicit slider indices; default: every slider + align "translate" (feet on the floor, centred), "similarity" (also scale), + or "none" (target already in our frame) + reg pull towards the character's own design (larger = smaller changes) + stride use every n-th of our vertices in the solve (speed) + max_dist ignore pairs further apart than this (m) + min_normal_dot ignore pairs whose surfaces face different ways (e.g. toe vs toe) + matched (vertex_count, 3) known target position per vertex (NaN = none), + e.g. target_from_cc_glb() for another CC character: solved directly, + point to point; target_v may then be None + landmarks {our global vertex index: target point (target frame)} pinning + features a surface fit lets slide (eye corners, nose tip, chin...), + weighted by landmark_weight + + Pairs are point-to-plane: only the distance along the surface normal counts, so + vertices may slide along the target surface (sampling density does not bias it). + """ + from scipy.spatial import cKDTree + target_v = None if target_v is None else np.asarray(target_v, float) + target_f = np.zeros((0, 3), int) if target_f is None else np.asarray(target_f, int) + mask = base.mask(meshes) + if sliders is None: + sliders = [i for i, s in enumerate(pack.sliders) + if categories is None or s["category"].startswith(tuple(categories))] + ours0 = base.verts[mask] + + T = np.eye(4) + if target_v is None and matched is None: + raise ValueError("give target_v (a surface) and/or matched (per-vertex targets)") + if target_v is None: + target_v = matched[mask][~np.isnan(matched[mask][:, 0])] + surface = False + else: + surface = True + if align in ("translate", "similarity"): + s = 1.0 + if align == "similarity": + s = np.ptp(ours0[:, 1]) / np.ptp(target_v[:, 1]) + c_t = np.array([np.mean(target_v[:, 0]), target_v[:, 1].min(), np.mean(target_v[:, 2])]) + c_o = np.array([np.mean(ours0[:, 0]), ours0[:, 1].min(), np.mean(ours0[:, 2])]) + T[:3, :3] *= s + T[:3, 3] = c_o - s * c_t + elif align != "none": + raise ValueError("align must be translate, similarity or none") + def to_ours(p): + return np.asarray(p, float) @ T[:3, :3].T + T[:3, 3] + if target_v is not None: + tv = to_ours(target_v) + tpts, tnrm = sample_surface(tv, target_f, samples, normals=True) + ttree = cKDTree(tpts) + + gidx, cols = _columns(pack, sliders, mask) + n = len(gidx) + rows = np.arange(0, n, stride) + A3 = _dense(cols, rows, n) # (rows, 3, C) + local = np.full(len(mask), -1) + local[gidx] = np.arange(n) + our_faces = local[base.faces[np.all(mask[base.faces], axis=1)]] + rev_pts, rev_nrm = tpts[::2], tnrm[::2] + history = [] + + def shape(tt, cc): + out = ours0.copy() + for k, (_, _, _, r, d) in enumerate(cc): + if tt[k]: + out[r] += tt[k] * d + return out + + pin_rows, pin_pts = np.zeros(0, int), np.zeros((0, 3)) + if matched is not None: + mm = to_ours(matched[gidx]) + have = ~np.isnan(mm[:, 0]) + pin_rows, pin_pts = np.flatnonzero(have), mm[have] + pin_w = np.ones(len(pin_rows)) + if landmarks: + lr = np.array([local[g] for g in landmarks], int) + if (lr < 0).any(): + raise ValueError("landmark vertices must be in the fitted meshes") + pin_rows = np.concatenate([pin_rows, lr]) + pin_pts = np.vstack([pin_pts, to_ours(np.array(list(landmarks.values()), float))]) + pin_w = np.concatenate([pin_w if matched is not None else np.zeros(0), np.full(len(lr), landmark_weight)]) + A_pin = _dense(cols, pin_rows, n) if len(pin_rows) else None + + def system(cc, tt, A, keep=None): + pins = None + if A_pin is not None: + Ap = A_pin if keep is None else A_pin[:, :, keep] + sw = np.sqrt(pin_w)[:, None, None] + pins = ((Ap * sw).reshape(-1, Ap.shape[2]), ((pin_pts - ours0[pin_rows]) * sw[:, :, 0]).ravel()) + if not surface: + cur = shape(tt, cc) + d = np.linalg.norm(cur[pin_rows] - pin_pts, axis=1) + return pins[0], pins[1], {"mean_m": float(d.mean()), "p95_m": float(np.percentile(d, 95)), "pairs": len(d)} + cur = shape(tt, cc) + onrm = vertex_normals(cur, our_faces) + cr, on = cur[rows], onrm[rows] + # forward: our vertex -> nearest target point, along the target normal + d, j = ttree.query(cr) + ok = (d < max_dist) & (np.abs(np.sum(tnrm[j] * on, 1)) > min_normal_dot) + nf = tnrm[j[ok]] + Af = np.einsum("rk,rkc->rc", nf, A[ok]) + bf = np.sum(nf * (tpts[j[ok]] - ours0[rows][ok]), 1) + # reverse: target point -> our nearest vertex, along our normal + dr, k = cKDTree(cr).query(rev_pts) + okr = (dr < max_dist) & (np.abs(np.sum(rev_nrm * on[k], 1)) > min_normal_dot) + nr = on[k[okr]] + Ar = np.einsum("rk,rkc->rc", nr, A[k[okr]]) * np.sqrt(reverse_weight) + br = np.sum(nr * (rev_pts[okr] - ours0[rows][k[okr]]), 1) * np.sqrt(reverse_weight) + plane = np.abs(np.sum(nf * (tpts[j[ok]] - cr[ok]), 1)) + stats = {"mean_m": float(plane.mean()), "p95_m": float(np.percentile(plane, 95)), + "pairs": int(ok.sum() + okr.sum())} + if pins is not None: + return np.vstack([Af, Ar, pins[0]]), np.concatenate([bf, br, pins[1]]), stats + return np.vstack([Af, Ar]), np.concatenate([bf, br]), stats + + t = np.zeros(len(cols)) + if not surface: + iterations = 1 # point to point with known pairs: linear, one solve + for it in range(iterations): + A, b, st = system(cols, t, A3) + history.append(dict(st, iteration=it)) + if verbose: + print("fit %d: distance mean %.2f mm p95 %.2f mm (%d pairs)" + % (it, st["mean_m"] * 1e3, st["p95_m"] * 1e3, st["pairs"])) + t = _solve(A, b, reg, 0.0, 1.0) + + # one side per slider: keep the dominant side, re-solve with single columns + best = {} + for k, (i, which, w, r, d) in enumerate(cols): + mag = t[k] * np.abs(d).sum() + if i not in best or mag > best[i][0]: + best[i] = (mag, k) + keep = sorted(k for _, k in best.values()) + cols1 = [cols[k] for k in keep] + A31 = A3[:, :, keep] + t1 = t[keep] + for _ in range(3 if surface else 1): + A, b, st = system(cols1, t1, A31, keep) + t1 = _solve(A, b, reg, 0.0, 1.0) + _, _, st = system(cols1, t1, A31, keep) + history.append(dict(st, iteration="final")) + if verbose: + print("fit final: distance mean %.2f mm p95 %.2f mm" % (st["mean_m"] * 1e3, st["p95_m"] * 1e3)) + + weights = pack.base_weights() + changed = {} + for tk, (i, which, w, _, _) in zip(t1, cols1): + if tk > 1e-4: + s = pack.sliders[i] + weights[i] = float(s["base"] + tk * (w - s["base"])) + changed[s["id"]] = weights[i] + return FitResult(weights, changed, T, history) diff --git a/crates/cc5-shape/python/cc5_shape/py.typed b/crates/cc5-shape/python/cc5_shape/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/crates/cc5-shape/src/expr.rs b/crates/cc5-shape/src/expr.rs new file mode 100644 index 0000000..b4368dd --- /dev/null +++ b/crates/cc5-shape/src/expr.rs @@ -0,0 +1,300 @@ +//! CC5 facial expressions (morph targets) on the CPU: load an expression pack and +//! evaluate a weighted mix into per-vertex deltas. +//! +//! A CC5 HD character has ~400 facial morphs per face mesh; Bevy loads at most 64 +//! morph targets per mesh, so they are kept out of the GLB (`fbx_to_glb.py` drops +//! them) and applied here instead, like the shaping sliders. The pack is built +//! from the reference FBX's shape keys by `build_expression_pack.py`: +//! +//! - `expressions.json` — `vertex_count` and `meshes` (as in `pack.json`), and per +//! expression its name, the meshes it moves and its delta block; +//! - `expressions.bin` — per block: `u32 index[count]` then `f32 delta[count * 3]`. +//! +//! Deltas are in the slider pack's global vertex space (`global = mesh.offset + +//! _CCIDX`), glTF frame (Y-up, meters), at weight 1. One expression covers every +//! mesh that has a morph of that name (body, brows, lashes, tear line, eye +//! occlusion, eyes, tongue), so a mix is `base + Σ wᵢ·Δᵢ` for the whole head. +//! Morph targets are linear, so this is exact; only nonzero weights cost anything. +//! +//! [`ArkitMap`] resolves ARKit-52 weights (Audio2Face) to expression weights. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +use crate::pack::{add_block, block_deltas, block_indices, PackMesh}; + +#[derive(Debug, Clone, Deserialize)] +pub struct Expression { + pub name: String, + /// Meshes this expression moves (glTF mesh names). + pub meshes: Vec, + /// `[byte_offset, count]` into `expressions.bin`. + pub verts: [u64; 2], +} + +#[derive(Debug, Deserialize)] +struct ExprJson { + version: u32, + vertex_count: usize, + meshes: Vec, + expressions: Vec, +} + +pub struct ExpressionPack { + pub vertex_count: usize, + pub meshes: Vec, + pub expressions: Vec, + by_name: HashMap, + bin: Vec, +} + +impl ExpressionPack { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("expressions.json"); + let pack: ExprJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + if pack.version != 1 { + bail!("unsupported expression pack version {}", pack.version); + } + let bin_path = dir.join("expressions.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let by_name = pack.expressions.iter().enumerate().map(|(i, e)| (e.name.clone(), i)).collect(); + let me = Self { vertex_count: pack.vertex_count, meshes: pack.meshes, expressions: pack.expressions, by_name, bin }; + me.validate()?; + Ok(me) + } + + fn validate(&self) -> Result<()> { + if self.by_name.len() != self.expressions.len() { + bail!("duplicate expression names"); + } + for e in &self.expressions { + let [off, n] = e.verts; + if (off + n * 16) as usize > self.bin.len() { + bail!("{}: block past end of expressions.bin", e.name); + } + if let Some(bad) = block_indices(&self.bin, off, n).find(|&i| i as usize >= self.vertex_count) { + bail!("{}: index {bad} out of range {}", e.name, self.vertex_count); + } + } + Ok(()) + } + + /// Number of expressions. + pub fn len(&self) -> usize { + self.expressions.len() + } + + pub fn is_empty(&self) -> bool { + self.expressions.is_empty() + } + + /// Expression names, in pack order (index = position). + pub fn names(&self) -> impl Iterator + '_ { + self.expressions.iter().map(|e| e.name.as_str()) + } + + /// Index of the expression with this name. + pub fn find(&self, name: &str) -> Option { + self.by_name.get(name).copied() + } + + /// Index-weight pairs from `{name: weight}`; unknown names are an error. + pub fn weights_from<'a>(&self, set: impl IntoIterator) -> Result> { + set.into_iter() + .map(|(k, w)| Ok((self.find(k).with_context(|| format!("no expression named {k:?}"))?, w))) + .collect() + } + + /// Sparse delta of expression `i` at weight 1: global vertex indices and deltas. + pub fn deltas(&self, i: usize) -> (Vec, Vec<[f32; 3]>) { + let [off, n] = self.expressions[i].verts; + (block_indices(&self.bin, off, n).collect(), block_deltas(&self.bin, off, n).collect()) + } + + /// `out[v] += Σ wᵢ·Δᵢ[v]` over `(expression index, weight)` pairs; zero weights + /// are skipped. `out` has one entry per global vertex (e.g. a [`ShapeDelta`]'s + /// `verts`, to layer expressions on a shaped character). + /// + /// [`ShapeDelta`]: crate::pack::ShapeDelta + pub fn add_to(&self, weights: &[(usize, f32)], out: &mut [[f32; 3]]) { + assert_eq!(out.len(), self.vertex_count, "one entry per global vertex"); + for &(i, w) in weights { + if w != 0.0 { + add_block(&self.bin, self.expressions[i].verts, w, out); + } + } + } + + /// Per global vertex delta (glTF frame, meters) for `(expression index, weight)` pairs. + pub fn evaluate(&self, weights: &[(usize, f32)]) -> Vec<[f32; 3]> { + let mut out = vec![[0.0; 3]; self.vertex_count]; + self.add_to(weights, &mut out); + out + } +} + +/// ARKit-52 blendshape names -> expression weights, from a runtime ARKit map +/// (`avatars/*.arkit_map*.json`): +/// `{"mapping": {arkit_name: [{"morph": name, "weight": scale, "meshes": [...]}]}}`. +/// Each ARKit weight drives every listed morph, scaled; `meshes` is ignored since +/// an expression already covers every mesh with that morph. +pub struct ArkitMap { + /// ARKit name -> `(expression index, scale)`. + pub mapping: HashMap>, + /// Morph names the map uses that the expression pack does not have. + pub missing: Vec, +} + +#[derive(Deserialize)] +struct ArkitTarget { + morph: String, + #[serde(default = "one")] + weight: f32, +} + +fn one() -> f32 { + 1.0 +} + +impl ArkitMap { + pub fn load(path: &Path, pack: &ExpressionPack) -> Result { + let v: serde_json::Value = serde_json::from_slice( + &std::fs::read(path).with_context(|| format!("reading {}", path.display()))?, + ) + .with_context(|| format!("parsing {}", path.display()))?; + Self::from_json(&v, pack) + } + + pub fn from_json(v: &serde_json::Value, pack: &ExpressionPack) -> Result { + let raw: HashMap> = + serde_json::from_value(v.get("mapping").context("ARKit map has no \"mapping\"")?.clone())?; + let (mut mapping, mut missing) = (HashMap::new(), Vec::new()); + for (arkit, targets) in raw { + let mut resolved = Vec::new(); + for t in targets { + match pack.find(&t.morph) { + Some(i) => resolved.push((i, t.weight)), + None => missing.push(t.morph), + } + } + mapping.insert(arkit, resolved); + } + missing.sort(); + missing.dedup(); + Ok(Self { mapping, missing }) + } + + /// Expression weights for ARKit weights (summed where morphs are shared); + /// unknown ARKit names are an error. + pub fn weights<'a>(&self, arkit: impl IntoIterator) -> Result> { + let mut acc: Vec<(usize, f32)> = Vec::new(); + for (name, w) in arkit { + let targets = self.mapping.get(name).with_context(|| format!("no ARKit blendshape {name:?} in the map"))?; + for &(i, s) in targets { + match acc.iter_mut().find(|(j, _)| *j == i) { + Some(e) => e.1 += w * s, + None => acc.push((i, w * s)), + } + } + } + Ok(acc) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(bin: &mut Vec, idx: &[u32], d: &[[f32; 3]]) -> [u64; 2] { + let off = bin.len() as u64; + for i in idx { + bin.extend_from_slice(&i.to_le_bytes()); + } + for v in d { + for c in v { + bin.extend_from_slice(&c.to_le_bytes()); + } + } + [off, idx.len() as u64] + } + + fn write_pack(dir: &Path) { + let mut bin = Vec::new(); + let blink = block(&mut bin, &[0, 3], &[[0.0, -0.01, 0.0], [0.0, -0.02, 0.0]]); + let jaw = block(&mut bin, &[1, 3], &[[0.0, -0.1, 0.0], [0.0, 0.0, 0.04]]); + let none = block(&mut bin, &[], &[]); + std::fs::write(dir.join("expressions.bin"), &bin).unwrap(); + let json = serde_json::json!({ + "version": 1, "vertex_count": 4, + "meshes": [{"name": "Body", "offset": 0, "count": 3}, {"name": "Lash", "offset": 3, "count": 1}], + "expressions": [ + {"name": "Eye_Blink_L", "meshes": ["Body", "Lash"], "verts": blink}, + {"name": "Jaw_Open", "meshes": ["Body", "Lash"], "verts": jaw}, + {"name": "V_None", "meshes": [], "verts": none} + ] + }); + std::fs::write(dir.join("expressions.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + } + + #[test] + fn evaluates_sparse_weighted_sum() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ExpressionPack::load(dir.path()).unwrap(); + assert_eq!(p.names().collect::>(), ["Eye_Blink_L", "Jaw_Open", "V_None"]); + assert_eq!(p.find("Jaw_Open"), Some(1)); + assert_eq!(p.find("Nope"), None); + assert_eq!(p.evaluate(&[]), vec![[0.0; 3]; 4]); + + let d = p.evaluate(&[(0, 1.0), (1, 0.5), (2, 1.0)]); + assert_eq!(d[0], [0.0, -0.01, 0.0]); + assert_eq!(d[1], [0.0, -0.05, 0.0]); + assert_eq!(d[2], [0.0; 3]); + assert_eq!(d[3], [0.0, -0.02, 0.02]); + + // Layered on an existing (shape) delta. + let mut out = vec![[1.0, 0.0, 0.0]; 4]; + p.add_to(&p.weights_from([("Jaw_Open", 1.0)]).unwrap(), &mut out); + assert_eq!(out[1], [1.0, -0.1, 0.0]); + assert!(p.weights_from([("Nope", 1.0)]).unwrap_err().to_string().contains("no expression")); + assert_eq!(p.deltas(0), (vec![0, 3], vec![[0.0, -0.01, 0.0], [0.0, -0.02, 0.0]])); + } + + #[test] + fn resolves_arkit_weights() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ExpressionPack::load(dir.path()).unwrap(); + let map = ArkitMap::from_json( + &serde_json::json!({"mapping": { + "eyeBlinkLeft": [{"morph": "Eye_Blink_L", "weight": 1.0, "meshes": ["Body"]}], + "jawOpen": [{"morph": "Jaw_Open", "weight": 0.5}, {"morph": "Eye_Blink_L", "weight": 0.25}], + "mouthClose": [{"morph": "Mouth_Close", "weight": 1.0}] + }}), + &p, + ) + .unwrap(); + assert_eq!(map.missing, ["Mouth_Close"]); + let w = map.weights([("eyeBlinkLeft", 0.5), ("jawOpen", 1.0), ("mouthClose", 1.0)]).unwrap(); + assert_eq!(w, vec![(0, 0.75), (1, 0.5)]); + assert!(map.weights([("tongueOut", 1.0)]).is_err()); + } + + #[test] + fn rejects_out_of_range_index() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let mut json: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("expressions.json")).unwrap()).unwrap(); + json["vertex_count"] = 3.into(); + std::fs::write(dir.path().join("expressions.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + let err = ExpressionPack::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/glb.rs b/crates/cc5-shape/src/glb.rs new file mode 100644 index 0000000..8ad42d1 --- /dev/null +++ b/crates/cc5-shape/src/glb.rs @@ -0,0 +1,336 @@ +//! Write a shaped copy of a GLB: apply a [`ShapeDelta`] to the bind mesh and skeleton. +//! +//! The GLB must carry the `_CCIDX` vertex attribute (CC5 vertex index per glTF +//! vertex, written by `fbx_to_glb.py`) so seam-split glTF vertices map back to +//! the pack's global vertex space. +//! +//! - Vertex positions: `bind + delta`, with the world-space delta taken into each +//! skinned mesh's bind space (`W(joint) · IBM(joint)`). +//! - Joints: every node's world translation moves by its pack delta, or by its +//! parent's when it has none (helpers ride along). Rotations are unchanged, so +//! a node's local translation changes by `parent_linear⁻¹ · (Δnode − Δparent)`. +//! - Inverse bind matrices: `IBM' = W'⁻¹ · W · IBM`, so the new bind mesh renders +//! exactly as written in the bind pose. + +use std::collections::HashMap; + +use anyhow::{anyhow, bail, Context, Result}; +use glam::{Mat3, Mat4, Quat, Vec3}; +use serde_json::Value; + +use crate::pack::{ShapeDelta, ShapePack}; +use crate::subdiv::SubdivMap; + +const GLB_MAGIC: u32 = 0x4654_6C67; +const CHUNK_JSON: u32 = 0x4E4F_534A; +const CHUNK_BIN: u32 = 0x004E_4942; + +struct Glb { + json: Value, + bin: Vec, +} + +fn parse(glb: &[u8]) -> Result { + let u32_at = |o: usize| -> Result { + Ok(u32::from_le_bytes(glb.get(o..o + 4).ok_or_else(|| anyhow!("truncated GLB"))?.try_into()?)) + }; + if u32_at(0)? != GLB_MAGIC { + bail!("not a GLB file"); + } + let (mut json, mut bin, mut off) = (None, None, 12usize); + while off + 8 <= glb.len() { + let (len, kind) = (u32_at(off)? as usize, u32_at(off + 4)?); + let body = glb.get(off + 8..off + 8 + len).ok_or_else(|| anyhow!("truncated GLB chunk"))?; + match kind { + CHUNK_JSON => json = Some(serde_json::from_slice(body)?), + CHUNK_BIN => bin = Some(body.to_vec()), + _ => {} + } + off += 8 + len; + } + Ok(Glb { json: json.context("GLB has no JSON chunk")?, bin: bin.context("GLB has no BIN chunk")? }) +} + +fn write(glb: &Glb) -> Result> { + let mut json = serde_json::to_vec(&glb.json)?; + while json.len() % 4 != 0 { + json.push(b' '); + } + let mut bin = glb.bin.clone(); + while bin.len() % 4 != 0 { + bin.push(0); + } + let total = 12 + 8 + json.len() + 8 + bin.len(); + let mut out = Vec::with_capacity(total); + for v in [GLB_MAGIC, 2, total as u32, json.len() as u32, CHUNK_JSON] { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(&json); + out.extend_from_slice(&(bin.len() as u32).to_le_bytes()); + out.extend_from_slice(&CHUNK_BIN.to_le_bytes()); + out.extend_from_slice(&bin); + Ok(out) +} + +/// Byte offset and stride of each element of a float accessor with `comps` components. +fn accessor_layout(json: &Value, acc: usize, comps: usize) -> Result<(usize, usize, usize)> { + let a = &json["accessors"][acc]; + if a["componentType"].as_u64() != Some(5126) { + bail!("accessor {acc} is not float"); + } + let bv = &json["bufferViews"][a["bufferView"].as_u64().context("sparse accessors unsupported")? as usize]; + let start = bv["byteOffset"].as_u64().unwrap_or(0) as usize + a["byteOffset"].as_u64().unwrap_or(0) as usize; + let stride = bv["byteStride"].as_u64().map(|s| s as usize).unwrap_or(4 * comps); + Ok((start, stride, a["count"].as_u64().context("accessor count")? as usize)) +} + +fn read_f32s(g: &Glb, acc: usize) -> Result> { + let (start, stride, count) = accessor_layout(&g.json, acc, N)?; + (0..count) + .map(|i| { + let b = start + i * stride; + let mut v = [0f32; N]; + for (k, x) in v.iter_mut().enumerate() { + let s = g.bin.get(b + 4 * k..b + 4 * k + 4).ok_or_else(|| anyhow!("accessor past BIN"))?; + *x = f32::from_le_bytes(s.try_into()?); + } + Ok(v) + }) + .collect() +} + +fn write_f32s(g: &mut Glb, acc: usize, vals: &[[f32; N]]) -> Result<()> { + let (start, stride, count) = accessor_layout(&g.json, acc, N)?; + if count != vals.len() { + bail!("accessor {acc}: {} values for {count} elements", vals.len()); + } + for (i, v) in vals.iter().enumerate() { + let b = start + i * stride; + for (k, x) in v.iter().enumerate() { + g.bin[b + 4 * k..b + 4 * k + 4].copy_from_slice(&x.to_le_bytes()); + } + } + Ok(()) +} + +fn node_local(node: &Value) -> Mat4 { + if let Some(m) = node["matrix"].as_array() { + let a: Vec = m.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect(); + return Mat4::from_cols_slice(&a); + } + let v = |k: &str, d: &[f32]| -> Vec { + node[k].as_array().map(|a| a.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect()).unwrap_or(d.to_vec()) + }; + let (t, r, s) = (v("translation", &[0.0; 3]), v("rotation", &[0.0, 0.0, 0.0, 1.0]), v("scale", &[1.0; 3])); + Mat4::from_scale_rotation_translation(Vec3::new(s[0], s[1], s[2]), Quat::from_xyzw(r[0], r[1], r[2], r[3]), Vec3::new(t[0], t[1], t[2])) +} + +/// Shaped copy of `glb` for `delta` (from `pack.evaluate`). +pub fn shape_glb(glb: &[u8], pack: &ShapePack, delta: &ShapeDelta) -> Result> { + if delta.verts.len() != pack.vertex_count { + bail!("delta has {} vertices, pack {}", delta.verts.len(), pack.vertex_count); + } + shape(glb, &pack.joints, &|n| pack.mesh_offset(n), delta) +} + +/// Shaped copy of an HD character GLB (CC5 export at `map.level`, `_CCIDX` in +/// CC5's HD vertex order) for an HD delta (`map.expand(&pack.evaluate(w))`). +pub fn shape_glb_hd(glb: &[u8], pack: &ShapePack, map: &SubdivMap, hd_delta: &ShapeDelta) -> Result> { + if hd_delta.verts.len() != map.vertex_count { + bail!("delta has {} vertices, HD map {}", hd_delta.verts.len(), map.vertex_count); + } + shape(glb, &pack.joints, &|n| map.mesh_offset(n), hd_delta) +} + +fn shape(glb: &[u8], joints: &[String], mesh_offset: &dyn Fn(&str) -> Option, delta: &ShapeDelta) -> Result> { + let mut g = parse(glb)?; + let nodes = g.json["nodes"].as_array().cloned().unwrap_or_default(); + let mut parent = vec![None; nodes.len()]; + for (i, n) in nodes.iter().enumerate() { + for c in n["children"].as_array().into_iter().flatten() { + parent[c.as_u64().context("child index")? as usize] = Some(i); + } + } + // Hierarchy order (parents first) and base world matrices. + let mut order: Vec = Vec::with_capacity(nodes.len()); + let mut placed = vec![false; nodes.len()]; + while order.len() < nodes.len() { + let before = order.len(); + for i in 0..nodes.len() { + if !placed[i] && parent[i].map_or(true, |p| placed[p]) { + placed[i] = true; + order.push(i); + } + } + if order.len() == before { + bail!("node hierarchy has a cycle"); + } + } + let local: Vec = nodes.iter().map(node_local).collect(); + let mut world = vec![Mat4::IDENTITY; nodes.len()]; + for &i in &order { + world[i] = parent[i].map_or(local[i], |p| world[p] * local[i]); + } + + // World translation delta per node: its pack delta, else its parent's. + let pack_joint: HashMap<&str, usize> = joints.iter().enumerate().map(|(i, n)| (n.as_str(), i)).collect(); + let mut dworld = vec![Vec3::ZERO; nodes.len()]; + for &i in &order { + let own = nodes[i]["name"].as_str().and_then(|n| pack_joint.get(n)); + dworld[i] = match own { + Some(&j) => Vec3::from(delta.joints[j]), + None => parent[i].map_or(Vec3::ZERO, |p| dworld[p]), + }; + } + for &i in &order { + if dworld[i] == parent[i].map_or(Vec3::ZERO, |p| dworld[p]) { + continue; + } + if nodes[i].get("matrix").is_some() { + bail!("node {i} uses a matrix; only TRS joints are supported"); + } + let plin = parent[i].map_or(Mat3::IDENTITY, |p| Mat3::from_mat4(world[p])); + let d = plin.inverse() * (dworld[i] - parent[i].map_or(Vec3::ZERO, |p| dworld[p])); + let n = &mut g.json["nodes"][i]; + let t = n["translation"].as_array().map(|a| a.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect::>()).unwrap_or(vec![0.0; 3]); + n["translation"] = serde_json::json!([t[0] + d.x, t[1] + d.y, t[2] + d.z]); + } + let world_new = |i: usize| Mat4::from_translation(dworld[i]) * world[i]; + + // Inverse bind matrices; remember each skin's bind-space transform. + let skins = g.json["skins"].as_array().cloned().unwrap_or_default(); + let mut mesh_space = HashMap::new(); + for (s, skin) in skins.iter().enumerate() { + let joints: Vec = skin["joints"].as_array().context("skin joints")?.iter().filter_map(|j| j.as_u64().map(|j| j as usize)).collect(); + let acc = skin["inverseBindMatrices"].as_u64().context("skin without inverseBindMatrices")? as usize; + let ibm: Vec<[f32; 16]> = read_f32s(&g, acc)?; + let base: Vec = ibm.iter().map(Mat4::from_cols_array).collect(); + mesh_space.insert(s, world[joints[0]] * base[0]); + let new: Vec<[f32; 16]> = joints.iter().zip(&base).map(|(&j, b)| (world_new(j).inverse() * world[j] * *b).to_cols_array()).collect(); + write_f32s(&mut g, acc, &new)?; + } + + // Vertex positions. + let mut skin_of_mesh = HashMap::new(); + for n in &nodes { + if let (Some(m), Some(s)) = (n["mesh"].as_u64(), n["skin"].as_u64()) { + skin_of_mesh.insert(m as usize, s as usize); + } + } + let meshes = g.json["meshes"].as_array().cloned().unwrap_or_default(); + for (m, mesh) in meshes.iter().enumerate() { + let name = mesh["name"].as_str().unwrap_or_default(); + let Some(offset) = mesh_offset(name) else { continue }; + let to_mesh = skin_of_mesh.get(&m).and_then(|s| mesh_space.get(s)).map_or(Mat3::IDENTITY, |w| Mat3::from_mat4(w.inverse())); + for (p, prim) in mesh["primitives"].as_array().into_iter().flatten().enumerate() { + let attrs = &prim["attributes"]; + let (Some(pos_acc), Some(idx_acc)) = (attrs["POSITION"].as_u64(), attrs["_CCIDX"].as_u64()) else { + bail!("mesh {name} primitive {p} lacks POSITION or _CCIDX"); + }; + let ccidx: Vec<[f32; 1]> = read_f32s(&g, idx_acc as usize)?; + let mut pos: Vec<[f32; 3]> = read_f32s(&g, pos_acc as usize)?; + for (v, [ci]) in pos.iter_mut().zip(&ccidx) { + let gi = offset as usize + *ci as usize; + let d = to_mesh * Vec3::from(*delta.verts.get(gi).ok_or_else(|| anyhow!("{name}: _CCIDX {ci} past pack"))?); + *v = (Vec3::from(*v) + d).into(); + } + write_f32s(&mut g, pos_acc as usize, &pos)?; + let (lo, hi) = pos.iter().fold((Vec3::splat(f32::MAX), Vec3::splat(f32::MIN)), |(lo, hi), v| (lo.min(Vec3::from(*v)), hi.max(Vec3::from(*v)))); + g.json["accessors"][pos_acc as usize]["min"] = serde_json::json!([lo.x, lo.y, lo.z]); + g.json["accessors"][pos_acc as usize]["max"] = serde_json::json!([hi.x, hi.y, hi.z]); + } + } + write(&g) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::ShapeDelta; + + /// Two-joint chain (Hip -> Head, Head rotated 90° about Y), one skinned mesh + /// with 3 glTF vertices mapping to 2 CC5 vertices (index 1 seam-split). + fn tiny_glb() -> Vec { + let mut bin = Vec::new(); + let mut push = |vals: &[f32]| { + let off = bin.len(); + for v in vals { + bin.extend_from_slice(&v.to_le_bytes()); + } + off + }; + let pos = push(&[0.0, 1.0, 0.0, 0.0, 1.7, 0.0, 0.0, 1.7, 0.0]); + let cc = push(&[0.0, 1.0, 1.0]); + let hip = Mat4::from_translation(Vec3::new(0.0, 1.0, 0.0)); + let head = hip * Mat4::from_rotation_translation(Quat::from_rotation_y(std::f32::consts::FRAC_PI_2), Vec3::new(0.0, 0.7, 0.0)); + let ibm = push(&[hip.inverse().to_cols_array(), head.inverse().to_cols_array()].concat()); + let json = serde_json::json!({ + "asset": {"version": "2.0"}, + "nodes": [ + {"name": "Hip", "translation": [0.0, 1.0, 0.0], "children": [1]}, + {"name": "Head", "translation": [0.0, 0.7, 0.0], "rotation": [0.0, 0.70710677, 0.0, 0.70710677]}, + {"name": "Body", "mesh": 0, "skin": 0} + ], + "skins": [{"joints": [0, 1], "inverseBindMatrices": 2}], + "meshes": [{"name": "CC_Base_Body", "primitives": [{"attributes": {"POSITION": 0, "_CCIDX": 1}}]}], + "buffers": [{"byteLength": bin.len()}], + "bufferViews": [ + {"buffer": 0, "byteOffset": pos, "byteLength": 36}, + {"buffer": 0, "byteOffset": cc, "byteLength": 12}, + {"buffer": 0, "byteOffset": ibm, "byteLength": 128} + ], + "accessors": [ + {"bufferView": 0, "componentType": 5126, "count": 3, "type": "VEC3"}, + {"bufferView": 1, "componentType": 5126, "count": 3, "type": "SCALAR"}, + {"bufferView": 2, "componentType": 5126, "count": 2, "type": "MAT4"} + ] + }); + write(&Glb { json, bin }).unwrap() + } + + fn pack(dir: &std::path::Path) -> ShapePack { + std::fs::write(dir.join("pack.bin"), []).unwrap(); + let json = serde_json::json!({"version": 1, "vertex_count": 2, + "meshes": [{"name": "CC_Base_Body", "offset": 0, "count": 2}], + "joints": ["Head", "Hip"], "sliders": []}); + std::fs::write(dir.join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + ShapePack::load(dir).unwrap() + } + + #[test] + fn moves_mesh_joints_and_keeps_skin_consistent() { + let dir = tempfile::tempdir().unwrap(); + let pack = pack(dir.path()); + // Hip down 0.1, Head down 0.15 (so the Hip->Head offset shrinks by 0.05); + // vertex 1 (both seam copies) down 0.15. + let delta = ShapeDelta { + verts: vec![[0.0, -0.1, 0.0], [0.0, -0.15, 0.0]], + joints: vec![[0.0, -0.15, 0.0], [0.0, -0.1, 0.0]], + }; + let out = parse(&shape_glb(&tiny_glb(), &pack, &delta).unwrap()).unwrap(); + let t = |i: usize| out.json["nodes"][i]["translation"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap() as f32).collect::>(); + assert!((t(0)[1] - 0.9).abs() < 1e-6, "{:?}", t(0)); + assert!((t(1)[1] - 0.65).abs() < 1e-6, "{:?}", t(1)); + let pos: Vec<[f32; 3]> = read_f32s(&out, 0).unwrap(); + assert!((pos[0][1] - 0.9).abs() < 1e-6 && (pos[1][1] - 1.55).abs() < 1e-6 && (pos[2][1] - 1.55).abs() < 1e-6, "{pos:?}"); + assert!((out.json["accessors"][0]["max"][1].as_f64().unwrap() - 1.55).abs() < 1e-6); + + // Bind renders the written mesh: world(joint) · IBM' is the same for both joints. + let local: Vec = out.json["nodes"].as_array().unwrap().iter().map(node_local).collect(); + let world = [local[0], local[0] * local[1]]; + let ibm: Vec<[f32; 16]> = read_f32s(&out, 2).unwrap(); + let m: Vec = world.iter().zip(&ibm).map(|(w, i)| *w * Mat4::from_cols_array(i)).collect(); + assert!(m[0].abs_diff_eq(m[1], 1e-5) && m[0].abs_diff_eq(Mat4::IDENTITY, 1e-5), "{m:?}"); + } + + #[test] + fn rejects_mesh_without_ccidx() { + let dir = tempfile::tempdir().unwrap(); + let pack = pack(dir.path()); + let mut g = parse(&tiny_glb()).unwrap(); + g.json["meshes"][0]["primitives"][0]["attributes"].as_object_mut().unwrap().remove("_CCIDX"); + let delta = ShapeDelta { verts: vec![[0.0; 3]; 2], joints: vec![[0.0; 3]; 2] }; + let err = shape_glb(&write(&g).unwrap(), &pack, &delta).unwrap_err().to_string(); + assert!(err.contains("_CCIDX"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/lib.rs b/crates/cc5-shape/src/lib.rs new file mode 100644 index 0000000..ca9d52e --- /dev/null +++ b/crates/cc5-shape/src/lib.rs @@ -0,0 +1,24 @@ +//! Character Creator 5 shaping sliders outside CC5. +//! +//! - [`pack`]: load a slider pack (`pack.json` + `pack.bin`, built from a CC5 +//! capture) and evaluate a slider mix into vertex and joint deltas. +//! - [`glb`]: write those deltas into a character GLB (bind mesh, joints and +//! inverse bind matrices). +//! - [`subdiv`]: drive CC5's HD (subdivided) mesh from the level-0 pack. +//! - [`expr`]: facial expressions (CC5's ~400 morph targets, more than Bevy's 64 +//! per mesh) as sparse vertex deltas in the same space, for CPU morphing. +//! +//! With the `python` feature this crate is also the CPython module `cc5_shape`. + +pub mod expr; +pub mod glb; +pub mod pack; +pub mod subdiv; + +#[cfg(feature = "python")] +mod python; + +pub use expr::{ArkitMap, ExpressionPack}; +pub use glb::{shape_glb, shape_glb_hd}; +pub use pack::{ShapeDelta, ShapePack, Slider, StateDeltas}; +pub use subdiv::SubdivMap; diff --git a/crates/cc5-shape/src/pack.rs b/crates/cc5-shape/src/pack.rs new file mode 100644 index 0000000..62f7ce5 --- /dev/null +++ b/crates/cc5-shape/src/pack.rs @@ -0,0 +1,396 @@ +//! CC5 shaping sliders outside CC5: load an editor pack and evaluate a slider mix. +//! +//! The pack is built from a CC5 capture (`cc5_capture_sliders.py`, MODE "bind") +//! by `build_editor_pack.py`: +//! +//! - `pack.json` — slider tree (id, name, category, range, base weight) and, per +//! slider, the captured min/max states as byte ranges into `pack.bin`; +//! - `pack.bin` — per block: `u32 index[count]` then `f32 delta[count * 3]`. +//! +//! Vertex deltas are in one global index space (CC5 bind-pose mesh order, meshes +//! concatenated); a GLB vertex maps to it through its `_CCIDX` attribute: +//! `global = mesh.offset + _CCIDX`. Joint deltas index `joints`. Everything is in +//! the glTF frame (Y-up, meters) and describes the *bind pose*: skeletal stretch +//! (e.g. Character Height) is already baked into the vertex deltas, so applying a +//! mix is `bind + Σ t_i · delta_i` for the mesh and the joints alike. +//! +//! CC5 sliders are linear on each side of their base weight (verified in CC5: a +//! slider at half weight and two sliders combined both reproduce the sum of the +//! captured deltas exactly), so a state captured at `weight` scales by +//! `(w - base) / (weight - base)`. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct PackMesh { + pub name: String, + pub offset: u32, + pub count: u32, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct PackState { + pub weight: f32, + /// `[byte_offset, count]` into `pack.bin`. + pub verts: [u64; 2], + pub joints: [u64; 2], +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Slider { + pub id: String, + pub name: String, + pub category: String, + pub min: f32, + pub max: f32, + /// Weight the character was captured at (its design value). + pub base: f32, + pub states: HashMap, +} + +#[derive(Debug, Deserialize)] +struct PackJson { + version: u32, + vertex_count: usize, + meshes: Vec, + joints: Vec, + sliders: Vec, +} + +/// One captured slider state, see [`ShapePack::state`]. +#[derive(Debug, Clone, PartialEq)] +pub struct StateDeltas { + pub weight: f32, + pub vert_index: Vec, + pub vert_delta: Vec<[f32; 3]>, + pub joint_index: Vec, + pub joint_delta: Vec<[f32; 3]>, +} + +/// Summed deltas for a slider mix, in the glTF frame (meters). +#[derive(Debug, Clone, PartialEq)] +pub struct ShapeDelta { + /// One entry per global vertex. + pub verts: Vec<[f32; 3]>, + /// One entry per `ShapePack::joints` entry. + pub joints: Vec<[f32; 3]>, +} + +pub struct ShapePack { + pub vertex_count: usize, + pub meshes: Vec, + pub joints: Vec, + pub sliders: Vec, + bin: Vec, +} + +impl ShapePack { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("pack.json"); + let pack: PackJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + if pack.version != 1 { + bail!("unsupported pack version {}", pack.version); + } + let bin_path = dir.join("pack.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let me = Self { + vertex_count: pack.vertex_count, + meshes: pack.meshes, + joints: pack.joints, + sliders: pack.sliders, + bin, + }; + me.validate()?; + Ok(me) + } + + fn validate(&self) -> Result<()> { + for s in &self.sliders { + for (which, st) in &s.states { + for (what, [off, n], limit) in [ + ("verts", st.verts, self.vertex_count), + ("joints", st.joints, self.joints.len()), + ] { + let end = off + n * 16; + if end as usize > self.bin.len() { + bail!("{} {which} {what}: block past end of pack.bin", s.name); + } + if let Some(bad) = self.indices(off, n).find(|&i| i as usize >= limit) { + bail!("{} {which} {what}: index {bad} out of range {limit}", s.name); + } + } + } + } + Ok(()) + } + + /// Global vertex offset of a mesh (by glTF mesh name). + pub fn mesh_offset(&self, name: &str) -> Option { + self.meshes.iter().find(|m| m.name == name).map(|m| m.offset) + } + + /// Base weights: the character exactly as captured. + pub fn base_weights(&self) -> Vec { + self.sliders.iter().map(|s| s.base).collect() + } + + /// Index of the slider with this id, or with this display name if that is unique. + pub fn find(&self, key: &str) -> Result { + if let Some(i) = self.sliders.iter().position(|s| s.id == key) { + return Ok(i); + } + let hits: Vec = (0..self.sliders.len()).filter(|&i| self.sliders[i].name == key).collect(); + match hits.as_slice() { + [i] => Ok(*i), + [] => bail!("no slider named {key:?}"), + _ => bail!( + "{key:?} names {} sliders ({}); use an id", + hits.len(), + hits.iter().map(|&i| self.sliders[i].category.as_str()).collect::>().join(", ") + ), + } + } + + /// Full weight vector: base weights with the given sliders (id or unique name) set. + pub fn weights_from<'a>(&self, set: impl IntoIterator) -> Result> { + let mut w = self.base_weights(); + for (key, value) in set { + let i = self.find(key)?; + let s = &self.sliders[i]; + if !(s.min..=s.max).contains(&value) { + bail!("{key}: {value} outside {}..{}", s.min, s.max); + } + w[i] = value; + } + Ok(w) + } + + /// A recipe for `weights`: the sliders that differ from the captured design, by id. + /// `{"format": "cc5-shape recipe", "version": 1, "sliders": {id: value}}` + pub fn recipe(&self, weights: &[f32]) -> serde_json::Value { + let sliders: serde_json::Map = self + .sliders + .iter() + .zip(weights) + .filter(|(s, w)| (**w - s.base).abs() > 1e-6) + .map(|(s, w)| (s.id.clone(), serde_json::json!(w))) + .collect(); + serde_json::json!({"format": "cc5-shape recipe", "version": 1, "sliders": sliders}) + } + + /// Weights from a recipe (`recipe()` output, or a plain `{id or name: value}` map). + /// Unknown sliders are an error: the recipe was made for a different pack. + pub fn weights_from_recipe(&self, recipe: &serde_json::Value) -> Result> { + let map = recipe.get("sliders").unwrap_or(recipe).as_object().context("recipe is not a JSON object")?; + let mut set = Vec::with_capacity(map.len()); + for (k, v) in map { + set.push((k.as_str(), v.as_f64().with_context(|| format!("{k}: not a number"))? as f32)); + } + self.weights_from(set) + } + + /// Raw captured state `which` ("min" or "max") of slider `i`: its weight, and the + /// sparse vertex and joint deltas at that weight (relative to the base weight). + /// For building solvers on top of the linear model (e.g. fitting to a target). + pub fn state(&self, i: usize, which: &str) -> Option { + let st = self.sliders.get(i)?.states.get(which)?; + let [vo, vn] = st.verts; + let [jo, jn] = st.joints; + Some(StateDeltas { + weight: st.weight, + vert_index: self.indices(vo, vn).collect(), + vert_delta: self.deltas(vo, vn).collect(), + joint_index: self.indices(jo, jn).collect(), + joint_delta: self.deltas(jo, jn).collect(), + }) + } + + fn indices(&self, off: u64, n: u64) -> impl Iterator + '_ { + block_indices(&self.bin, off, n) + } + + fn deltas(&self, off: u64, n: u64) -> impl Iterator + '_ { + block_deltas(&self.bin, off, n) + } + + fn add_block(&self, block: [u64; 2], t: f32, out: &mut [[f32; 3]]) { + add_block(&self.bin, block, t, out); + } + + /// Captured state and scale for slider `i` at weight `w`, if it moves at all. + fn state_for(&self, i: usize, w: f32) -> Option<(&PackState, f32)> { + let s = &self.sliders[i]; + let d = w - s.base; + if d.abs() < 1e-6 { + return None; + } + let st = s.states.get(if d > 0.0 { "max" } else { "min" })?; + let span = st.weight - s.base; + (span.abs() > 1e-9).then(|| (st, d / span)) + } + + /// Sum of deltas for `weights` (one per slider, in slider order). + pub fn evaluate(&self, weights: &[f32]) -> ShapeDelta { + assert_eq!(weights.len(), self.sliders.len(), "one weight per slider"); + let mut out = ShapeDelta { + verts: vec![[0.0; 3]; self.vertex_count], + joints: vec![[0.0; 3]; self.joints.len()], + }; + for (i, &w) in weights.iter().enumerate() { + if let Some((st, t)) = self.state_for(i, w) { + self.add_block(st.verts, t, &mut out.verts); + self.add_block(st.joints, t, &mut out.joints); + } + } + out + } + + /// Sliders grouped by category path, sorted, for building a UI tree. + pub fn categories(&self) -> Vec<(String, Vec)> { + let mut by: HashMap<&str, Vec> = HashMap::new(); + for (i, s) in self.sliders.iter().enumerate() { + by.entry(s.category.as_str()).or_default().push(i); + } + let mut out: Vec<(String, Vec)> = by + .into_iter() + .map(|(c, mut v)| { + v.sort_by(|&a, &b| self.sliders[a].name.cmp(&self.sliders[b].name)); + (c.to_string(), v) + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } +} + +// Sparse delta blocks (`pack.bin`, `expressions.bin`): `u32 index[n]` then +// `f32 delta[n * 3]`, starting at byte `off`. Bounds are checked at load. + +pub(crate) fn block_indices(bin: &[u8], off: u64, n: u64) -> impl Iterator + '_ { + let start = off as usize; + (0..n as usize).map(move |i| { + let b = start + 4 * i; + u32::from_le_bytes(bin[b..b + 4].try_into().unwrap()) + }) +} + +pub(crate) fn block_deltas(bin: &[u8], off: u64, n: u64) -> impl Iterator + '_ { + let start = (off + 4 * n) as usize; + (0..n as usize).map(move |i| { + let b = start + 12 * i; + let f = |k: usize| f32::from_le_bytes(bin[b + 4 * k..b + 4 * k + 4].try_into().unwrap()); + [f(0), f(1), f(2)] + }) +} + +/// `out[index] += t * delta` for one block. +pub(crate) fn add_block(bin: &[u8], [off, n]: [u64; 2], t: f32, out: &mut [[f32; 3]]) { + for (i, d) in block_indices(bin, off, n).zip(block_deltas(bin, off, n)) { + let o = &mut out[i as usize]; + o[0] += t * d[0]; + o[1] += t * d[1]; + o[2] += t * d[2]; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(bin: &mut Vec, idx: &[u32], d: &[[f32; 3]]) -> [u64; 2] { + let off = bin.len() as u64; + for i in idx { + bin.extend_from_slice(&i.to_le_bytes()); + } + for v in d { + for c in v { + bin.extend_from_slice(&c.to_le_bytes()); + } + } + [off, idx.len() as u64] + } + + fn write_pack(dir: &Path) { + let mut bin = Vec::new(); + let h_max_v = block(&mut bin, &[0, 2], &[[0.0, -0.1, 0.0], [0.0, -0.2, 0.0]]); + let h_max_j = block(&mut bin, &[1], &[[0.0, -0.1, 0.0]]); + let h_min_v = block(&mut bin, &[2], &[[0.0, 0.4, 0.0]]); + let h_min_j = block(&mut bin, &[], &[]); + let f_max_v = block(&mut bin, &[1], &[[0.05, 0.0, 0.0]]); + let f_max_j = block(&mut bin, &[], &[]); + std::fs::write(dir.join("pack.bin"), &bin).unwrap(); + let json = serde_json::json!({ + "version": 1, "vertex_count": 3, + "meshes": [{"name": "Body", "offset": 0, "count": 3}], + "joints": ["Hip", "Head"], + "sliders": [ + {"id": "h", "name": "Height", "category": "Body/Full", "min": -1.0, "max": 1.0, "base": 0.0, + "states": {"max": {"weight": 1.0, "verts": h_max_v, "joints": h_max_j}, + "min": {"weight": -1.0, "verts": h_min_v, "joints": h_min_j}}}, + {"id": "f", "name": "Fat", "category": "Body/Full", "min": 0.0, "max": 1.0, "base": 0.0, + "states": {"max": {"weight": 1.0, "verts": f_max_v, "joints": f_max_j}}} + ] + }); + std::fs::write(dir.join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + } + + #[test] + fn evaluates_linear_mix_per_side() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + assert_eq!(p.evaluate(&p.base_weights()).verts, vec![[0.0; 3]; 3]); + + let d = p.evaluate(&[0.5, 1.0]); + assert_eq!(d.verts[0], [0.0, -0.05, 0.0]); + assert_eq!(d.verts[1], [0.05, 0.0, 0.0]); + assert_eq!(d.verts[2], [0.0, -0.1, 0.0]); + assert_eq!(d.joints, vec![[0.0; 3], [0.0, -0.05, 0.0]]); + + // Below base uses the min state, scaled by its own span. + let d = p.evaluate(&[-0.25, 0.0]); + assert_eq!(d.verts[2], [0.0, 0.1, 0.0]); + assert_eq!(p.categories(), vec![("Body/Full".to_string(), vec![1, 0])]); + } + + #[test] + fn resolves_weights_by_id_or_unique_name() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + assert_eq!(p.weights_from([("Height", 0.5), ("f", 1.0)]).unwrap(), vec![0.5, 1.0]); + assert!(p.weights_from([("Height", 2.0)]).unwrap_err().to_string().contains("outside")); + assert!(p.find("Nope").unwrap_err().to_string().contains("no slider")); + } + + #[test] + fn recipe_round_trips() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + let r = p.recipe(&[0.5, 0.0]); + assert_eq!(r["sliders"], serde_json::json!({"h": 0.5})); + assert_eq!(p.weights_from_recipe(&r).unwrap(), vec![0.5, 0.0]); + assert_eq!(p.weights_from_recipe(&serde_json::json!({"Fat": 1.0})).unwrap(), vec![0.0, 1.0]); + assert!(p.weights_from_recipe(&serde_json::json!({"sliders": {"zz": 1.0}})).is_err()); + } + + #[test] + fn rejects_out_of_range_index() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let mut json: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("pack.json")).unwrap()).unwrap(); + json["vertex_count"] = 2.into(); + std::fs::write(dir.path().join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + let err = ShapePack::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/python.rs b/crates/cc5-shape/src/python.rs new file mode 100644 index 0000000..8497919 --- /dev/null +++ b/crates/cc5-shape/src/python.rs @@ -0,0 +1,359 @@ +//! CPython bindings: `import cc5_shape`. +//! +//! ```python +//! import cc5_shape +//! pack = cc5_shape.Pack("path/to/pack") +//! w = pack.weights({"Character Height": 0.5, "Body Fat A": 0.3}) +//! open("shaped.glb", "wb").write(pack.shape_glb(open("character.glb", "rb").read(), w)) +//! verts, joints = pack.evaluate(w) # bytes (f32 xyz per vertex), {joint: (x, y, z)} +//! +//! ex = cc5_shape.Expressions("path/to/expressions") +//! d = ex.evaluate({"Eye_Blink_L": 1.0, "Jaw_Open": 0.3}) # bytes, f32 xyz per vertex +//! arkit = cc5_shape.ArkitMap("beautiful.arkit_map.json", ex) +//! d = ex.evaluate(arkit.weights({"jawOpen": 0.4})) +//! +//! hd = cc5_shape.SubdivMap("path/to/hd_map") # CC5 HD mesh (subdivision 1 or 2) +//! open("shaped_hd.glb", "wb").write(pack.shape_glb(open("character_hd.glb", "rb").read(), w, hd=hd)) +//! ``` +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict}; + +use crate::expr::{ArkitMap, ExpressionPack}; +use crate::pack::ShapePack; +use crate::subdiv::SubdivMap; + +fn err(e: anyhow::Error) -> PyErr { + PyValueError::new_err(format!("{e:#}")) +} + +fn f32_bytes<'py>(py: Python<'py>, v: &[[f32; 3]]) -> Bound<'py, PyBytes> { + let mut bytes = Vec::with_capacity(v.len() * 12); + for p in v { + for c in p { + bytes.extend_from_slice(&c.to_le_bytes()); + } + } + PyBytes::new(py, &bytes) +} + +/// A CC5 slider pack (`pack.json` + `pack.bin`). +#[pyclass(name = "Pack", module = "cc5_shape", frozen)] +struct PyPack { + inner: ShapePack, +} + +impl PyPack { + fn check(&self, weights: &[f32]) -> PyResult<()> { + if weights.len() != self.inner.sliders.len() { + return Err(PyValueError::new_err(format!( + "expected {} weights (one per slider), got {}", + self.inner.sliders.len(), + weights.len() + ))); + } + Ok(()) + } +} + +#[pymethods] +impl PyPack { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: ShapePack::load(&dir).map_err(err)? }) + } + + /// Number of vertices in the pack's global vertex space. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// Joint names, in the order `evaluate` reports them. + #[getter] + fn joints(&self) -> Vec { + self.inner.joints.clone() + } + + /// `[(mesh name, global vertex offset, vertex count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Sliders as dicts: id, name, category, min, max, base (the captured design value). + #[getter] + fn sliders<'py>(&self, py: Python<'py>) -> PyResult>> { + self.inner + .sliders + .iter() + .map(|s| { + let d = PyDict::new(py); + d.set_item("id", &s.id)?; + d.set_item("name", &s.name)?; + d.set_item("category", &s.category)?; + d.set_item("min", s.min)?; + d.set_item("max", s.max)?; + d.set_item("base", s.base)?; + Ok(d) + }) + .collect() + } + + /// Sliders grouped by category: `[(category, [slider index, ...])]`, sorted. + fn categories(&self) -> Vec<(String, Vec)> { + self.inner.categories() + } + + /// Index of a slider by id, or by display name when that is unique. + fn find(&self, key: &str) -> PyResult { + self.inner.find(key).map_err(err) + } + + /// The character as captured: one weight per slider. + fn base_weights(&self) -> Vec { + self.inner.base_weights() + } + + /// Base weights with `{id or name: value}` applied (range-checked). + #[pyo3(signature = (set=None))] + fn weights(&self, set: Option>) -> PyResult> { + let set = set.unwrap_or_default(); + self.inner.weights_from(set.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err) + } + + /// Recipe JSON (str) for a weight vector: only sliders changed from the design, by id. + fn recipe(&self, weights: Vec) -> PyResult { + self.check(&weights)?; + Ok(serde_json::to_string_pretty(&self.inner.recipe(&weights)).unwrap()) + } + + /// Weights from a recipe JSON string (`recipe()` output or a plain {id or name: value}). + fn weights_from_recipe(&self, recipe: &str) -> PyResult> { + let v: serde_json::Value = serde_json::from_str(recipe).map_err(|e| err(e.into()))?; + self.inner.weights_from_recipe(&v).map_err(err) + } + + /// `(vertex deltas, joint deltas)` for a weight vector, in the glTF frame (meters): + /// vertex deltas as little-endian f32 bytes (x, y, z per global vertex; e.g. + /// `numpy.frombuffer(v, "(&self, py: Python<'py>, weights: Vec) -> PyResult<(Bound<'py, PyBytes>, HashMap)> { + self.check(&weights)?; + let d = py.detach(|| self.inner.evaluate(&weights)); + let joints = self + .inner + .joints + .iter() + .zip(&d.joints) + .filter(|(_, j)| j.iter().any(|c| *c != 0.0)) + .map(|(n, j)| (n.clone(), (j[0], j[1], j[2]))) + .collect(); + Ok((f32_bytes(py, &d.verts), joints)) + } + + /// Raw captured state of slider `i` (`which` = "min" or "max"), or None: + /// `(weight, vertex indices u32 bytes, vertex deltas f32 xyz bytes, + /// joint indices u32 bytes, joint deltas f32 xyz bytes)`, glTF frame (m). + fn state<'py>(&self, py: Python<'py>, i: usize, which: &str) -> Option<(f32, Bound<'py, PyBytes>, Bound<'py, PyBytes>, Bound<'py, PyBytes>, Bound<'py, PyBytes>)> { + let s = self.inner.state(i, which)?; + let u = |v: &[u32]| PyBytes::new(py, &v.iter().flat_map(|x| x.to_le_bytes()).collect::>()); + let f = |v: &[[f32; 3]]| PyBytes::new(py, &v.iter().flatten().flat_map(|x| x.to_le_bytes()).collect::>()); + Some((s.weight, u(&s.vert_index), f(&s.vert_delta), u(&s.joint_index), f(&s.joint_delta))) + } + + /// A shaped copy of `glb` (which must carry `_CCIDX`) for a weight vector. + /// With `hd` (a `SubdivMap`), `glb` is the character's CC5 HD export. + #[pyo3(signature = (glb, weights, hd=None))] + fn shape_glb<'py>(&self, py: Python<'py>, glb: &[u8], weights: Vec, hd: Option<&PySubdivMap>) -> PyResult> { + self.check(&weights)?; + if let Some(hd) = hd { + hd.inner.check(&self.inner).map_err(err)?; + } + let out = py.detach(|| { + let d = self.inner.evaluate(&weights); + match hd { + Some(hd) => crate::glb::shape_glb_hd(glb, &self.inner, &hd.inner, &hd.inner.expand(&d)), + None => crate::glb::shape_glb(glb, &self.inner, &d), + } + }); + Ok(PyBytes::new(py, &out.map_err(err)?)) + } + + fn __repr__(&self) -> String { + format!( + "", + self.inner.sliders.len(), + self.inner.vertex_count, + self.inner.joints.len() + ) + } +} + +/// A CC5 expression pack (`expressions.json` + `expressions.bin`): facial morphs +/// as sparse deltas in the slider pack's global vertex space. +#[pyclass(name = "Expressions", module = "cc5_shape", frozen)] +struct PyExpressions { + inner: ExpressionPack, +} + +#[pymethods] +impl PyExpressions { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: ExpressionPack::load(&dir).map_err(err)? }) + } + + /// Number of vertices in the global vertex space. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// `[(mesh name, global vertex offset, vertex count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Expression names, in pack order. + #[getter] + fn names(&self) -> Vec { + self.inner.names().map(str::to_owned).collect() + } + + /// Index of an expression by name. + fn find(&self, name: &str) -> PyResult { + self.inner.find(name).ok_or_else(|| PyValueError::new_err(format!("no expression named {name:?}"))) + } + + /// Meshes expression `name` moves. + fn meshes_of(&self, name: &str) -> PyResult> { + Ok(self.inner.expressions[self.find(name)?].meshes.clone()) + } + + /// Sparse delta of one expression at weight 1: `(u32 indices bytes, f32 xyz bytes)`. + fn deltas<'py>(&self, py: Python<'py>, name: &str) -> PyResult<(Bound<'py, PyBytes>, Bound<'py, PyBytes>)> { + let (idx, d) = self.inner.deltas(self.find(name)?); + let idx: Vec = idx.iter().flat_map(|i| i.to_le_bytes()).collect(); + Ok((PyBytes::new(py, &idx), f32_bytes(py, &d))) + } + + /// Vertex deltas for `{name: weight}` (glTF frame, meters) as little-endian f32 + /// bytes, x, y, z per global vertex. + fn evaluate<'py>(&self, py: Python<'py>, weights: HashMap) -> PyResult> { + let w = self.inner.weights_from(weights.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err)?; + let d = py.detach(|| self.inner.evaluate(&w)); + Ok(f32_bytes(py, &d)) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __repr__(&self) -> String { + format!("", self.inner.len(), self.inner.vertex_count) + } +} + +/// Map from the pack's (level 0) vertex space to CC5's HD mesh (`hd_map.json` + `.bin`). +#[pyclass(name = "SubdivMap", module = "cc5_shape", frozen)] +struct PySubdivMap { + inner: SubdivMap, +} + +#[pymethods] +impl PySubdivMap { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: SubdivMap::load(&dir).map_err(err)? }) + } + + /// CC5 subdivision level (1 or 2). + #[getter] + fn level(&self) -> u32 { + self.inner.level + } + + /// HD global vertex count. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// `[(mesh name, HD offset, HD count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Base-space per-vertex f32 xyz bytes (e.g. `Pack.evaluate`'s vertex deltas) + /// -> HD-space f32 xyz bytes. + fn expand<'py>(&self, py: Python<'py>, verts: &[u8]) -> PyResult> { + if verts.len() != self.inner.base_vertex_count * 12 { + return Err(PyValueError::new_err(format!( + "expected {} bytes ({} base vertices x 3 f32), got {}", + self.inner.base_vertex_count * 12, + self.inner.base_vertex_count, + verts.len() + ))); + } + let v: Vec<[f32; 3]> = verts + .chunks_exact(12) + .map(|c| std::array::from_fn(|k| f32::from_le_bytes(c[4 * k..4 * k + 4].try_into().unwrap()))) + .collect(); + let out = py.detach(|| self.inner.expand_verts(&v)); + Ok(f32_bytes(py, &out)) + } + + fn __repr__(&self) -> String { + format!("", self.inner.level, self.inner.vertex_count, self.inner.base_vertex_count) + } +} + +/// ARKit-52 -> expression weights, from a runtime ARKit map JSON. +#[pyclass(name = "ArkitMap", module = "cc5_shape", frozen)] +struct PyArkitMap { + inner: ArkitMap, + names: Vec, +} + +#[pymethods] +impl PyArkitMap { + #[new] + fn new(path: std::path::PathBuf, expressions: &PyExpressions) -> PyResult { + let inner = ArkitMap::load(&path, &expressions.inner).map_err(err)?; + Ok(Self { inner, names: expressions.inner.names().map(str::to_owned).collect() }) + } + + /// ARKit names in the map, sorted. + #[getter] + fn arkit_names(&self) -> Vec { + let mut v: Vec = self.inner.mapping.keys().cloned().collect(); + v.sort(); + v + } + + /// Morph names the map uses that the expression pack lacks. + #[getter] + fn missing(&self) -> Vec { + self.inner.missing.clone() + } + + /// `{expression name: weight}` for `{arkit name: weight}` (for `Expressions.evaluate`). + fn weights(&self, arkit: HashMap) -> PyResult> { + let w = self.inner.weights(arkit.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err)?; + Ok(w.into_iter().map(|(i, v)| (self.names[i].clone(), v)).collect()) + } +} + +#[pymodule] +fn cc5_shape(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/crates/cc5-shape/src/subdiv.rs b/crates/cc5-shape/src/subdiv.rs new file mode 100644 index 0000000..66116c8 --- /dev/null +++ b/crates/cc5-shape/src/subdiv.rs @@ -0,0 +1,212 @@ +//! Drive CC5's HD mesh (subdivision level 1 or 2) from a level-0 slider pack. +//! +//! CC5 HD characters render a Catmull-Clark subdivision of the base mesh plus +//! per-character sculpted detail. Subdivision is linear, so an HD vertex moves +//! by a fixed weighted sum of base-vertex moves: `hd_delta = S · base_delta`. +//! `build_hd_map.py` writes `S` per subdivided mesh, rows in CC5's HD vertex +//! order (= the HD character GLB's `_CCIDX`): +//! +//! - `hd_map.json` — `{"level", "vertex_count", "base_vertex_count", "meshes": +//! [{"name", "offset", "count", "base_offset", "base_count", "stencil": +//! [byte_offset, nnz] | null}]}`; meshes in the pack's order, `offset`s in the +//! HD global vertex space; +//! - `hd_map.bin` — per stencil: `u32 indptr[count + 1]`, `u32 base_index[nnz]` +//! (mesh-local), `f32 weight[nnz]`. +//! +//! Meshes without a stencil (hair, brows, lashes, clothes) map 1:1. The sculpted +//! detail is already in the HD GLB's bind positions. Sliders that only reshape +//! the base mesh are reproduced exactly (p99 ≤ 0.02 mm vs CC5); HD presets and +//! large body sliders also change the HD level directly, which a level-0 pack +//! cannot see (p99 up to ~0.8 mm, max ~3 mm). + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +use crate::pack::{ShapeDelta, ShapePack}; + +#[derive(Debug, Clone, Deserialize)] +pub struct HdMesh { + pub name: String, + pub offset: u32, + pub count: u32, + pub base_offset: u32, + pub base_count: u32, + stencil: Option<[u64; 2]>, +} + +#[derive(Deserialize)] +struct MapJson { + level: u32, + vertex_count: usize, + base_vertex_count: usize, + meshes: Vec, +} + +/// CSR rows = HD vertices of one mesh, columns = its base vertices. +struct Stencil { + indptr: Vec, + indices: Vec, + weights: Vec, +} + +pub struct SubdivMap { + /// CC5 subdivision level of the HD mesh (1 or 2). + pub level: u32, + /// HD global vertex count. + pub vertex_count: usize, + /// The slider pack's (level 0) global vertex count. + pub base_vertex_count: usize, + pub meshes: Vec, + stencils: Vec>, +} + +fn u32s(b: &[u8]) -> Vec { + b.chunks_exact(4).map(|c| u32::from_le_bytes(c.try_into().unwrap())).collect() +} + +impl SubdivMap { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("hd_map.json"); + let j: MapJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + let bin_path = dir.join("hd_map.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let mut stencils = Vec::with_capacity(j.meshes.len()); + for m in &j.meshes { + let Some([off, nnz]) = m.stencil else { + if m.count != m.base_count { + bail!("{}: {} HD vs {} base vertices but no stencil", m.name, m.count, m.base_count); + } + stencils.push(None); + continue; + }; + let (off, nnz, rows) = (off as usize, nnz as usize, m.count as usize); + let end = off + 4 * (rows + 1) + 8 * nnz; + if end > bin.len() { + bail!("{}: stencil past end of hd_map.bin", m.name); + } + let indptr = u32s(&bin[off..off + 4 * (rows + 1)]); + let i0 = off + 4 * (rows + 1); + let indices = u32s(&bin[i0..i0 + 4 * nnz]); + let weights = bin[i0 + 4 * nnz..end].chunks_exact(4).map(|c| f32::from_le_bytes(c.try_into().unwrap())).collect(); + if indptr.last().copied() != Some(nnz as u32) || indptr.windows(2).any(|w| w[0] > w[1]) { + bail!("{}: malformed stencil row pointers", m.name); + } + if let Some(bad) = indices.iter().find(|&&i| i >= m.base_count) { + bail!("{}: stencil column {bad} out of range {}", m.name, m.base_count); + } + stencils.push(Some(Stencil { indptr, indices, weights })); + } + let end = j.meshes.iter().map(|m| (m.offset + m.count) as usize).max().unwrap_or(0); + if end > j.vertex_count { + bail!("hd_map.json: meshes extend past vertex_count"); + } + Ok(Self { level: j.level, vertex_count: j.vertex_count, base_vertex_count: j.base_vertex_count, meshes: j.meshes, stencils }) + } + + /// Error unless this map was built for `pack`'s vertex space. + pub fn check(&self, pack: &ShapePack) -> Result<()> { + if self.base_vertex_count != pack.vertex_count { + bail!("HD map is for {} base vertices, pack has {}", self.base_vertex_count, pack.vertex_count); + } + for m in &self.meshes { + match pack.meshes.iter().find(|p| p.name == m.name) { + Some(p) if p.offset == m.base_offset && p.count == m.base_count => {} + _ => bail!("HD map mesh {} does not match the pack", m.name), + } + } + Ok(()) + } + + /// HD global vertex offset of a mesh (by glTF mesh name). + pub fn mesh_offset(&self, name: &str) -> Option { + self.meshes.iter().find(|m| m.name == name).map(|m| m.offset) + } + + /// Per-vertex values (deltas) in the base space -> the HD space. + pub fn expand_verts(&self, base: &[[f32; 3]]) -> Vec<[f32; 3]> { + assert_eq!(base.len(), self.base_vertex_count, "one entry per base vertex"); + let mut out = vec![[0.0; 3]; self.vertex_count]; + for (m, st) in self.meshes.iter().zip(&self.stencils) { + let src = &base[m.base_offset as usize..(m.base_offset + m.base_count) as usize]; + let dst = &mut out[m.offset as usize..(m.offset + m.count) as usize]; + match st { + None => dst.copy_from_slice(src), + Some(s) => { + for (r, d) in dst.iter_mut().enumerate() { + let (a, b) = (s.indptr[r] as usize, s.indptr[r + 1] as usize); + let mut acc = [0.0f32; 3]; + for k in a..b { + let (w, v) = (s.weights[k], src[s.indices[k] as usize]); + if v != [0.0; 3] { + acc[0] += w * v[0]; + acc[1] += w * v[1]; + acc[2] += w * v[2]; + } + } + *d = acc; + } + } + } + } + out + } + + /// A base-space [`ShapeDelta`] (from `ShapePack::evaluate`) in the HD space; + /// joints are unchanged. + pub fn expand(&self, d: &ShapeDelta) -> ShapeDelta { + ShapeDelta { verts: self.expand_verts(&d.verts), joints: d.joints.clone() } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Mesh "A" (2 verts) maps 1:1; mesh "B" (2 base verts) subdivides into 3. + fn map(dir: &Path) -> SubdivMap { + let mut bin = Vec::new(); + for v in [0u32, 1, 2, 4] { + bin.extend(v.to_le_bytes()); // indptr: rows {0}, {1}, {0, 1} + } + for v in [0u32, 1, 0, 1] { + bin.extend(v.to_le_bytes()); + } + for w in [1.0f32, 1.0, 0.5, 0.5] { + bin.extend(w.to_le_bytes()); + } + std::fs::write(dir.join("hd_map.bin"), bin).unwrap(); + std::fs::write( + dir.join("hd_map.json"), + r#"{"level": 1, "vertex_count": 5, "base_vertex_count": 4, "meshes": [ + {"name": "A", "offset": 0, "count": 2, "base_offset": 0, "base_count": 2, "stencil": null}, + {"name": "B", "offset": 2, "count": 3, "base_offset": 2, "base_count": 2, "stencil": [0, 4]}]}"#, + ) + .unwrap(); + SubdivMap::load(dir).unwrap() + } + + #[test] + fn expands_identity_and_stencil_meshes() { + let dir = tempfile::tempdir().unwrap(); + let m = map(dir.path()); + let out = m.expand_verts(&[[1.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 4.0, 2.0]]); + assert_eq!(out, vec![[1.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 4.0, 2.0], [0.0, 3.0, 1.0]]); + assert_eq!(m.mesh_offset("B"), Some(2)); + } + + #[test] + fn rejects_out_of_range_columns() { + let dir = tempfile::tempdir().unwrap(); + map(dir.path()); + let mut bin = std::fs::read(dir.path().join("hd_map.bin")).unwrap(); + bin[16..20].copy_from_slice(&7u32.to_le_bytes()); + std::fs::write(dir.path().join("hd_map.bin"), bin).unwrap(); + let err = SubdivMap::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/docs/design/avatar-editor.md b/docs/design/avatar-editor.md new file mode 100644 index 0000000..ddb767f --- /dev/null +++ b/docs/design/avatar-editor.md @@ -0,0 +1,148 @@ +# Avatar shape editor (CC5 sliders outside CC5) + +`examples/avatar_editor.rs` reshapes a Character Creator 5 character with CC5's own +Morphs-panel sliders (body type, height, face features — about 3,400 for a CC3+/HD +character) without CC5. `examples/shape_avatar.rs` does the same headless from a +weights JSON. Both write a shaped GLB for the normal avatar pipeline. + +## How it works + +CC5 is used once, unattended, to record what every slider does to *this* +character (`remotemedia-sdk/scripts/avatars/cc5_capture_sliders.py`): + +- **bind mode**: per slider state, the displayed bind-pose mesh from CC5's + subdivision-0 OBJ export. This already contains the skeletal stretch — many + sliders act through the skeleton (Character Height moves CC5's internal mesh + <1 cm but the bones ~12 cm). +- **capture mode**: per slider state, skin-bone positions (CC5 live pose). The + pack builder maps them into the bind pose (`bind_joints.py`: chain rebuilt + with per-bone pose rotations, grounding anchored on rigidly skinned joints). + +`build_editor_pack.py` turns that into `pack.json` + `pack.bin` +([`cc5_shape::ShapePack`](../../crates/cc5-shape/src/pack.rs); the +[`cc5-shape`](../../crates/cc5-shape/README.md) crate is also a Python module, +`import cc5_shape`): sparse vertex and joint +deltas in the glTF frame. The reference GLB (Blender conversion of CC5's FBX +export, `fbx_to_glb.py`) carries a `_CCIDX` vertex attribute, the CC5 vertex +index of every glTF vertex, so seam-split vertices map back exactly. + +A slider mix is `bind + Σ tᵢ·Δᵢ` for vertices and joints, `tᵢ` linear on each +side of the slider's captured value. On CC5's internal (pre-skinning) mesh, +sliders are exactly linear and additive (0 error for a half weight and for two +sliders combined); for sliders that move bones, the bind-pose shape includes +skinning, so mixes are a close linear approximation there. +[`shape_glb`](../../crates/cc5-shape/src/glb.rs) writes it into a GLB: positions, +joint translations (rotations unchanged), and inverse bind matrices +`IBM' = W'⁻¹·W·IBM` so the new bind mesh renders exactly at bind. + +## Building the inputs + +On the Windows machine with CC5 (5.13), Blender 4.3 and Python: + +```powershell +remotemedia-sdk\scripts\avatars\cc5_editor\build_cc5_editor_pack.ps1 ` + -Project C:\Users\me\Documents\avatars\Beautiful.ccProject -OutDir D:\cc5_editor\beautiful +``` + +It drives CC5 with no clicks (loads CC5's slider library via "Load CC5 HD +Neutral Base" — without that step CC5 exposes no head/body sliders to Python — +opens the project, captures, exports the reference FBX), converts with Blender, +bakes hair/brow alpha (`bake_alpha.py`) and builds the pack. The first run takes +hours (bind capture ≈ 3.5 s per slider state); reruns resume. Any running CC5 +is closed first; the project is never saved. + +## Running + +``` +cargo run --example avatar_editor -- D:\cc5_editor\beautiful\character.glb D:\cc5_editor\beautiful\pack [shaped.glb] +cargo run --example shape_avatar -- character.glb pack weights.json shaped.glb +``` + +Editor: category tree with search, per-slider reset, "Reset all", "Save GLB"; +with `--expressions `, an "Expressions" section (see below). +Right-drag orbits, middle-drag pans, wheel zooms. `weights.json` maps slider +names or ids to values (`{"Character Height": 0.5}`). + +## Accuracy (Beautiful / HD Ariana, Character Height = 1 vs CC5's own FBX export) + +| | error | +|---|---| +| bind mesh, 67,419 vertices | 0.314 cm, uniform (CC5's OBJ and FBX exports ground the character 3 mm apart); shape itself exact | +| joints | median 0.35 cm, max 0.79 cm (mostly the same 3 mm shift; fingers worst) | +| skin consistency `W·IBM` | 1e-6 | + +For scale, the slider moves vertices up to 12.3 cm and joints up to 11.8 cm. + +## Facial expressions (CPU morphing) + +A CC5 HD character carries ~400 facial morph targets per face mesh (body 349, +brows/lashes 395, tear line 437...). Bevy 0.15 refuses meshes with more than 64 +(`MAX_MORPH_WEIGHTS`), so the reference GLB has none (`fbx_to_glb.py` drops them) +and the expressions live in a sidecar *expression pack*, applied on the CPU like +the shaping sliders: + +```powershell +# Blender headless (re-runs itself under Blender when started with plain Python) +python remotemedia-sdk\scripts\avatars\cc5_editor\build_expression_pack.py ` + D:\cc5_editor\beautiful\ref\character.fbx D:\cc5_editor\beautiful\pack D:\cc5_editor\beautiful\expressions +cargo run --example avatar_editor -- character.glb pack --expressions expressions +``` + +`expressions.json` + `expressions.bin` ([`cc5_shape::ExpressionPack`](../../crates/cc5-shape/src/expr.rs)): +per shape-key name, the sparse delta `key − Basis` at weight 1, merged over every +mesh that has that key, in the slider pack's global vertex space and glTF frame +(the FBX's vertex order is CC5's). Beautiful: 480 expressions (56 empty +correctives, e.g. `Mouth_Lips_Together_*`, which CC5 exports as zero), 785k +vertex deltas, 12.6 MB. A mix is `bind + shape Δ + Σ wᵢ·exprᵢ`; morphing is +linear, so this is exact — against Blender evaluating the FBX's shape keys +(one key at 1, one at 0.5, two 3-key mixes across body, lashes, brows, tear +line, eye occlusion, eyes and tongue; up to 29 mm of motion) the max error is +1.2e-4 mm. + +The editor's "Expressions" section (0..1 sliders, shares the search box) layers +expressions on the shaped positions in `apply_shape`; an expression-only change +rewrites positions without touching joints. Expressions are a preview there: +"Save GLB" and recipes hold the shape only. Only morphs are covered: CC5's jaw +and eye *bones* (Jaw_Open also rotates CC_Base_JawRoot in CC5) are not part of the +pack, exactly as with glTF morph targets. `cc5_shape::ArkitMap` resolves ARKit-52 +weights (Audio2Face) through a runtime `*.arkit_map*.json` (`{"mapping": +{arkitName: [{morph, weight, meshes}]}}`; `meshes` is ignored since an expression +spans all meshes). + +### Runtime: meshes with more than 64 morphs (design, not implemented) + +cc-render's runtime (`src/cc_render/bevy_app`) drives morph weights per frame +from Audio2Face ARKit weights. For a character whose GLB exceeds 64 morphs per +mesh it would switch that mesh to the CPU path: + +1. Load the GLB without morphs (as now) and the character's expression pack + + ARKit map. At spawn, record per primitive what the editor records: base + positions, `_CCIDX` globals, and the bind-space rotation (`MeshRec`). +2. Per frame: ARKit weights → `ArkitMap::weights` → `ExpressionPack::add_to` + into a reused per-character buffer (zero only the vertices touched last frame), + then rewrite `ATTRIBUTE_POSITION` of the affected primitives only (face, brows, + lashes, tear line, occlusion, eyes, tongue — not body-wide). Cost is linear in + the active expressions' vertex counts (typically 20–50 active ARKit targets, + a few ×10k vertex adds) plus the position upload (~15k body vertices ≈ 180 KB), + well within a frame. Skip the write when weights are unchanged. +3. Normals: morph targets in the GLB carry normal deltas; the CPU path would + either recompute face normals on the touched region or (simplest first) keep + bind normals, as the editor does. Bone-driven parts (jaw, eyes) stay on the + skeleton as today. +4. Keep the GPU morph path for characters with ≤ 64 targets; pick per mesh at + load (count targets in the glTF, or the presence of an expression pack). + +Alternatives considered: splitting a mesh into ≤64-target primitives breaks +along seams and still caps per primitive; a custom storage-buffer morph shader +avoids the CPU upload but needs a forked skinning/morph pipeline in Bevy 0.15. + +## Limits + +- The reference GLB drops CC5's ~400 facial morph targets (Bevy loads at most + 64); they are applied on the CPU from the expression pack instead (above). + `fbx_to_glb.py --keep-morphs` keeps them for other consumers. +- Joint *rotations* are not captured; sliders are applied as joint translations. +- Normals are not recomputed after reshaping. +- The pack is specific to the character it was captured from. +- The deltas are derived from Reallusion's slider content packs; check the + Reallusion content licence before distributing packs or shaped characters. diff --git a/examples/avatar_editor.rs b/examples/avatar_editor.rs new file mode 100644 index 0000000..d6f31eb --- /dev/null +++ b/examples/avatar_editor.rs @@ -0,0 +1,1123 @@ +//! CC5-style avatar shape editor: the sliders captured from Character Creator 5, +//! applied live to the character outside CC5. +//! +//! Usage: avatar_editor character.glb pack_dir [shaped_out.glb] [--recipe recipe.json] +//! [--style other_character.glb] [--expressions expressions_dir] +//! +//! `character.glb` is the CC5 reference export converted with `fbx_to_glb.py` +//! (it carries the `_CCIDX` vertex attribute); `pack_dir` holds `pack.json` and +//! `pack.bin` from `build_editor_pack.py`. Every slider change re-evaluates the +//! mix and rewrites the bind mesh, joint translations and inverse bind matrices +//! the same way `shape_glb` does; "Save GLB" writes that as a GLB. A recipe (the +//! changed sliders as JSON, e.g. from `cc5_shape.fit`) can be opened with +//! `--recipe` or loaded/saved from the panel. +//! +//! `--style` loads another CC character (a GLB, e.g. the one a recipe was fitted +//! to) for two optional view layers: "Surface" puts its skin / eye / teeth +//! materials on our CC_Base_* meshes (same CC UV layout, matched by material +//! name); "Assets" shows its hair, clothes and accessories, bound to *our* +//! skeleton by joint name so they follow our shape, in place of ours. Its body is +//! never shown. +//! +//! `--expressions` loads the character's facial expression pack +//! (`build_expression_pack.py`: CC5's ~400 morph targets, more than Bevy's 64 per +//! mesh) for an "Expressions" section: 0..1 sliders applied on the CPU on top of +//! the shape (positions = bind + shape delta + sum of w_i * expression_i). +//! Expressions are a preview layer: "Save GLB" and recipes hold the shape only. +//! +//! `--hd map_dir` edits CC5's HD (subdivided) character instead: `character.glb` +//! is then the HD export (`build_cc5_editor_pack.ps1 -HdLevel 1`: `hd/character_hd.glb`) +//! and `map_dir` its `hd_map.json` + `.bin`, which spreads the pack's (level 0) +//! slider and expression moves over the HD mesh. "Save GLB" writes the HD GLB. +//! +//! Controls: right-drag orbits, middle-drag pans, wheel zooms towards the cursor +//! (down to 2 cm); Head / Body / Feet buttons frame those parts. +//! +//! `--snapshot out.png [--view head|body|feet] [--yaw deg] [--pitch deg] [--zoom k] +//! [--pan right,up (m)] [--size WxH] [--no-cc5]` renders one frame without the panel once the character +//! and its CC5 materials are in, writes it to `out.png` and exits (for comparing +//! the look against CC5's viewport). `--ev100` sets the exposure as usual. +use bevy::gltf::{Gltf, GltfPlugin}; +use bevy::input::mouse::{AccumulatedMouseMotion, AccumulatedMouseScroll}; +use bevy::core_pipeline::{bloom::Bloom, tonemapping::Tonemapping, Skybox}; +use bevy::core_pipeline::experimental::taa::{TemporalAntiAliasPlugin, TemporalAntiAliasing}; +use bevy::pbr::{ScreenSpaceAmbientOcclusion, ShadowFilteringMethod}; +use bevy::math::{Affine3A, Mat3}; +use bevy::pbr::{light_consts::lux, CascadeShadowConfigBuilder, DirectionalLightShadowMap}; +use bevy::prelude::*; +use bevy::render::camera::Exposure; +use bevy::render::view::{ColorGrading, ColorGradingGlobal}; +use bevy::render::mesh::skinning::{SkinnedMesh, SkinnedMeshInverseBindposes}; +use bevy::render::mesh::{MeshVertexAttribute, VertexAttributeValues}; +use bevy::render::render_resource::VertexFormat; +use bevy::picking::mesh_picking::ray_cast::{MeshRayCast, RayCastSettings}; +use bevy::render::view::NoFrustumCulling; +use bevy_egui::{egui, EguiContexts, EguiPlugin}; +use cc_render_plugin::cc_render::shape::{ShapeDelta, ShapePack}; +use cc5_shape::{shape_glb_hd, ExpressionPack, SubdivMap}; +use cc_render_plugin::cc_render::shape_glb::shape_glb; +use cc_render_plugin::cc_render::cc5_materials::{Cc5Material, Cc5Materials, Cc5MaterialsPlugin, Cc5MaterialsSet}; +use std::collections::HashMap; +use std::path::PathBuf; + +const ATTRIBUTE_CCIDX: MeshVertexAttribute = MeshVertexAttribute::new("CcIdx", 0x0CC5_1D_E7, VertexFormat::Float32); + +#[derive(Resource)] +struct Editor { + /// Camera exposure (EV100); the portrait rig is lit for studio levels, not daylight. + ev100: f32, + pack: ShapePack, + categories: Vec<(String, Vec)>, + weights: Vec, + delta: ShapeDelta, + dirty: bool, + /// `--expressions`: the pack and one weight per expression (0 = neutral). + expr: Option, + expr_weights: Vec, + /// Only expression weights changed: positions are rewritten, joints are not. + expr_dirty: bool, + /// `--hd`: the character is CC5's HD mesh; vertex deltas go through this map. + hd: Option, + search: String, + glb_path: PathBuf, + out_path: PathBuf, + recipe_path: String, + status: String, + /// Framing requested by the UI: (joints whose midpoint to aim at, camera distance); + /// no joints = whole body. + focus_request: Option<(&'static [&'static str], f32)>, +} + +/// Base state of the spawned character, captured once it has settled. +#[derive(Resource, Default)] +struct Rig { + ready: bool, + settle_frames: u32, + meshes: Vec, + joints: Vec, + skins: Vec, +} + +struct MeshRec { + handle: Handle, + base: Vec<[f32; 3]>, + global: Vec, + to_mesh: Mat3, +} + +struct JointRec { + entity: Entity, + name: String, + pack: Option, + parent: Option, + base_local: Vec3, + parent_lin_inv: Mat3, + base_world: Affine3A, +} + +struct SkinRec { + handle: Handle, + joints: Vec, + base: Vec, +} + +/// Root of the character being edited (the pack's character). +#[derive(Component)] +struct OurCharacter; + +/// Root of the `--style` character. +#[derive(Component)] +struct StyleCharacter; + +/// `--style` view layers. +#[derive(Resource, Default)] +struct Style { + path: Option, + our_gltf: Handle, + style_gltf: Handle, + ready: bool, + surface: bool, + assets: bool, + applied: Option<(bool, bool)>, + /// Our CC_Base_* primitives: entity, original material, its glTF material name. + our_base: Vec<(Entity, Handle, Option)>, + /// Our hair / clothes / accessories. + our_assets: Vec, + style_assets: Vec, + style_materials: HashMap>, + status: String, +} + +/// `--snapshot`: render a camera preset to a file, then exit. +#[derive(Resource)] +struct Snapshot { + path: PathBuf, + view: String, + yaw: f32, + pitch: f32, + zoom: f32, + /// Focus offset along the camera's right / up (m). + pan: Vec2, + /// Seconds since the rig was ready and the CC5 material count last changed. + stable: f32, + cc5_count: usize, + /// Frames since the materials settled. + step: u32, + /// Seconds since the camera preset was applied. + framed: f32, +} + +#[derive(Component)] +struct Orbit { + focus: Vec3, + yaw: f32, + pitch: f32, + dist: f32, +} + +/// Default camera exposure: Bevy's default (EV100 9.7, daylight) leaves the +/// portrait rig (a few hundred lux) far too dark. +const DEFAULT_EV100: f32 = 7.25; + +/// `--snapshot`: seconds to render the framed view before the shot (shader and +/// pipeline compilation for new views and materials is asynchronous). +const SNAPSHOT_SETTLE_SECS: f32 = 6.0; + +/// Size of the key light for its soft (PCSS) shadows. +const KEY_SOFT_SHADOW: f32 = 6.0; + +/// Vertical field of view: a portrait lens. Bevy's default 45 degrees distorts +/// close-ups of the face like a wide angle. +const FOV_DEG: f32 = 25.0; + +/// Camera distances below are framed for 45 degrees; scale them to FOV_DEG. +fn fov_scale() -> f32 { + (22.5f32.to_radians()).tan() / (FOV_DEG.to_radians() * 0.5).tan() +} + +fn main() -> anyhow::Result<()> { + let mut args: Vec = std::env::args().collect(); + let mut opt = |flag: &str| -> anyhow::Result> { + match args.iter().position(|a| a == flag) { + Some(i) if i + 1 < args.len() => Ok(Some(args.drain(i..i + 2).nth(1).unwrap())), + Some(_) => anyhow::bail!("{flag} needs a file"), + None => Ok(None), + } + }; + let recipe_arg = opt("--recipe")?; + let ev100: f32 = opt("--ev100")?.map(|v| v.parse()).transpose()?.unwrap_or(DEFAULT_EV100); + let style_arg = opt("--style")?.map(std::fs::canonicalize).transpose()?; + let expr = opt("--expressions")?.map(|d| ExpressionPack::load(std::path::Path::new(&d))).transpose()?; + let hd = opt("--hd")?.map(|d| SubdivMap::load(std::path::Path::new(&d))).transpose()?; + let snapshot_arg = opt("--snapshot")?; + let view = opt("--view")?.unwrap_or_else(|| "head".into()); + let deg = |v: Option| -> anyhow::Result { Ok(v.map(|v| v.parse::()).transpose()?.unwrap_or(0.0).to_radians()) }; + let (yaw, pitch) = (deg(opt("--yaw")?)?, deg(opt("--pitch")?)?); + let zoom: f32 = opt("--zoom")?.map(|v| v.parse()).transpose()?.unwrap_or(1.0); + let pan = match opt("--pan")? { + Some(p) => { + let (x, y) = p.split_once(',').ok_or_else(|| anyhow::anyhow!("--pan right,up"))?; + Vec2::new(x.parse()?, y.parse()?) + } + None => Vec2::ZERO, + }; + let size = opt("--size")?.unwrap_or_else(|| "1139x978".into()); + let (w, h) = size.split_once('x').ok_or_else(|| anyhow::anyhow!("--size WxH"))?; + let size = (w.parse::()?, h.parse::()?); + let no_cc5 = match args.iter().position(|a| a == "--no-cc5") { + Some(i) => { + args.remove(i); + true + } + None => false, + }; + anyhow::ensure!(args.len() == 3 || args.len() == 4, "character.glb pack_dir [shaped_out.glb] [--recipe recipe.json] required"); + let glb_path = std::fs::canonicalize(&args[1])?; + let pack = ShapePack::load(std::path::Path::new(&args[2]))?; + let out_path = args.get(3).map(PathBuf::from).unwrap_or_else(|| glb_path.with_extension("shaped.glb")); + let weights = match &recipe_arg { + Some(r) => pack.weights_from_recipe(&serde_json::from_slice(&std::fs::read(r)?)?)?, + None => pack.base_weights(), + }; + let delta = pack.evaluate(&weights); + if let Some(x) = &expr { + anyhow::ensure!( + x.vertex_count == pack.vertex_count, + "expression pack has {} vertices, slider pack {}: built for another character", + x.vertex_count, + pack.vertex_count + ); + } + if let Some(h) = &hd { + h.check(&pack).map_err(|e| anyhow::anyhow!("--hd map does not belong to this pack: {e:#}"))?; + } + let editor = Editor { + categories: pack.categories(), + status: format!("{} sliders in {} categories", pack.sliders.len(), pack.categories().len()), + pack, + weights, + delta, + dirty: false, + expr_weights: vec![0.0; expr.as_ref().map_or(0, |x| x.len())], + expr, + expr_dirty: false, + hd, + search: String::new(), + glb_path: glb_path.clone(), + focus_request: None, + ev100, + recipe_path: recipe_arg.unwrap_or_else(|| out_path.with_extension("recipe.json").to_string_lossy().into_owned()), + out_path, + }; + let asset_dir = glb_path.parent().unwrap().to_string_lossy().to_string(); + let mut window = Window { title: "CC5 avatar editor".into(), ..default() }; + if snapshot_arg.is_some() { + window.resolution = bevy::window::WindowResolution::new(size.0, size.1).with_scale_factor_override(1.0); + } + let snap = snapshot_arg.map(|p| Snapshot { + path: PathBuf::from(p), + view, + yaw, + pitch, + zoom, + pan, + stable: 0.0, + cc5_count: 0, + step: 0, + framed: 0.0, + }); + let mut app = App::new(); + app.add_plugins( + DefaultPlugins + .set(WindowPlugin { primary_window: Some(window), ..default() }) + .set(AssetPlugin { file_path: asset_dir, ..default() }) + .set( + GltfPlugin::default() + .add_custom_vertex_attribute("CCIDX", ATTRIBUTE_CCIDX) + .add_custom_vertex_attribute("_CCIDX", ATTRIBUTE_CCIDX), + ), + ) + .add_plugins(EguiPlugin) + .add_plugins(TemporalAntiAliasPlugin) + .add_plugins(Cc5MaterialsPlugin) + .insert_resource(DirectionalLightShadowMap { size: 4096 }) + .insert_resource(editor) + .init_resource::() + .insert_resource(Style { path: style_arg, ..default() }) + .insert_resource(Cc5Materials { enabled: !no_cc5, ..default() }) + .add_systems(Startup, setup) + .add_systems( + Update, + ( + no_culling, + capture_rig, + style_setup, + ui.run_if(not(resource_exists::)), + style_apply, + apply_shape, + snapshot.run_if(resource_exists::), + orbit_camera, + ) + .chain() + .after(Cc5MaterialsSet), + ); + if let Some(s) = snap { + app.insert_resource(s); + } + app.run(); + Ok(()) +} + +/// Studio backdrop like CC5's viewport: soft grey-blue, darkening above eye level +/// (at the editor's 25 degree lens the top of a level close-up is only ~12 degrees +/// up). A small sRGB cubemap for `Skybox`: it stays out of the lighting and of +/// mesh picking. +fn studio_backdrop(images: &mut Assets) -> Handle { + use bevy::render::render_asset::RenderAssetUsages; + use bevy::render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureViewDescriptor, TextureViewDimension}; + const N: usize = 64; + let base = Vec3::new(84.0, 90.0, 104.0); + let mut data = Vec::with_capacity(N * N * 6 * 4); + for face in 0..6 { + for y in 0..N { + for x in 0..N { + let s = (x as f32 + 0.5) / N as f32 * 2.0 - 1.0; + let t = (y as f32 + 0.5) / N as f32 * 2.0 - 1.0; + // cube face order +X -X +Y -Y +Z -Z, t pointing down the face + let dir = match face { + 0 => Vec3::new(1.0, -t, -s), + 1 => Vec3::new(-1.0, -t, s), + 2 => Vec3::new(s, 1.0, t), + 3 => Vec3::new(s, -1.0, -t), + 4 => Vec3::new(s, -t, 1.0), + _ => Vec3::new(-s, -t, -1.0), + } + .normalize(); + let top = ((dir.y + 0.02) / 0.26).clamp(0.0, 1.0); + let c = base * (1.0 - 0.5 * top * top * (3.0 - 2.0 * top)); + data.extend_from_slice(&[c.x as u8, c.y as u8, c.z as u8, 255]); + } + } + } + let mut image = Image::new( + Extent3d { width: N as u32, height: N as u32, depth_or_array_layers: 6 }, + TextureDimension::D2, + data, + TextureFormat::Rgba8UnormSrgb, + RenderAssetUsages::RENDER_WORLD, + ); + image.texture_view_descriptor = Some(TextureViewDescriptor { dimension: Some(TextureViewDimension::Cube), ..default() }); + images.add(image) +} + +/// Backdrop luminance (cd/m²) for a texel of 1.0, such that the backdrop shows +/// the same at any exposure (it is a backdrop, not part of the lit scene). +fn backdrop_brightness(ev100: f32) -> f32 { + // Bevy's exposure: 1 / (1.2 * 2^ev100); 0.55 = calibrated against CC5's viewport. + 0.4 * 1.2 * 2f32.powf(ev100) +} + +fn setup( + mut commands: Commands, + assets: Res, + editor: Res, + mut style: ResMut