diff --git a/.gitignore b/.gitignore index 7c765e7..479ca98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target/ Cargo.lock *.bk +target-*/ diff --git a/Cargo.toml b/Cargo.toml index 4ca45ab..c8f1f56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ publish = false # bevy / wgpu / bevy_rapier3d deps don't unify against the host's, and # heavy renderer stack compiles in isolation. [workspace] +# `cc5-shape`: CC5 shaping sliders (Rust + Python); `cc5-hair`: CC5 hair -> strand parts. +members = ["crates/cc5-shape", "crates/cc5-hair", "crates/cc5-fbx", "crates/cc5-skin"] [profile.release] opt-level = 3 @@ -50,7 +52,8 @@ tracing = "0.1" # explicitly carry that default set here so the standalone-workspace # build doesn't drop `x11`, `multi_threaded`, etc. and break under # winit / wgpu. -bevy = { version = "0.15", features = ["jpeg", "tonemapping_luts", "ktx2", "zstd"] } +# experimental_pbr_pcss: soft (PCSS) directional shadows for the avatar editor's studio key light. +bevy = { version = "0.15", features = ["jpeg", "tonemapping_luts", "ktx2", "zstd", "experimental_pbr_pcss"] } # bevy_rapier3d shipped unconditionally — no nested cargo features. bevy_rapier3d = { version = "=0.28.0", default-features = false, features = ["dim3", "async-collider"] } # wgpu pinned to the version Bevy 0.15.3 bundles. Aliased to `wgpu_pin` @@ -66,3 +69,15 @@ crossbeam-channel = "0.5" # a few regexes — bring in `regex` directly. regex = "1" tempfile = "3.27.0" +# CC5 shaping sliders (avatar editor); also a standalone crate + Python module. +cc5-shape = { path = "crates/cc5-shape" } + +[dev-dependencies] +# `examples/avatar_editor.rs` UI; 0.32 is the Bevy 0.15 line. +bevy_egui = "0.32" +# live hair editing in the avatar editor (CC5 hair -> strands) +cc5-hair = { path = "crates/cc5-hair" } +# avatar editor: skin layers / skins captured from CC5 +cc5-skin = { path = "crates/cc5-skin" } +# avatar editor: GPU-ribbon hair expansion in parallel (already in the tree via cc5-shape) +rayon = "1" diff --git a/crates/cc5-fbx/Cargo.toml b/crates/cc5-fbx/Cargo.toml new file mode 100644 index 0000000..8eb2a22 --- /dev/null +++ b/crates/cc5-fbx/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "cc5-fbx" +version = "0.1.0" +edition = "2021" +publish = false +description = "Read Character Creator 5 FBX exports (ufbx) and write the GLBs cc-render loads, without Blender." + +[lib] +name = "cc5_fbx" + +[[bin]] +name = "cc5-fbx" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +# 0.29 = the glam Bevy 0.15 uses +glam = "0.29" +serde = "1" +serde_json = { version = "1", features = ["preserve_order"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +rayon = "1" +ufbx = "0.11" diff --git a/crates/cc5-fbx/src/cc5json.rs b/crates/cc5-fbx/src/cc5json.rs new file mode 100644 index 0000000..9ab28c6 --- /dev/null +++ b/crates/cc5-fbx/src/cc5json.rs @@ -0,0 +1,194 @@ +//! CC5's export sidecar JSON (next to the FBX): material tints and opacity +//! (bake_alpha.py's authoritative first pass) and the CC5 shader settings and +//! maps as `extras.cc5` (glb_cc5_materials.py): +//! +//! ```json +//! {"shader": "RLHair", "maps": {"Hair Flow Map": "character.cc5maps/..._flow.png", ...}, +//! "srgb": ["..."], "vars": {...}, "sss": {...}, "wrinkle": {"maps": {...}, "rules": [...], ...}} +//! ``` + +use std::collections::HashMap; +use std::path::{Component, Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde_json::{json, Map, Value}; + +pub struct Cc5Json { + pub path: PathBuf, + /// Every material under Meshes (bake_alpha.py's `load_cc5_json`). + specs: HashMap, + /// Materials with a CC5 custom shader, anywhere in the file. + shaders: HashMap, +} + +/// Writes the maps `extras.cc5` names into one folder (TGA re-encoded as PNG). +pub struct Maps { + base: PathBuf, + dir: PathBuf, + done: HashMap, + pub missing: Vec, +} + +impl Maps { + /// Maps named relative to `json`'s folder, written into `dir`. + pub fn new(json: &Path, dir: PathBuf) -> Self { + Self { base: json.parent().unwrap_or(Path::new(".")).to_path_buf(), dir, done: HashMap::new(), missing: Vec::new() } + } + + /// The map's path relative to `dir`'s parent, or None when the file is missing. + fn get(&mut self, rel: &str) -> Result> { + let path = normalize(&self.base.join(rel.replace('\\', "/"))); + if let Some(r) = self.done.get(&path) { + return Ok(Some(r.clone())); + } + if !path.is_file() { + self.missing.push(path); + return Ok(None); + } + let (bytes, mime) = crate::glb::file_bytes(&path)?; + let stem = path.file_stem().unwrap().to_string_lossy(); + let fname = format!("{stem}.{}", if mime == "image/png" { "png" } else { "jpg" }); + std::fs::create_dir_all(&self.dir)?; + let out = self.dir.join(&fname); + // shared between rebuilds: rewrite only when it differs + if std::fs::metadata(&out).map(|m| m.len() as usize != bytes.len()).unwrap_or(true) { + std::fs::write(&out, &bytes).with_context(|| format!("writing {}", out.display()))?; + } + let r = format!("{}/{fname}", self.dir.file_name().unwrap().to_string_lossy()); + self.done.insert(path, r.clone()); + Ok(Some(r)) + } +} + +fn normalize(p: &Path) -> PathBuf { + let mut out = PathBuf::new(); + for c in p.components() { + match c { + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + c => out.push(c), + } + } + out +} + +/// Colour maps (sRGB); the rest are data. +fn is_srgb(slot: &str) -> bool { + let l = slot.to_lowercase(); + l.contains("basecolor") || l.contains("diffuse") || l == "sclera" || l.contains("color") +} + +fn clean(vars: Option<&Value>) -> Value { + let mut m = Map::new(); + if let Some(o) = vars.and_then(Value::as_object) { + for (k, v) in o { + if !k.to_lowercase().starts_with("padding") { + m.insert(k.clone(), v.clone()); + } + } + } + Value::Object(m) +} + +impl Cc5Json { + pub fn load(path: &Path) -> Result { + let data: Value = serde_json::from_slice(&std::fs::read(path).with_context(|| format!("reading {}", path.display()))?) + .with_context(|| format!("parsing {}", path.display()))?; + let mut specs = HashMap::new(); + if let Some((rig, r)) = data.as_object().and_then(|o| o.iter().next()) { + let obj = r.get("Object").and_then(Value::as_object); + let inner = obj.and_then(|o| o.get(rig).or_else(|| o.values().next())); + if let Some(meshes) = inner.and_then(|i| i.get("Meshes")).and_then(Value::as_object) { + for m in meshes.values() { + if let Some(mats) = m.get("Materials").and_then(Value::as_object) { + for (name, mat) in mats { + if mat.is_object() { + specs.insert(name.clone(), mat.clone()); + } + } + } + } + } + } + let mut shaders = HashMap::new(); + fn walk(v: &Value, out: &mut HashMap) { + match v { + Value::Object(o) => { + for (k, x) in o { + if x.get("Custom Shader").is_some() { + out.insert(k.clone(), x.clone()); + } else { + walk(x, out); + } + } + } + Value::Array(a) => a.iter().for_each(|x| walk(x, out)), + _ => {} + } + } + walk(&data, &mut shaders); + Ok(Self { path: path.to_path_buf(), specs, shaders }) + } + + /// CC5's diffuse tint and opacity; transparent families (`*_Transparency`, + /// Std_Eyelash) blend at full alpha (their maps carry it). + pub fn apply(&self, name: &str, factor: &mut [f32; 4], blend: &mut bool) { + if let Some(spec) = self.specs.get(name) { + if let Some(dc) = spec.get("Diffuse Color").and_then(Value::as_array).filter(|a| a.len() >= 3) { + for k in 0..3 { + factor[k] = (dc[k].as_f64().unwrap_or(255.0) as f32 / 255.0).clamp(0.0, 1.0); + } + } + if let Some(o) = spec.get("Opacity").and_then(Value::as_f64) { + factor[3] = (o as f32).clamp(0.0, 1.0); + } + if factor[3] < 0.999 { + *blend = true; + } + } + if name.ends_with("_Transparency") || name == "Std_Eyelash" { + *blend = true; + factor[3] = 1.0; + } + } + + /// `extras.cc5` for a material with a CC5 custom shader. + pub fn extras(&self, name: &str, maps: &mut Maps) -> Result> { + let Some(cm) = self.shaders.get(name) else { return Ok(None) }; + let cs = &cm["Custom Shader"]; + let mut info = json!({"shader": cs.get("Shader Name").and_then(Value::as_str).unwrap_or(""), "maps": {}, "srgb": [], + "vars": clean(cs.get("Variable"))}); + if let Some(imgs) = cs.get("Image").and_then(Value::as_object) { + for (slot, t) in imgs { + let Some(p) = t.get("Texture Path").and_then(Value::as_str).filter(|p| !p.is_empty()) else { continue }; + if let Some(r) = maps.get(p)? { + info["maps"][slot] = json!(r); + if is_srgb(slot) { + info["srgb"].as_array_mut().unwrap().push(json!(slot)); + } + } + } + } + if let Some(s) = cm.get("Subsurface Scatter").filter(|s| !s.is_null() && s.as_object().map_or(true, |o| !o.is_empty())) { + info["sss"] = s.clone(); + } + if let Some(w) = cm.get("Wrinkle") { + if let Some(tex) = w.get("Textures").and_then(Value::as_object).filter(|t| !t.is_empty()) { + let mut wm = Map::new(); + for (slot, t) in tex { + let Some(p) = t.get("Texture Path").and_then(Value::as_str).filter(|p| !p.is_empty()) else { continue }; + if let Some(r) = maps.get(p)? { + wm.insert(slot.clone(), json!(r)); + } + } + info["wrinkle"] = json!({"maps": wm, "rules": w.get("WrinkleRules").cloned().unwrap_or(json!([])), + "rule_weights": w.get("WrinkleRuleWeights").cloned().unwrap_or(json!([])), + "ease_strength": w.get("WrinkleEaseStrength").cloned().unwrap_or(Value::Null), + "overall_weight": w.get("WrinkleOverallWeight").cloned().unwrap_or(Value::Null)}); + } + } + Ok(Some(info)) + } +} diff --git a/crates/cc5-fbx/src/dump.rs b/crates/cc5-fbx/src/dump.rs new file mode 100644 index 0000000..4714f3b --- /dev/null +++ b/crates/cc5-fbx/src/dump.rs @@ -0,0 +1,67 @@ +//! The joints / weights / rest-position dump (as Blender's fbx_to_glb.py wrote +//! it), for the pack builders (build_editor_pack.py, build_hd_map.py) and +//! `cc5-hair regions`. In Blender's frame (Z up, metres): +//! +//! ```json +//! {"joints": {name: [x, y, z]}, // bind-pose joint positions +//! "meshes": {name: vertex count}, // control points +//! "mesh_order": [name, ...], +//! "parents": {name: parent or null}, +//! "weights": {mesh: [[[joint, w], ...] per vertex]}} +//! ``` +//! plus `.verts.f32`: every mesh's rest positions (f32 xyz), in `mesh_order`. + +use std::path::Path; + +use anyhow::{Context, Result}; +use glam::Vec3; +use serde_json::{json, Map, Value}; + +use crate::Character; + +/// Y-up -> Blender's Z-up. +fn z_up(p: Vec3) -> [f32; 3] { + [p.x, -p.z, p.y] +} + +fn num(x: f32) -> Value { + json!(x as f64) +} + +pub fn write(c: &Character, path: &Path) -> Result<()> { + let mut joints = Map::new(); + let mut parents = Map::new(); + for j in &c.joints { + joints.insert(j.name.clone(), json!(z_up(j.world.w_axis.truncate()).map(num))); + parents.insert(j.name.clone(), j.parent.map_or(Value::Null, |p| json!(c.joints[p].name))); + } + let mut meshes = Map::new(); + let mut weights = Map::new(); + let mut flat: Vec = Vec::new(); + for m in &c.meshes { + meshes.insert(m.name.clone(), json!(m.points.len())); + weights.insert( + m.name.clone(), + Value::Array( + m.weights + .iter() + .map(|ws| json!(ws.iter().map(|&(j, w)| json!([c.joints[j as usize].name, (w as f64 * 1e6).round() / 1e6])).collect::>())) + .collect(), + ), + ); + for p in &m.points { + for x in z_up(*p) { + flat.extend_from_slice(&x.to_le_bytes()); + } + } + } + let order: Vec<&str> = c.meshes.iter().map(|m| m.name.as_str()).collect(); + let out = json!({"joints": joints, "meshes": meshes, "mesh_order": order, "parents": parents, "weights": weights}); + let mut text = Vec::new(); + let mut ser = serde_json::Serializer::with_formatter(&mut text, serde_json::ser::PrettyFormatter::with_indent(b" ")); + serde::Serialize::serialize(&out, &mut ser)?; + std::fs::write(path, text).with_context(|| format!("writing {}", path.display()))?; + let vp = format!("{}.verts.f32", path.display()); + std::fs::write(&vp, flat).with_context(|| format!("writing {vp}"))?; + Ok(()) +} diff --git a/crates/cc5-fbx/src/expressions.rs b/crates/cc5-fbx/src/expressions.rs new file mode 100644 index 0000000..b310653 --- /dev/null +++ b/crates/cc5-fbx/src/expressions.rs @@ -0,0 +1,86 @@ +//! The avatar editor's expression pack (expressions.json + expressions.bin) from +//! the reference FBX's facial morphs, for CPU morphing (`cc5_shape::ExpressionPack`); +//! was build_expression_pack.py (Blender). +//! +//! CC5 exports ~400 facial morphs per face mesh; Bevy loads at most 64 morph +//! targets per mesh, so the editor applies them on the CPU instead. +//! +//! - expressions.json: `{"version": 1, "vertex_count", "meshes" (as pack.json), +//! "expressions": [{name, meshes: [mesh names it moves], verts: [byte_offset, count]}]}`, +//! one entry per morph name merged over every mesh that has it (body first: +//! its morphs are the canonical list); +//! - expressions.bin: per block u32 index[count] then f32 delta[count * 3]: +//! morph at weight 1 minus the rest shape, glTF frame (metres, Y up), global +//! (pack) vertex index. + +use std::io::Write; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde_json::{json, Value}; + +use crate::Character; + +/// Deltas smaller than this (metres) are dropped. +const EPS: f32 = 1e-6; + +pub fn write(c: &Character, pack_dir: &Path, out_dir: &Path) -> Result<()> { + let pack: Value = serde_json::from_slice(&std::fs::read(pack_dir.join("pack.json")).with_context(|| format!("reading {}", pack_dir.join("pack.json").display()))?)?; + let offsets: std::collections::HashMap<&str, (u64, u64)> = pack["meshes"] + .as_array() + .context("pack.json: no meshes")? + .iter() + .filter_map(|m| Some((m["name"].as_str()?, (m["offset"].as_u64()?, m["count"].as_u64()?)))) + .collect(); + // body first, then by name (as the Blender script) + let mut meshes: Vec<_> = c.meshes.iter().filter(|m| !m.shapes.is_empty()).collect(); + meshes.sort_by_key(|m| m.name != "CC_Base_Body"); + // {name: [(global index, delta), mesh]} in first-seen order + let mut order: Vec = Vec::new(); + let mut found: std::collections::HashMap, String)>> = Default::default(); + for m in &meshes { + let Some(&(off, count)) = offsets.get(m.name.as_str()) else { + eprintln!("skipping {}: not in the slider pack", m.name); + continue; + }; + if m.points.len() as u64 != count { + bail!("{}: {} vertices, pack has {count}", m.name, m.points.len()); + } + for s in &m.shapes { + let d: Vec<(u32, [f32; 3])> = s.deltas.iter().filter(|(_, d)| d.abs().max_element() >= EPS).map(|(i, d)| (*i + off as u32, d.to_array())).collect(); + if !found.contains_key(&s.name) { + order.push(s.name.clone()); + } + found.entry(s.name.clone()).or_default().push((d, m.name.clone())); + } + } + std::fs::create_dir_all(out_dir)?; + let mut bin = std::io::BufWriter::new(std::fs::File::create(out_dir.join("expressions.bin"))?); + let (mut pos, mut total) = (0u64, 0usize); + let mut exprs = Vec::new(); + for name in &order { + let parts = &found[name]; + let mut all: Vec<(u32, [f32; 3])> = parts.iter().flat_map(|(d, _)| d.iter().copied()).collect(); + all.sort_by_key(|x| x.0); // stable: as numpy's argsort(kind="stable") + for (i, _) in &all { + bin.write_all(&i.to_le_bytes())?; + } + for (_, d) in &all { + for x in d { + bin.write_all(&x.to_le_bytes())?; + } + } + let meshes: Vec<&str> = parts.iter().filter(|(d, _)| !d.is_empty()).map(|(_, m)| m.as_str()).collect(); + exprs.push(json!({"name": name, "meshes": meshes, "verts": [pos, all.len()]})); + pos += all.len() as u64 * 16; + total += all.len(); + } + bin.flush()?; + let out = json!({"version": 1, "units": "m", "frame": "glTF Y-up (cc5-fbx)", + "source": c.path.file_name().map(|f| f.to_string_lossy().into_owned()).unwrap_or_default(), + "vertex_count": pack["vertex_count"], "meshes": pack["meshes"], "expressions": exprs}); + std::fs::write(out_dir.join("expressions.json"), serde_json::to_string_pretty(&out)?)?; + let empty = exprs.iter().filter(|e| e["verts"][1] == 0).count(); + println!("expressions: {} ({} move nothing) over {} meshes, {} vertex deltas, {:.1} MB", exprs.len(), empty, meshes.len(), total, pos as f64 / 1e6); + Ok(()) +} diff --git a/crates/cc5-fbx/src/glb.rs b/crates/cc5-fbx/src/glb.rs new file mode 100644 index 0000000..d51d087 --- /dev/null +++ b/crates/cc5-fbx/src/glb.rs @@ -0,0 +1,329 @@ +//! The GLB writer. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use glam::Vec3; +use rayon::prelude::*; +use serde_json::{json, Value}; + +use crate::cc5json::{Cc5Json, Maps}; +use crate::Character; + +#[derive(Default)] +pub struct GlbOptions { + /// CC5's sidecar JSON: its tints and opacity (bake_alpha.py's first pass) and + /// `extras.cc5` (glb_cc5_materials.py). + pub cc5_json: Option, + /// Where the `extras.cc5` maps are written (default: next to the JSON, + /// `.cc5maps`); referenced as `/`. + pub maps_dir: Option, +} + +#[derive(Default)] +struct Buf { + bin: Vec, + views: Vec, + accessors: Vec, +} + +const ARRAY: u32 = 34962; +const ELEMENTS: u32 = 34963; + +impl Buf { + fn view(&mut self, bytes: &[u8], target: Option) -> usize { + while self.bin.len() % 4 != 0 { + self.bin.push(0); + } + let mut v = json!({"buffer": 0, "byteOffset": self.bin.len(), "byteLength": bytes.len()}); + if let Some(t) = target { + v["target"] = json!(t); + } + self.bin.extend_from_slice(bytes); + self.views.push(v); + self.views.len() - 1 + } + + fn accessor(&mut self, bytes: &[u8], component: u32, count: usize, ty: &str, target: Option) -> usize { + let view = self.view(bytes, target); + self.accessors.push(json!({"bufferView": view, "componentType": component, "count": count, "type": ty})); + self.accessors.len() - 1 + } + + fn f32s(&mut self, v: &[f32], width: usize, ty: &str, target: Option) -> usize { + let bytes: Vec = v.iter().flat_map(|x| x.to_le_bytes()).collect(); + self.accessor(&bytes, 5126, v.len() / width, ty, target) + } + + fn vec3s(&mut self, v: &[Vec3], min_max: bool) -> usize { + let flat: Vec = v.iter().flat_map(|p| p.to_array()).collect(); + let a = self.f32s(&flat, 3, "VEC3", Some(ARRAY)); + if min_max && !v.is_empty() { + let (lo, hi) = v.iter().fold((Vec3::MAX, Vec3::MIN), |(lo, hi), p| (lo.min(*p), hi.max(*p))); + self.accessors[a]["min"] = json!(lo.to_array()); + self.accessors[a]["max"] = json!(hi.to_array()); + } + a + } +} + +/// What a material's base colour image is made of. +#[derive(Clone, PartialEq, Eq, Hash)] +enum Img { + /// A file as it is (JPEG / PNG; anything else re-encoded as PNG). + File(PathBuf), + /// Diffuse RGB (white without one) + opacity as alpha, PNG (as Blender's exporter). + Merge(Option, PathBuf), +} + +fn stem(p: &Path) -> String { + p.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default() +} + +fn png(img: &image::DynamicImage) -> Result> { + use image::codecs::png::{CompressionType, FilterType, PngEncoder}; + let mut out = Vec::new(); + img.write_with_encoder(PngEncoder::new_with_quality(&mut out, CompressionType::Fast, FilterType::Adaptive))?; + Ok(out) +} + +fn open(p: &Path) -> Result { + if p.extension().is_some_and(|e| e.eq_ignore_ascii_case("tga")) { + let b = std::fs::read(p).with_context(|| format!("reading {}", p.display()))?; + return crate::tga::decode(&b).with_context(|| format!("decoding {}", p.display())); + } + image::open(p).with_context(|| format!("reading {}", p.display())) +} + +impl Img { + fn name(&self) -> String { + match self { + Img::File(p) => stem(p), + Img::Merge(Some(d), o) => format!("{}-{}", stem(d), stem(o)), + Img::Merge(None, o) => stem(o), + } + } + + /// (bytes, MIME type) + fn encode(&self) -> Result<(Vec, &'static str)> { + match self { + Img::File(p) => file_bytes(p), + Img::Merge(d, o) => { + use image::imageops::{resize, FilterType}; + let mut op = open(o)?.to_luma8(); + let mut rgb = match d { + Some(d) => Some(open(d)?.to_rgb8()), + None => None, + }; + let (w, h) = rgb.as_ref().map_or(op.dimensions(), |c| (c.width().max(op.width()), c.height().max(op.height()))); + if op.dimensions() != (w, h) { + op = resize(&op, w, h, FilterType::CatmullRom); + } + if let Some(c) = &mut rgb { + if c.dimensions() != (w, h) { + *c = resize(c, w, h, FilterType::CatmullRom); + } + } + let out = image::RgbaImage::from_fn(w, h, |x, y| { + let c = rgb.as_ref().map_or([255; 3], |c| c.get_pixel(x, y).0); + image::Rgba([c[0], c[1], c[2], op.get_pixel(x, y).0[0]]) + }); + Ok((png(&image::DynamicImage::ImageRgba8(out))?, "image/png")) + } + } + } +} + +/// An image file's bytes as glTF takes them: JPEG / PNG as they are, others as PNG. +pub(crate) fn file_bytes(p: &Path) -> Result<(Vec, &'static str)> { + let ext = p.extension().map(|e| e.to_string_lossy().to_lowercase()).unwrap_or_default(); + match ext.as_str() { + "jpg" | "jpeg" => Ok((std::fs::read(p).with_context(|| format!("reading {}", p.display()))?, "image/jpeg")), + "png" => Ok((std::fs::read(p).with_context(|| format!("reading {}", p.display()))?, "image/png")), + _ => Ok((png(&open(p)?)?, "image/png")), + } +} + +pub fn write(c: &Character, opts: &GlbOptions) -> Result> { + let mut b = Buf::default(); + let nj = c.joints.len(); + + // images (each once), encoded in parallel + let mut imgs: Vec = Vec::new(); + let img_of = |i: Img, imgs: &mut Vec| match imgs.iter().position(|x| *x == i) { + Some(k) => k, + None => { + imgs.push(i); + imgs.len() - 1 + } + }; + let mut mat_imgs = Vec::new(); + for m in &c.materials { + let base = match (&m.diffuse, &m.opacity) { + (d, Some(o)) => Some(img_of(Img::Merge(d.clone(), o.clone()), &mut imgs)), + (Some(d), None) => Some(img_of(Img::File(d.clone()), &mut imgs)), + (None, None) => None, + }; + let normal = m.normal.as_ref().map(|n| img_of(Img::File(n.clone()), &mut imgs)); + mat_imgs.push((base, normal)); + } + let encoded: Vec<(Vec, &str)> = imgs.par_iter().map(Img::encode).collect::>()?; + let mut images = Vec::new(); + for (i, (bytes, mime)) in imgs.iter().zip(&encoded) { + let view = b.view(bytes, None); + images.push(json!({"bufferView": view, "mimeType": mime, "name": i.name()})); + } + let textures: Vec = (0..images.len()).map(|i| json!({"sampler": 0, "source": i})).collect(); + + // materials + let cc5 = opts.cc5_json.as_deref().map(Cc5Json::load).transpose()?; + let mut maps = cc5.as_ref().map(|j| { + let dir = opts.maps_dir.clone().unwrap_or_else(|| j.path.with_extension("cc5maps")); + Maps::new(&j.path, dir) + }); + let mut materials = Vec::new(); + for (m, (base, normal)) in c.materials.iter().zip(&mat_imgs) { + let mut factor = m.factor; + let mut blend = m.opacity.is_some() || factor[3] < 1.0; + let mut v = json!({"name": m.name, "doubleSided": true, + "pbrMetallicRoughness": {"metallicFactor": 0.0, "roughnessFactor": m.roughness}}); + if let (Some(j), Some(maps)) = (&cc5, &mut maps) { + j.apply(&m.name, &mut factor, &mut blend); + if let Some(x) = j.extras(&m.name, maps)? { + v["extras"] = json!({"cc5": x}); + } + } + if let Some(t) = base { + v["pbrMetallicRoughness"]["baseColorTexture"] = json!({"index": t}); + } + if factor != [1.0; 4] { + v["pbrMetallicRoughness"]["baseColorFactor"] = json!(factor); + } + if let Some(t) = normal { + v["normalTexture"] = json!({"index": t}); + } + if blend { + v["alphaMode"] = json!("BLEND"); + } + materials.push(v); + } + if let Some(m) = &maps { + for p in &m.missing { + eprintln!(" cc5 map missing: {}", p.display()); + } + } + + // nodes: joints, then mesh nodes, then the "Armature" root + let mut nodes = Vec::new(); + for (i, j) in c.joints.iter().enumerate() { + let local = j.parent.map_or(j.world, |p| c.joints[p].world.inverse() * j.world); + let (s, r, t) = local.to_scale_rotation_translation(); + let children: Vec = (0..nj).filter(|&k| c.joints[k].parent == Some(i)).collect(); + let mut n = json!({"name": j.name, "translation": t.to_array(), "rotation": r.normalize().to_array(), "scale": s.to_array()}); + if !children.is_empty() { + n["children"] = json!(children); + } + nodes.push(n); + } + let ibm: Vec = c.joints.iter().flat_map(|j| j.world.inverse().to_cols_array()).collect(); + let ibm = b.f32s(&ibm, 16, "MAT4", None); + let wide = nj > 256; + let mut meshes = Vec::new(); + for (mi, m) in c.meshes.iter().enumerate() { + let mut prims = Vec::new(); + for p in &m.prims { + let ccidx: Vec = p.cp.iter().map(|&i| i as f32).collect(); + let a_idx = b.f32s(&ccidx, 1, "SCALAR", Some(ARRAY)); + let a_pos = b.vec3s(&p.pos, true); + let a_nrm = b.vec3s(&p.nrm, false); + let uv: Vec = p.uv.iter().flat_map(|u| u.to_array()).collect(); + let a_uv = b.f32s(&uv, 2, "VEC2", Some(ARRAY)); + let (mut js, mut ws) = (Vec::new(), Vec::new()); + for &cp in &p.cp { + let mut w = m.weights[cp as usize].clone(); + w.sort_by(|a, b| b.1.total_cmp(&a.1)); + w.truncate(4); + let sum: f32 = w.iter().map(|x| x.1).sum(); + if w.is_empty() || sum <= 0.0 { + w = vec![(0, 1.0)]; + } + let sum: f32 = w.iter().map(|x| x.1).sum(); + for k in 0..4 { + let (j, wt) = w.get(k).map_or((0, 0.0), |&(j, wt)| (j, wt / sum)); + js.push(j); + ws.push(wt); + } + } + let a_j = if wide { + let bytes: Vec = js.iter().flat_map(|&j| (j as u16).to_le_bytes()).collect(); + b.accessor(&bytes, 5123, p.cp.len(), "VEC4", Some(ARRAY)) + } else { + let bytes: Vec = js.iter().map(|&j| j as u8).collect(); + b.accessor(&bytes, 5121, p.cp.len(), "VEC4", Some(ARRAY)) + }; + let a_w = b.f32s(&ws, 4, "VEC4", Some(ARRAY)); + let a_i = if p.cp.len() <= u16::MAX as usize { + let bytes: Vec = p.tris.iter().flat_map(|&i| (i as u16).to_le_bytes()).collect(); + b.accessor(&bytes, 5123, p.tris.len(), "SCALAR", Some(ELEMENTS)) + } else { + let bytes: Vec = p.tris.iter().flat_map(|&i| i.to_le_bytes()).collect(); + b.accessor(&bytes, 5125, p.tris.len(), "SCALAR", Some(ELEMENTS)) + }; + let mut prim = json!({"attributes": {"_CCIDX": a_idx, "POSITION": a_pos, "NORMAL": a_nrm, "TEXCOORD_0": a_uv, + "JOINTS_0": a_j, "WEIGHTS_0": a_w}, "indices": a_i}); + if let Some(mat) = p.material { + prim["material"] = json!(mat); + } + prims.push(prim); + } + meshes.push(json!({"name": m.name, "primitives": prims})); + nodes.push(json!({"name": m.name, "mesh": mi, "skin": 0})); + } + let mut root_children: Vec = (0..nj).filter(|&k| c.joints[k].parent.is_none()).collect(); + root_children.extend(nj..nj + c.meshes.len()); + nodes.push(json!({"name": "Armature", "children": root_children})); + let root = nodes.len() - 1; + + while b.bin.len() % 4 != 0 { + b.bin.push(0); + } + let mut doc = json!({ + "asset": {"version": "2.0", "generator": "cc5-fbx (ufbx)"}, + "scene": 0, + "scenes": [{"name": "Scene", "nodes": [root]}], + "nodes": nodes, + "meshes": meshes, + "skins": [{"name": "Armature", "joints": (0..nj).collect::>(), "inverseBindMatrices": ibm}], + "accessors": b.accessors, + "bufferViews": b.views, + "buffers": [{"byteLength": b.bin.len()}], + }); + if !materials.is_empty() { + doc["materials"] = json!(materials); + } + if !images.is_empty() { + doc["images"] = json!(images); + doc["textures"] = json!(textures); + doc["samplers"] = json!([{"magFilter": 9729, "minFilter": 9987}]); + } + Ok(glb_bytes(&doc, &b.bin)) +} + +pub(crate) fn glb_bytes(doc: &Value, bin: &[u8]) -> Vec { + let mut js = serde_json::to_vec(doc).unwrap(); + while js.len() % 4 != 0 { + js.push(b' '); + } + let total = 12 + 8 + js.len() + 8 + bin.len(); + let mut out = Vec::with_capacity(total); + out.extend_from_slice(b"glTF"); + out.extend_from_slice(&2u32.to_le_bytes()); + out.extend_from_slice(&(total as u32).to_le_bytes()); + out.extend_from_slice(&(js.len() as u32).to_le_bytes()); + out.extend_from_slice(b"JSON"); + out.extend_from_slice(&js); + out.extend_from_slice(&(bin.len() as u32).to_le_bytes()); + out.extend_from_slice(b"BIN\0"); + out.extend_from_slice(bin); + out +} diff --git a/crates/cc5-fbx/src/lib.rs b/crates/cc5-fbx/src/lib.rs new file mode 100644 index 0000000..cee953f --- /dev/null +++ b/crates/cc5-fbx/src/lib.rs @@ -0,0 +1,295 @@ +//! Character Creator 5 FBX exports, read with ufbx and written as the GLBs +//! cc-render loads: what Blender's FBX import + glTF export made +//! (`fbx_to_glb.py`), without Blender. +//! +//! - Space: metres, Y up. Meshes are in world space (bind pose) in identity mesh +//! nodes; joints keep the FBX's own orientations, under an "Armature" root. +//! - Every vertex carries `_CCIDX`, its FBX control point (= CC5's vertex order), +//! so seam-split vertices map back to CC5's mesh. +//! - Materials as Blender's importer + exporter made them: the diffuse map with +//! the opacity map in its alpha (PNG), the normal map, roughness from the FBX +//! shininess. With a CC5 sidecar JSON (`cc5json`), also CC5's tints and opacity +//! and its shader settings as `extras.cc5` (glb_cc5_materials.py). +//! - The dump (`dump`) is `fbx_to_glb.py`'s: joints, parents, per-vertex weights +//! and the rest positions in Blender's frame (Z up), for the pack builders. + +pub mod cc5json; +pub mod dump; +pub mod expressions; +mod glb; +mod tga; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context, Result}; +use glam::{Mat3, Mat4, Vec2, Vec3, Vec4}; + +pub use glb::GlbOptions; + +pub struct Character { + pub path: PathBuf, + /// Parents before children. + pub joints: Vec, + /// Sorted by name (Blender's object order). + pub meshes: Vec, + pub materials: Vec, +} + +pub struct Joint { + pub name: String, + pub parent: Option, + /// Bind-pose world matrix. + pub world: Mat4, +} + +pub struct Mesh { + pub name: String, + /// Control points, world space (bind pose). + pub points: Vec, + /// Per control point: (joint, weight) in joint order, weight > 0. + pub weights: Vec>, + /// One per material used. + pub prims: Vec, + /// Morph targets (facial expressions), at full weight. + pub shapes: Vec, +} + +/// A morph target: control point and its world-space offset at weight 1. +pub struct Shape { + pub name: String, + pub deltas: Vec<(u32, Vec3)>, +} + +/// Vertices split by (control point, normal, UV), triangles. +pub struct Prim { + pub material: Option, + pub cp: Vec, + pub pos: Vec, + pub nrm: Vec, + pub uv: Vec, + pub tris: Vec, +} + +pub struct Material { + pub name: String, + pub diffuse: Option, + pub opacity: Option, + pub normal: Option, + /// RGB: the diffuse colour when there is no diffuse map; A: 1 - transparency + /// when there is no opacity map (as Blender's importer). + pub factor: [f32; 4], + /// Blender's importer: 1 - sqrt(shininess) / 10. + pub roughness: f32, +} + +fn mat4(m: &ufbx::Matrix) -> Mat4 { + Mat4::from_cols( + Vec4::new(m.m00 as f32, m.m10 as f32, m.m20 as f32, 0.0), + Vec4::new(m.m01 as f32, m.m11 as f32, m.m21 as f32, 0.0), + Vec4::new(m.m02 as f32, m.m12 as f32, m.m22 as f32, 0.0), + Vec4::new(m.m03 as f32, m.m13 as f32, m.m23 as f32, 1.0), + ) +} + +fn v3(v: &ufbx::Vec3) -> Vec3 { + Vec3::new(v.x as f32, v.y as f32, v.z as f32) +} + +impl Character { + pub fn load(path: &Path) -> Result { + let opts = ufbx::LoadOpts { + target_axes: ufbx::CoordinateAxes::right_handed_y_up(), + target_unit_meters: 1.0, + space_conversion: ufbx::SpaceConversion::ModifyGeometry, + generate_missing_normals: true, + ..Default::default() + }; + let s = ufbx::load_file(&path.to_string_lossy(), opts).map_err(|e| anyhow!("{}: {:?}", path.display(), e.description))?; + let dir = path.parent().unwrap_or(Path::new(".")); + + // joints: the skeleton's bones plus any other node a skin binds to (CC5 + // accessories skin to their own mesh node); parents first + let mut is_joint: HashMap> = HashMap::new(); + for n in s.nodes.iter() { + if n.bone.is_some() { + is_joint.insert(n.element.element_id, None); + } + } + for c in s.skin_clusters.iter() { + if let Some(b) = c.bone_node.as_ref() { + is_joint.insert(b.element.element_id, Some(mat4(&c.bind_to_world))); + } + } + let mut joints = Vec::new(); + let mut joint_of: HashMap = HashMap::new(); + fn walk(n: &ufbx::Node, parent: Option, is_joint: &HashMap>, joints: &mut Vec, joint_of: &mut HashMap) { + let mut p = parent; + if let Some(bind) = is_joint.get(&n.element.element_id) { + // names stay unique, as Blender makes them (a Smart Hair group and an + // accessory can each bring a "CC_Base_Pivot": the later one is ".001") + let base = n.element.name.to_string(); + let mut name = base.clone(); + let mut k = 0; + while joints.iter().any(|j: &Joint| j.name == name) { + k += 1; + name = format!("{base}.{k:03}"); + } + joint_of.insert(n.element.element_id, joints.len()); + joints.push(Joint { name, parent, world: bind.unwrap_or_else(|| mat4(&n.node_to_world)) }); + p = Some(joints.len() - 1); + } + for c in n.children.iter() { + walk(c, p, is_joint, joints, joint_of); + } + } + walk(&s.root_node, None, &is_joint, &mut joints, &mut joint_of); + + let materials = s.materials.iter().map(|m| material(m, dir)).collect(); + + let mut meshes = Vec::new(); + for node in s.nodes.iter() { + let Some(mesh) = node.mesh.as_ref() else { continue }; + let g2w = mat4(&node.geometry_to_world); + let nmat = Mat3::from_mat4(g2w).inverse().transpose(); + let points: Vec = mesh.vertices.iter().map(|p| g2w.transform_point3(v3(p))).collect(); + let weights = match mesh.skin_deformers.first() { + Some(skin) => { + let cj: Vec> = skin.clusters.iter().map(|c| c.bone_node.as_ref().and_then(|b| joint_of.get(&b.element.element_id)).map(|&j| j as u32)).collect(); + skin.vertices + .iter() + .map(|sv| { + let mut w: Vec<(u32, f32)> = (sv.weight_begin..sv.weight_begin + sv.num_weights) + .filter_map(|i| { + let w = &skin.weights[i as usize]; + let j = cj[w.cluster_index as usize]?; + (w.weight > 0.0).then_some((j, w.weight as f32)) + }) + .collect(); + w.sort_by_key(|x| x.0); // joint order, as Blender's vertex groups + w + }) + .collect() + } + None => { + // unskinned: follow the nearest joint above the node + let mut j = None; + let mut p = node.parent.as_ref(); + while let Some(n) = p { + if let Some(&i) = joint_of.get(&n.element.element_id) { + j = Some(i as u32); + break; + } + p = n.parent.as_ref(); + } + vec![j.map(|j| vec![(j, 1.0)]).unwrap_or_default(); points.len()] + } + }; + let mat_index: Vec = mesh.materials.iter().map(|m| m.element.typed_id as usize).collect(); + let mut prims: Vec = Vec::new(); + let mut keys: Vec> = Vec::new(); + let mut tri = vec![0u32; mesh.max_face_triangles * 3]; + let has_uv = mesh.vertex_uv.exists; + for (fi, face) in mesh.faces.iter().enumerate() { + let material = mesh.face_material.get(fi).and_then(|&m| mat_index.get(m as usize).copied()); + let pi = match prims.iter().position(|p| p.material == material) { + Some(i) => i, + None => { + prims.push(Prim { material, cp: vec![], pos: vec![], nrm: vec![], uv: vec![], tris: vec![] }); + keys.push(HashMap::new()); + prims.len() - 1 + } + }; + let nt = mesh.triangulate_face(&mut tri, *face) as usize; + let p = &mut prims[pi]; + for &c in &tri[..nt * 3] { + let c = c as usize; + let cp = mesh.vertex_indices[c]; + let n = (nmat * v3(&mesh.vertex_normal.values[mesh.vertex_normal.indices[c] as usize])).normalize_or_zero(); + let uv = if has_uv { + let t = &mesh.vertex_uv.values[mesh.vertex_uv.indices[c] as usize]; + Vec2::new(t.x as f32, 1.0 - t.y as f32) + } else { + Vec2::ZERO + }; + let key = (cp, n.to_array().map(f32::to_bits), uv.to_array().map(f32::to_bits)); + let v = *keys[pi].entry(key).or_insert_with(|| { + p.cp.push(cp); + p.pos.push(points[cp as usize]); + p.nrm.push(n); + p.uv.push(uv); + p.cp.len() as u32 - 1 + }); + p.tris.push(v); + } + } + prims.sort_by_key(|p| p.material); + let lin = Mat3::from_mat4(g2w); + let mut shapes = Vec::new(); + for bd in mesh.blend_deformers.iter() { + for ch in bd.channels.iter() { + // the shape at full weight (in-betweens are not used) + let shape: Option<&ufbx::BlendShape> = ch.keyframes.iter().find(|k| (k.target_weight - 1.0).abs() < 1e-6).map(|k| &*k.shape).or(ch.target_shape.as_deref()); + let Some(shape) = shape else { continue }; + let deltas = shape.offset_vertices.iter().zip(shape.position_offsets.iter()).map(|(&i, d)| (i, lin * v3(d))).collect(); + shapes.push(Shape { name: ch.element.name.to_string(), deltas }); + } + } + meshes.push(Mesh { name: node.element.name.to_string(), points, weights, prims, shapes }); + } + meshes.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(Self { path: path.to_path_buf(), joints, meshes, materials }) + } + + pub fn mesh(&self, name: &str) -> Option<&Mesh> { + self.meshes.iter().find(|m| m.name == name) + } + + /// The GLB (see the crate docs). + pub fn to_glb(&self, opts: &GlbOptions) -> Result> { + glb::write(self, opts) + } +} + +fn material(m: &ufbx::Material, dir: &Path) -> Material { + let tex = |props: &[&str]| { + m.textures.iter().find(|t| props.contains(&t.material_prop.as_ref())).and_then(|t| texture_path(&t.texture, dir)) + }; + let diffuse = tex(&["DiffuseColor"]); + let opacity = tex(&["TransparentColor", "TransparencyFactor"]); + let normal = tex(&["NormalMap", "Bump"]); + let dc = &m.fbx.diffuse_color.value_vec4; + let rgb = if diffuse.is_some() { [1.0; 3] } else { [dc.x as f32, dc.y as f32, dc.z as f32] }; + let a = if opacity.is_some() { 1.0 } else { 1.0 - m.fbx.transparency_factor.value_vec4.x as f32 }; + let shininess = m.fbx.specular_exponent.value_vec4.x.max(0.0) as f32; + Material { + name: m.element.name.to_string(), + diffuse, + opacity, + normal, + factor: [rgb[0], rgb[1], rgb[2], a.clamp(0.0, 1.0)], + roughness: (1.0 - shininess.sqrt() / 10.0).clamp(0.0, 1.0), + } +} + +/// A texture's file: as written, else relative to the FBX, else in its `.fbm`. +fn texture_path(t: &ufbx::Texture, dir: &Path) -> Option { + let fix = |s: &str| s.replace('\\', "/"); + let cands = [ + PathBuf::from(fix(&t.absolute_filename)), + dir.join(fix(&t.relative_filename)), + PathBuf::from(fix(&t.filename)), + ]; + cands.into_iter().find(|p| !p.as_os_str().is_empty() && p.is_file()).or_else(|| { + let name = Path::new(&fix(&t.relative_filename)).file_name()?.to_owned(); + std::fs::read_dir(dir).ok()?.flatten().map(|e| e.path().join(&name)).find(|p| p.is_file()) + }) +} + +/// Read a CC5 FBX and write its GLB. +pub fn convert(fbx: &Path, glb_path: &Path, opts: &GlbOptions) -> Result { + let c = Character::load(fbx)?; + let bytes = c.to_glb(opts)?; + std::fs::write(glb_path, bytes).with_context(|| format!("writing {}", glb_path.display()))?; + Ok(c) +} diff --git a/crates/cc5-fbx/src/main.rs b/crates/cc5-fbx/src/main.rs new file mode 100644 index 0000000..a03db6e --- /dev/null +++ b/crates/cc5-fbx/src/main.rs @@ -0,0 +1,56 @@ +//! `cc5-fbx [--dump ] [--cc5-json ]` +//! `cc5-fbx expressions `: the editor's expression pack +//! +//! A CC5 FBX export as a GLB (`fbx_to_glb.py` without Blender). `--dump` also +//! writes the joints / weights / rest-position dump the pack builders read. +//! `--cc5-json` adds CC5's material tints and `extras.cc5` (maps written to +//! `.cc5maps/`), as bake_alpha.py's first pass + glb_cc5_materials.py. + +use std::path::PathBuf; + +use anyhow::{bail, Result}; +use cc5_fbx::{Character, GlbOptions}; + +fn main() -> Result<()> { + let all: Vec = std::env::args().skip(1).collect(); + if all.first().map(String::as_str) == Some("expressions") { + // cc5-fbx expressions + let [_, fbx, pack, out] = <[String; 4]>::try_from(all).map_err(|_| anyhow::anyhow!("usage: cc5-fbx expressions "))?; + let c = Character::load(std::path::Path::new(&fbx))?; + return cc5_fbx::expressions::write(&c, std::path::Path::new(&pack), std::path::Path::new(&out)); + } + let mut args = all.into_iter(); + let (mut pos, mut dump, mut cc5) = (Vec::new(), None, None); + while let Some(a) = args.next() { + match a.as_str() { + "--dump" => dump = args.next().map(PathBuf::from), + "--cc5-json" => cc5 = args.next().map(PathBuf::from), + "--keep-morphs" => bail!("--keep-morphs: morph targets are not written (nothing uses them)"), + _ if a.starts_with("--") => bail!("unknown option {a}"), + _ => pos.push(PathBuf::from(a)), + } + } + let [fbx, out] = <[PathBuf; 2]>::try_from(pos).map_err(|_| anyhow::anyhow!("usage: cc5-fbx [--dump ] [--cc5-json ]"))?; + let t = std::time::Instant::now(); + let c = Character::load(&fbx)?; + let t_load = t.elapsed().as_secs_f32(); + let maps_dir = cc5.as_ref().map(|_| out.with_extension("cc5maps")); + let bytes = c.to_glb(&GlbOptions { cc5_json: cc5, maps_dir })?; + std::fs::write(&out, &bytes)?; + if let Some(d) = &dump { + cc5_fbx::dump::write(&c, d)?; + } + let verts: usize = c.meshes.iter().map(|m| m.points.len()).sum(); + println!( + "wrote {}: {} meshes, {} joints, {} materials, {} control points, {:.1} MB (load {:.2} s, total {:.2} s)", + out.display(), + c.meshes.len(), + c.joints.len(), + c.materials.len(), + verts, + bytes.len() as f64 / 1e6, + t_load, + t.elapsed().as_secs_f32() + ); + Ok(()) +} diff --git a/crates/cc5-fbx/src/tga.rs b/crates/cc5-fbx/src/tga.rs new file mode 100644 index 0000000..1b7ebc5 --- /dev/null +++ b/crates/cc5-fbx/src/tga.rs @@ -0,0 +1,92 @@ +//! TGA (CC5's shader masks): uncompressed or RLE, true colour (24 / 32 bit) or +//! greyscale (8 bit). Decoded here so the `image` crate's features (shared with +//! Bevy's) stay as they are. + +use anyhow::{bail, ensure, Result}; +use image::DynamicImage; + +pub fn decode(b: &[u8]) -> Result { + ensure!(b.len() >= 18, "TGA: short header"); + let id_len = b[0] as usize; + let cmap_type = b[1]; + let kind = b[2]; + let cmap_len = u16::from_le_bytes([b[5], b[6]]) as usize; + let cmap_bits = b[7] as usize; + let w = u16::from_le_bytes([b[12], b[13]]) as usize; + let h = u16::from_le_bytes([b[14], b[15]]) as usize; + let bpp = b[16] as usize; + let desc = b[17]; + ensure!(cmap_type == 0, "TGA: colour-mapped images are not supported"); + let (rle, grey) = match kind { + 2 => (false, false), + 3 => (false, true), + 10 => (true, false), + 11 => (true, true), + _ => bail!("TGA: image type {kind} not supported"), + }; + let px = bpp / 8; + ensure!(matches!((grey, px), (true, 1) | (false, 3) | (false, 4)), "TGA: {bpp} bit {} not supported", if grey { "greyscale" } else { "colour" }); + let mut at = 18 + id_len + cmap_len * cmap_bits.div_ceil(8); + let n = w * h * px; + let mut data = Vec::with_capacity(n); + if rle { + while data.len() < n { + ensure!(at < b.len(), "TGA: truncated"); + let head = b[at] as usize; + at += 1; + let count = (head & 0x7f) + 1; + if head & 0x80 != 0 { + ensure!(at + px <= b.len(), "TGA: truncated"); + for _ in 0..count { + data.extend_from_slice(&b[at..at + px]); + } + at += px; + } else { + ensure!(at + count * px <= b.len(), "TGA: truncated"); + data.extend_from_slice(&b[at..at + count * px]); + at += count * px; + } + } + data.truncate(n); + } else { + ensure!(at + n <= b.len(), "TGA: truncated"); + data.extend_from_slice(&b[at..at + n]); + } + // BGR(A) -> RGB(A) + if !grey { + for p in data.chunks_exact_mut(px) { + p.swap(0, 2); + } + } + // rows are stored bottom-up unless the descriptor says top-down + if desc & 0x20 == 0 { + let row = w * px; + let mut flipped = Vec::with_capacity(n); + for y in (0..h).rev() { + flipped.extend_from_slice(&data[y * row..(y + 1) * row]); + } + data = flipped; + } + let (w, h) = (w as u32, h as u32); + Ok(match px { + 1 => DynamicImage::ImageLuma8(image::GrayImage::from_raw(w, h, data).unwrap()), + 3 => DynamicImage::ImageRgb8(image::RgbImage::from_raw(w, h, data).unwrap()), + _ => DynamicImage::ImageRgba8(image::RgbaImage::from_raw(w, h, data).unwrap()), + }) +} + +#[cfg(test)] +mod tests { + #[test] + fn rle_bottom_up_bgr() { + // 2x2, 24 bit, RLE: bottom row = 2x red (run), top row = blue, green (raw) + let mut b = vec![0u8, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 24, 0]; + b.extend_from_slice(&[0x81, 0, 0, 255]); + b.extend_from_slice(&[0x01, 255, 0, 0, 0, 255, 0]); + let img = super::decode(&b).unwrap().to_rgb8(); + assert_eq!(img.get_pixel(0, 0).0, [0, 0, 255]); + assert_eq!(img.get_pixel(1, 0).0, [0, 255, 0]); + assert_eq!(img.get_pixel(0, 1).0, [255, 0, 0]); + assert_eq!(img.get_pixel(1, 1).0, [255, 0, 0]); + } +} diff --git a/crates/cc5-hair/Cargo.toml b/crates/cc5-hair/Cargo.toml new file mode 100644 index 0000000..f64f038 --- /dev/null +++ b/crates/cc5-hair/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "cc5-hair" +version = "0.1.0" +edition = "2021" +publish = false +description = "Convert Character Creator 5 hair (Smart Hair cards) into strand hair parts for cc-render, in Rust." + +[lib] +name = "cc5_hair" + +[[bin]] +name = "cc5-hair" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +# 0.29 = the glam Bevy 0.15 uses +glam = "0.29" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +gltf = { version = "1.4", default-features = false, features = ["names", "extras", "utils"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +# nearest point on the body (rapier's version: one copy in the tree) +parry3d = "0.17" +rand = "0.8" +rayon = "1" +cc5-fbx = { path = "../cc5-fbx" } +# `check`: the editor's conform +cc5-shape = { path = "../cc5-shape" } diff --git a/crates/cc5-hair/src/cards.rs b/crates/cc5-hair/src/cards.rs new file mode 100644 index 0000000..7ad9653 --- /dev/null +++ b/crates/cc5-hair/src/cards.rs @@ -0,0 +1,393 @@ +//! CC5 hair cards -> strands (hair_build.py `cards_to_strands`, `_patch_strands`). +//! +//! Per card (connected component of the hair material's triangles): strands at +//! texture columns that hold hair, from the root end (the card end nearer the +//! head) to where the texture's strands end, coloured from the texture, gathered +//! into clumps, spread a little off the card plane. UVs are glTF's (v down) with +//! images row 0 at the top: the same texels Blender's UVs address. + +use glam::{Vec2, Vec3}; +use rayon::prelude::*; + +use crate::head::{Body, Surface}; +use crate::settings::Convert; +use crate::{srgb_to_linear, uniform_polyline, Rng, Strand}; + +/// An image sampled as 0..1 floats (colour as stored, i.e. sRGB-encoded; +/// alpha straight). +pub struct Tex { + pub w: usize, + pub h: usize, + pub img: std::sync::Arc, +} + +impl Tex { + pub fn new(img: std::sync::Arc) -> Self { + Self { w: img.width() as usize, h: img.height() as usize, img } + } + + /// Nearest texel at `uv` (wrapping). + pub fn sample(&self, uv: Vec2) -> [f32; 4] { + let x = ((uv.x.rem_euclid(1.0) * self.w as f32) as usize).min(self.w - 1); + let y = ((uv.y.rem_euclid(1.0) * self.h as f32) as usize).min(self.h - 1); + self.img.get_pixel(x as u32, y as u32).0.map(|c| c as f32 / 255.0) + } +} + +/// One card: its triangles' corners (positions and UVs, in the card's +/// (across, along) UV frame). +struct Card { + cp: Vec, + cu: Vec, + swap: bool, + area: f32, +} + +impl Card { + /// The 3D point at `uv` on the card, if a triangle covers it. + fn point(&self, uv: Vec2) -> Option { + for t in 0..self.cp.len() / 3 { + let (a, b, c) = (self.cu[3 * t], self.cu[3 * t + 1], self.cu[3 * t + 2]); + let (v0, v1, d) = (b - a, c - a, uv - a); + let mut den = v0.x * v1.y - v1.x * v0.y; + if den.abs() < 1e-12 { + den = 1e-12; + } + let s = (d.x * v1.y - v1.x * d.y) / den; + let tt = (v0.x * d.y - d.x * v0.y) / den; + if s >= -1e-4 && tt >= -1e-4 && s + tt <= 1.0 + 1e-4 { + let [p0, p1, p2] = [self.cp[3 * t], self.cp[3 * t + 1], self.cp[3 * t + 2]]; + return Some(p0 * (1.0 - s - tt) + p1 * s + p2 * tt); + } + } + None + } + + /// Texture UV of a point in the card's frame. + fn tex_uv(&self, q: Vec2) -> Vec2 { + if self.swap { + Vec2::new(q.y, q.x) + } else { + q + } + } + + fn bounds(&self) -> (Vec2, Vec2) { + self.cu.iter().fold((Vec2::MAX, Vec2::MIN), |(lo, hi), u| (lo.min(*u), hi.max(*u))) + } +} + +/// Split the hair material's triangles into cards (components over shared +/// vertices). +fn cards(pos: &[Vec3], uv: &[Vec2], tris: &[[u32; 3]]) -> Vec { + let mut parent: Vec = (0..pos.len() as u32).collect(); + fn find(p: &mut [u32], mut x: u32) -> u32 { + while p[x as usize] != x { + p[x as usize] = p[p[x as usize] as usize]; + x = p[x as usize]; + } + x + } + for t in tris { + for (u, v) in [(t[0], t[1]), (t[1], t[2])] { + let (ru, rv) = (find(&mut parent, u), find(&mut parent, v)); + if ru != rv { + parent[ru as usize] = rv; + } + } + } + let mut groups: std::collections::BTreeMap> = Default::default(); + for (i, t) in tris.iter().enumerate() { + let r = find(&mut parent, t[0]); + groups.entry(r).or_default().push(i); + } + groups + .into_values() + .map(|ti| { + let cp: Vec = ti.iter().flat_map(|&i| tris[i].map(|v| pos[v as usize])).collect(); + let mut cu: Vec = ti.iter().flat_map(|&i| tris[i].map(|v| uv[v as usize])).collect(); + let area: f32 = cp.chunks_exact(3).map(|c| (c[1] - c[0]).cross(c[2] - c[0]).length() / 2.0).sum(); + let (lo, hi) = cu.iter().fold((Vec2::MAX, Vec2::MIN), |(lo, hi), u| (lo.min(*u), hi.max(*u))); + // strands run along the card's longer UV axis (most CC5 strips are + // vertical; some short-hair cards lie sideways) + let swap = (hi.x - lo.x) > 1.2 * (hi.y - lo.y); + if swap { + cu.iter_mut().for_each(|u| *u = Vec2::new(u.y, u.x)); + } + Card { cp, cu, swap, area } + }) + .collect() +} + +/// Strands for one source layer (the hair material's triangles). Not pushed +/// off the head yet (see `shape::finish`). +pub fn cards_to_strands( + pos: &[Vec3], + uv: &[Vec2], + tris: &[[u32; 3]], + img: Option<&Tex>, + flow: Option<&Tex>, + conv: &Convert, + body: &Body, + seed: u64, +) -> Vec { + let cards = cards(pos, uv, tris); + cards + .par_iter() + .enumerate() + .flat_map_iter(|(ci, card)| { + let mut rng = Rng::new(seed ^ (ci as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + card_strands(card, img, flow, conv, body, &mut rng) + }) + .collect() +} + +fn card_strands(card: &Card, img: Option<&Tex>, flow: Option<&Tex>, conv: &Convert, body: &Body, rng: &mut Rng) -> Vec { + let mut out = Vec::new(); + let (lo, hi) = card.bounds(); + let (umin, umax, vmin, vmax) = (lo.x, hi.x, lo.y, hi.y); + if umax - umin < 1e-4 || vmax - vmin < 1e-4 { + return out; + } + let np = conv.points.max(3); + if ((vmax - vmin) as f64) < conv.patch_aspect * (umax - umin) as f64 { + return patch_strands(card, img, flow, conv, rng, np); + } + // root end: the v end whose points are nearer the head + let um = (umin + umax) / 2.0; + let e0 = card.point(Vec2::new(um, vmin + 0.02 * (vmax - vmin))); + let e1 = card.point(Vec2::new(um, vmax - 0.02 * (vmax - vmin))); + let (d0, d1) = match (e0, e1) { + (Some(a), Some(b)) => (body.nearest(a).dist, body.nearest(b).dist), + _ => (0.0, 0.0), + }; + let (v_root, v_tip) = if d0 <= d1 { (vmin, vmax) } else { (vmax, vmin) }; + let n_try = conv.min_tries.max((card.area as f64 * 1e4 * conv.strands_per_cm2 * 1.5) as usize); + let vs: Vec = (0..32).map(|i| v_root + (v_tip - v_root) * i as f32 / 31.0).collect(); + let jit = conv.thickness; + // clumps across the card: one tone and one offset off the card each + let mut clumps: Vec<(f32, f64, f64)> = Vec::new(); + let mut spread = 0.0f64; + if conv.clumps_per_cm > 0.0 { + let vm = (vmin + vmax) / 2.0; + let width_m = match (card.point(Vec2::new(umin + 0.02 * (umax - umin), vm)), card.point(Vec2::new(umax - 0.02 * (umax - umin), vm))) { + (Some(a), Some(b)) => a.distance(b) as f64, + _ => 0.01, + }; + let n_c = ((width_m * 100.0 * conv.clumps_per_cm).round() as usize).max(1); + for k in 0..n_c { + let c = umin + ((k as f64 + 0.5 + rng.uniform(-0.3, 0.3)) * (umax - umin) as f64 / n_c as f64) as f32; + clumps.push((c, rng.uniform(conv.clump_shade[0], conv.clump_shade[1]), rng.gauss(0.0, jit))); + } + spread = conv.clump_spread * (umax - umin) as f64 / n_c as f64; + } + let nrm = { + let c = &card.cp; + (c[1] - c[0]).cross(c[2] - c[0]).normalize_or_zero() + }; + for _ in 0..n_try { + let (u0, c_shade, c_off) = if clumps.is_empty() { + (rng.uniform(umin as f64, umax as f64) as f32, 1.0, None) + } else { + let (c, s, o) = clumps[rng.below(clumps.len())]; + ((rng.gauss(c as f64, spread) as f32).clamp(umin, umax), s, Some(o)) + }; + let (t_start, t_end, rgb, opacity); + if let Some(img) = img { + let tex: Vec<[f32; 4]> = vs.iter().map(|&v| img.sample(card.tex_uv(Vec2::new(u0, v)))).collect(); + let a: Vec = tex.iter().map(|t| t[3]).collect(); + let amax = a.iter().cloned().fold(0.0f32, f32::max); + let level = if conv.accept_on == "max" { amax } else { a.iter().sum::() / a.len() as f32 }; + if (level as f64) < rng.uniform(conv.accept[0], conv.accept[1]) { + continue; // no strands in this column of the texture + } + // strand texels: relative to the column's peak (faint short-hair maps count too) + let thr = 0.06f32.max(0.45 * amax); + let hair: Vec = (0..a.len()).filter(|&i| a[i] > thr).collect(); + if hair.len() < 3 { + continue; + } + let (first, last) = (hair[0], *hair.last().unwrap()); + t_end = (last + 1) as f32 / 32.0; + // the strand starts where the texture's hair does (cards often begin transparent) + let h0 = if conv.start_at_hair { first } else { 0 }; + t_start = h0 as f32 / 32.0; + // colour from solid texels only: CC5 maps fade to pale colours where alpha is low + let col_a = &a[h0..=last]; + let mut cols: Vec<[f32; 3]> = tex[h0..=last].iter().map(|t| [t[0], t[1], t[2]]).collect(); + let n = cols.len(); + let mut tt: Vec = (0..n).map(|i| if n > 1 { i as f32 / (n - 1) as f32 } else { 0.0 }).collect(); + if col_a.iter().filter(|&&x| x > 0.4).count() >= 2 { + let keep: Vec = (0..n).filter(|&i| col_a[i] > 0.4).collect(); + cols = keep.iter().map(|&i| cols[i]).collect(); + tt = keep.iter().map(|&i| tt[i]).collect(); + } + rgb = (0..np) + .map(|i| { + let x = i as f32 / (np - 1) as f32; + Vec3::from(std::array::from_fn::(|k| interp(x, &tt, &cols, k))) + }) + .collect::>(); + let solid: Vec = col_a.iter().cloned().filter(|&x| x > 0.05).collect(); + opacity = if solid.is_empty() { 1.0 } else { solid.iter().sum::() / solid.len() as f32 }.clamp(0.15, 1.0); + } else { + t_start = 0.0; + t_end = 1.0; + rgb = vec![Vec3::new(0.5, 0.4, 0.3); np]; + opacity = 1.0; + } + let pts: Vec = (0..np) + .filter_map(|i| { + let t = t_start + (t_end - t_start) * i as f32 / (np - 1) as f32; + card.point(Vec2::new(u0, v_root + (v_tip - v_root) * t)) + }) + .collect(); + if pts.len() < 3 { + continue; + } + // thickness: strands spread a little off the card plane + let off = match c_off { + None => nrm * rng.gauss(0.0, jit) as f32 + Vec3::from(std::array::from_fn(|_| rng.gauss(0.0, jit * 0.3) as f32)), + Some(o) => nrm * (o + rng.gauss(0.0, jit * 0.15)) as f32 + Vec3::from(std::array::from_fn(|_| rng.gauss(0.0, jit * 0.1) as f32)), + }; + let m = pts.len(); + let pts: Vec = pts.iter().enumerate().map(|(i, p)| *p + off * (0.3 + 0.7 * i as f32 / (m - 1) as f32)).collect(); + let pts = uniform_polyline(&pts, np); + let width = rng.uniform(conv.width[0], conv.width[1]) * (opacity as f64).powf(conv.opacity_power); + let shade = rng.uniform(conv.shade[0], conv.shade[1]) * c_shade; + out.push(Strand { + pts, + id: rng.f() as f32, + width: width as f32, + shade: shade as f32, + rgb: rgb.iter().map(|c| Vec3::new(srgb_to_linear(c.x), srgb_to_linear(c.y), srgb_to_linear(c.z))).collect(), + }); + } + out +} + +/// Short strands on a "patch" card (short-hair texture painted over a wide +/// area): many strands where the texture has hair, each marching along the +/// card's flow map (CC5's: R = 0.5 + 0.5 du, G along the strand). +fn patch_strands(card: &Card, img: Option<&Tex>, flow: Option<&Tex>, conv: &Convert, rng: &mut Rng, np: usize) -> Vec { + let mut out = Vec::new(); + let (lo, hi) = card.bounds(); + let uv_area: f32 = card.cu.chunks_exact(3).map(|c| ((c[1].x - c[0].x) * (c[2].y - c[0].y) - (c[2].x - c[0].x) * (c[1].y - c[0].y)).abs() / 2.0).sum(); + if uv_area < 1e-9 { + return out; + } + let scale = (card.area / uv_area).sqrt(); // metres per UV unit + let n = (card.area as f64 * 1e4 * conv.patch_density) as usize; + for _ in 0..n { + let p0 = Vec2::new(rng.uniform(lo.x as f64, hi.x as f64) as f32, rng.uniform(lo.y as f64, hi.y as f64) as f32); + if let Some(img) = img { + if (img.sample(card.tex_uv(p0))[3] as f64) < rng.uniform(0.08, 0.5) { + continue; + } + } + let lk = (conv.patch_length * rng.uniform(0.5, 1.2)) as f32 / scale; // UV units + let mut uv = vec![p0]; + for _ in 1..np { + let q = *uv.last().unwrap(); + // CC5's flow map, in glTF UVs (v down): G encodes the direction down the image + let mut d = match flow { + Some(f) => { + let s = f.sample(card.tex_uv(q)); + let d = Vec2::new(s[0] * 2.0 - 1.0, -(s[1] * 2.0 - 1.0)); + if card.swap { + Vec2::new(d.y, d.x) + } else { + d + } + } + None => Vec2::new(0.0, 1.0), + }; + d = if d.length() > 1e-3 { d.normalize() } else { Vec2::new(0.0, 1.0) }; + uv.push(q + d * lk / (np - 1) as f32); + } + // the strand ends where it leaves the card + let pts: Vec = uv.iter().map_while(|q| card.point(*q)).collect(); + if pts.len() < 3 { + continue; + } + let pts = uniform_polyline(&pts, np); + let rgb: Vec = match img { + Some(img) => { + let tex: Vec<[f32; 3]> = uv.iter().map(|q| { + let t = img.sample(card.tex_uv(*q)); + [t[0], t[1], t[2]] + }).collect(); + let tt: Vec = (0..tex.len()).map(|i| i as f32 / (tex.len() - 1) as f32).collect(); + (0..np).map(|i| Vec3::from(std::array::from_fn::(|k| interp(i as f32 / (np - 1) as f32, &tt, &tex, k)))).collect() + } + None => vec![Vec3::new(0.3, 0.25, 0.2); np], + }; + out.push(Strand { + pts, + id: rng.f() as f32, + width: rng.uniform(conv.width[0], conv.width[1]) as f32, + shade: rng.uniform(conv.shade[0], conv.shade[1]) as f32, + rgb: rgb.iter().map(|c| Vec3::new(srgb_to_linear(c.x), srgb_to_linear(c.y), srgb_to_linear(c.z))).collect(), + }); + } + out +} + +/// `np.interp(x, xs, ys[:, k])` for increasing `xs`. +fn interp(x: f32, xs: &[f32], ys: &[[f32; 3]], k: usize) -> f32 { + if xs.is_empty() { + return 0.0; + } + if x <= xs[0] { + return ys[0][k]; + } + for i in 1..xs.len() { + if x <= xs[i] { + let f = if xs[i] > xs[i - 1] { (x - xs[i - 1]) / (xs[i] - xs[i - 1]) } else { 1.0 }; + return ys[i - 1][k] + (ys[i][k] - ys[i - 1][k]) * f; + } + } + ys[ys.len() - 1][k] +} + +/// The source's other material layers (scalp, caps): strands stay above them. +pub fn scalp_surface(pos: &[Vec3], tris: Vec<[u32; 3]>) -> Option { + Surface::new(pos.to_vec(), tris) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A unit quad card in the XY plane, UVs = positions. + fn quad() -> Card { + let p = [Vec3::new(0.0, 0.0, 0.0), Vec3::new(1.0, 0.0, 0.0), Vec3::new(1.0, 1.0, 0.0), Vec3::new(0.0, 1.0, 0.0)]; + let tris = [[0, 1, 2], [0, 2, 3]]; + let cp = tris.iter().flat_map(|t| t.map(|i| p[i])).collect(); + let cu = tris.iter().flat_map(|t| t.map(|i| Vec2::new(p[i].x, p[i].y))).collect(); + Card { cp, cu, swap: false, area: 1.0 } + } + + #[test] + fn card_points_follow_uvs() { + let c = quad(); + assert!(c.point(Vec2::new(0.25, 0.75)).unwrap().distance(Vec3::new(0.25, 0.75, 0.0)) < 1e-5); + assert!(c.point(Vec2::new(0.9, 0.1)).unwrap().distance(Vec3::new(0.9, 0.1, 0.0)) < 1e-5); + assert!(c.point(Vec2::new(1.5, 0.5)).is_none()); + } + + #[test] + fn components_split_cards() { + // two separate quads -> two cards; a wide one swaps its axes + let pos = vec![ + Vec3::ZERO, Vec3::X, Vec3::new(1.0, 1.0, 0.0), Vec3::Y, + Vec3::new(5.0, 0.0, 0.0), Vec3::new(7.0, 0.0, 0.0), Vec3::new(7.0, 0.5, 0.0), Vec3::new(5.0, 0.5, 0.0), + ]; + let uv: Vec = pos.iter().map(|p| Vec2::new(p.x, p.y)).collect(); + let tris = vec![[0, 1, 2], [0, 2, 3], [4, 5, 6], [4, 6, 7]]; + let cs = cards(&pos, &uv, &tris); + assert_eq!(cs.len(), 2); + assert_eq!(cs.iter().filter(|c| c.swap).count(), 1); + assert!((cs.iter().map(|c| c.area).sum::() - 2.0).abs() < 1e-5); + } +} diff --git a/crates/cc5-hair/src/check.rs b/crates/cc5-hair/src/check.rs new file mode 100644 index 0000000..ade680a --- /dev/null +++ b/crates/cc5-hair/src/check.rs @@ -0,0 +1,104 @@ +//! `cc5-hair check`: the editor's hair conform against CC5 itself on whole +//! recipes (was check_conform_cc5.py). +//! +//! Reads the bind-pose OBJs cc5_export_smart_hair.py writes in "shapes" mode +//! (`//shape_base.obj`, `shape_.obj`), binds every hair +//! object of the base OBJ to CC_Base_Body as the editor does (layers CC5 moves +//! rigidly: "head_box"; other scalp hair to the head; beards / brows to the +//! face), applies the body's move to each recipe and reports the distance to +//! where CC5 put the hair, next to how far CC5 moved it. + +use std::path::Path; + +use anyhow::{anyhow, Result}; +use cc5_shape::{Conform, ConformOptions}; +use glam::DVec3; + +use crate::follow::{key, layer_modes, not_hair, read_obj}; +use crate::regions::regions; + +fn percentile(v: &mut [f64], q: f64) -> f64 { + v.sort_by(f64::total_cmp); + let x = q / 100.0 * (v.len() - 1) as f64; + let (i, f) = (x.floor() as usize, x.fract()); + v[i] + (v[(i + 1).min(v.len() - 1)] - v[i]) * f +} + +fn f32s(p: &[DVec3]) -> Vec<[f32; 3]> { + p.iter().map(|v| v.as_vec3().to_array()).collect() +} + +/// Print the conform error of `group` (a folder under `root`) on every recipe. +pub fn check(root: &Path, group: &str) -> Result<()> { + let d = root.join(group); + let base = read_obj(&d.join("shape_base.obj"))?; + let bname = base.keys().find(|k| k.starts_with("CC_Base_Body")).ok_or_else(|| anyhow!("{group}: no CC_Base_Body in shape_base.obj"))?.clone(); + let body = &base[&bname]; + let reg = regions(body); + let hair: Vec<&String> = base.keys().filter(|k| !not_hair(k)).collect(); + println!( + "{group}: body {} verts (scalp {}, face {} anchors); hair objects {:?}", + body.len(), + reg.scalp.len(), + reg.face.len(), + hair.iter().map(|k| (k.as_str(), base[*k].len())).collect::>() + ); + let modes = layer_modes(&d, 0.05)?; + let body32 = f32s(body); + let mut conf = Vec::new(); + for k in &hair { + let pts = &base[*k]; + let mut z: Vec = pts.iter().map(|p| p.z).collect(); + let lk = k.to_lowercase(); + // scalp hair: most vertices above the eyes (as the catalog's kind) + let scalp = percentile(&mut z, 50.0) > 1.62 && !["brow", "beard", "mous", "goat", "stubble"].iter().any(|w| lk.contains(w)); + let (mode, anchors, opts) = if modes.get(&key(k)).map(String::as_str) == Some("head_box") { + ("head_box", ®.head_box, ConformOptions::HEAD_BOX) + } else if scalp { + ("scalp", ®.head, ConformOptions::SCALP) + } else { + ("skin", ®.face, ConformOptions::SKIN) + }; + conf.push((k.to_string(), mode, Conform::bind(&body32, anchors, &f32s(pts), opts)?)); + } + let mut recipes: Vec<_> = std::fs::read_dir(&d)? + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter_map(|f| f.strip_prefix("shape_").and_then(|r| r.strip_suffix(".obj")).map(str::to_string)) + .filter(|r| r != "base") + .collect(); + recipes.sort(); + for r in recipes { + let sh = read_obj(&d.join(format!("shape_{r}.obj")))?; + let Some(moved) = sh.get(&bname).filter(|m| m.len() == body.len()) else { + println!(" recipe {r}: body missing / vertex count changed"); + continue; + }; + let db: Vec<[f32; 3]> = moved.iter().zip(body).map(|(a, b)| (*a - *b).as_vec3().to_array()).collect(); + let mut bm: Vec = db.iter().map(|v| glam::Vec3::from(*v).length() as f64).collect(); + println!(" recipe {r}: body moved p99 {:.1} mm", percentile(&mut bm, 99.0) * 1000.0); + for (k, mode, c) in &conf { + let a = &base[k]; + let Some(b) = sh.get(k).filter(|b| b.len() == a.len()) else { + println!(" {:<28} missing / vertex count changed", k); + continue; + }; + let pred = c.apply(&db); + let mut e: Vec = b.iter().zip(a).zip(&pred).map(|((b, a), p)| ((*b - *a) - DVec3::from(glam::Vec3::from(*p).as_dvec3())).length() * 1000.0).collect(); + let mut mv: Vec = b.iter().zip(a).map(|(b, a)| (*b - *a).length() * 1000.0).collect(); + let name: String = k.chars().take(20).collect(); + println!( + " {:<20} {:<8} CC5 moved it p50 {:5.1} p99 {:5.1} | conform error p50 {:5.2} p90 {:5.2} p99 {:5.2} max {:5.2} mm", + name, + mode, + percentile(&mut mv, 50.0), + percentile(&mut mv, 99.0), + percentile(&mut e, 50.0), + percentile(&mut e, 90.0), + percentile(&mut e, 99.0), + e.iter().cloned().fold(0.0, f64::max) + ); + } + } + Ok(()) +} diff --git a/crates/cc5-hair/src/follow.rs b/crates/cc5-hair/src/follow.rs new file mode 100644 index 0000000..d84b900 --- /dev/null +++ b/crates/cc5-hair/src/follow.rs @@ -0,0 +1,285 @@ +//! Which hair layers CC5 moves rigidly with the head box or turns with a bone +//! (port of cc5_layer_follow.py), from cc5_export_smart_hair.py's "shapes" OBJs +//! (`shape_base.obj`, `shape_.obj`) and "bones" dumps +//! (`bones_.json`) in a group's folder. +//! +//! The catalog's `follow` gets `{mesh: "head_box" | "bone:@x,y,z"}` +//! for the part meshes made from those layers; avatar_editor conforms them so. + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; + +use anyhow::{Context, Result}; +use glam::{DMat3, DMat4, DQuat, DVec3, DVec4}; +use serde_json::Value; + +/// Layers that are not hair (body, eyes, teeth, accessories). +pub fn not_hair(name: &str) -> bool { + let n = name.to_lowercase(); + n.starts_with("cc_base_") || ["eyelash", "earring", "heels", "shoe", "cloth", "teeth", "tongue"].iter().any(|w| n.contains(w)) +} + +/// Layer names as glTF importers / OBJ writers spell them. +pub fn key(name: &str) -> String { + let base = name.split('.').next().unwrap_or("").to_lowercase(); + let mut out = String::new(); + let mut gap = false; + for ch in base.chars() { + if ch.is_ascii_alphanumeric() { + if gap && !out.is_empty() { + out.push('_'); + } + out.push(ch); + gap = false; + } else { + gap = true; + } + } + out +} + +/// {group name: vertices (m, CC5 frame: Z up, face -Y)} of a CC5 bind OBJ (cm). +/// A group's vertices are the ones its faces use (CC5 writes them before `g`). +pub fn read_obj(path: &Path) -> Result>> { + let text = std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + let mut verts = Vec::new(); + let mut used: BTreeMap> = BTreeMap::new(); + let mut cur: Option = None; + for line in text.lines() { + if let Some(r) = line.strip_prefix("v ") { + let v: Vec = r.split_whitespace().take(3).filter_map(|x| x.parse().ok()).collect(); + if v.len() == 3 { + verts.push(DVec3::new(v[0], v[1], v[2]) / 100.0); + } + } else if let Some(r) = line.strip_prefix("g ") { + let g = r.trim().to_string(); + used.entry(g.clone()).or_default(); + cur = Some(g); + } else if let (Some(r), Some(g)) = (line.strip_prefix("f "), cur.as_ref()) { + let set = used.get_mut(g).unwrap(); + for t in r.split_whitespace() { + if let Some(i) = t.split('/').next().and_then(|i| i.parse::().ok()) { + set.insert(i - 1); + } + } + } + } + Ok(used.into_iter().filter(|(_, ix)| !ix.is_empty()).map(|(k, ix)| (k, ix.into_iter().map(|i| verts[i]).collect())).collect()) +} + +fn shapes(dir: &Path) -> Result<(BTreeMap>, Vec<(String, BTreeMap>)>)> { + let base = read_obj(&dir.join("shape_base.obj"))?; + let mut out = Vec::new(); + let mut names: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).map(|e| e.file_name().to_string_lossy().to_string()).collect(); + names.sort(); + for n in names { + if let Some(r) = n.strip_prefix("shape_").and_then(|r| r.strip_suffix(".obj")) { + // slider recipes only: the group on another base character + // (cc5_export_on_avatar.py, `../_avatars/.obj`) is not a shape change + let avatar = dir.parent().is_some_and(|p| p.join("_avatars").join(format!("{r}.obj")).exists()); + if r != "base" && !avatar { + out.push((r.to_string(), read_obj(&dir.join(&n))?)); + } + } + } + Ok((base, out)) +} + +/// Least-squares affine fit `B ≈ [A 1] M` (M: 4×3 as rows of a DMat4's xyz). +fn affine_fit(a: &[DVec3], b: &[DVec3]) -> [[f64; 3]; 4] { + let mut ata = [[0.0f64; 4]; 4]; + let mut atb = [[0.0f64; 3]; 4]; + for (p, q) in a.iter().zip(b) { + let x = [p.x, p.y, p.z, 1.0]; + for i in 0..4 { + for j in 0..4 { + ata[i][j] += x[i] * x[j]; + } + for k in 0..3 { + atb[i][k] += x[i] * q[k]; + } + } + } + let inv = DMat4::from_cols_array_2d(&ata).inverse(); + let mut m = [[0.0f64; 3]; 4]; + for k in 0..3 { + let col = inv * DVec4::new(atb[0][k], atb[1][k], atb[2][k], atb[3][k]); + for i in 0..4 { + m[i][k] = col[i]; + } + } + m +} + +/// {layer key: "head_box" | "surface"}: a layer CC5 moved by an exact per-axis +/// scale + translation in every recipe is rigid with the head box. +pub fn layer_modes(dir: &Path, tol_mm: f64) -> Result> { + let (base, recs) = shapes(dir)?; + let mut out = HashMap::new(); + for (k, a) in &base { + if not_hair(k) || recs.is_empty() { + continue; + } + let (mut rigid, mut moved) = (true, false); + for (_, sh) in &recs { + let Some(b) = sh.get(k).filter(|b| b.len() == a.len()) else { + rigid = false; + break; + }; + let m = affine_fit(a, b); + let diag_only = (0..3).all(|i| (0..3).all(|j| i == j || m[i][j].abs() < 1e-4)); + let resid = a.iter().zip(b).map(|(p, q)| { + let f = DVec3::new( + p.x * m[0][0] + p.y * m[1][0] + p.z * m[2][0] + m[3][0], + p.x * m[0][1] + p.y * m[1][1] + p.z * m[2][1] + m[3][1], + p.x * m[0][2] + p.y * m[1][2] + p.z * m[2][2] + m[3][2], + ); + f.distance(*q) + }).fold(0.0, f64::max) * 1000.0; + moved |= a.iter().zip(b).map(|(p, q)| p.distance(*q)).fold(0.0, f64::max) > 0.001; + rigid &= diag_only && resid < tol_mm; + } + out.insert(key(k), if rigid && moved { "head_box" } else { "surface" }.to_string()); + } + Ok(out) +} + +/// A bone dump entry -> (rotation, scale, position in m), CC5 frame. +fn xf(b: &Value) -> Option<(DMat3, DVec3, DVec3)> { + let f = |k: &str| -> Option> { b.get(k)?.as_array()?.iter().map(|v| v.as_f64()).collect() }; + let (r, s, t) = (f("r")?, f("s")?, f("t")?); + Some((DMat3::from_quat(DQuat::from_xyzw(r[0], r[1], r[2], r[3])), DVec3::new(s[0], s[1], s[2]), DVec3::new(t[0], t[1], t[2]) / 100.0)) +} + +fn angle(r: DMat3) -> f64 { + (((r.x_axis.x + r.y_axis.y + r.z_axis.z) - 1.0) / 2.0).clamp(-1.0, 1.0).acos().to_degrees() +} + +/// {layer key: "bone:@x,y,z"} (glTF frame) for layers CC5 turns with a +/// bone: the bone whose transform change reproduces the layer in every recipe +/// gives the attach point; the CC_Base_ joint turning the same way is the one +/// our rig has. +pub fn bone_modes(dir: &Path, tol_mm: f64) -> Result> { + let mut out = HashMap::new(); + let base_bones = dir.join("bones_base.json"); + if !base_bones.exists() { + return Ok(out); + } + let (base, recs) = shapes(dir)?; + let load = |p: &Path| -> Result> { + let v: Value = serde_json::from_str(&std::fs::read_to_string(p)?)?; + Ok(v.get("bones").and_then(|b| b.as_object()).cloned().unwrap_or_default()) + }; + let b0 = load(&base_bones)?; + let mut rb = Vec::new(); + for (r, sh) in &recs { + let p = dir.join(format!("bones_{r}.json")); + if p.exists() { + rb.push((sh, load(&p)?)); + } + } + if rb.is_empty() { + return Ok(out); + } + for (k, a) in &base { + if not_hair(k) { + continue; + } + // the bone reproducing the layer's move exactly, in every recipe + let attach = b0.iter().find_map(|(name, x0)| { + let (r0, s0, t0) = xf(x0)?; + for (sh, b1) in &rb { + let b = sh.get(k).filter(|b| b.len() == a.len())?; + let (r1, s1, t1) = xf(b1.get(name)?)?; + let ratio = s1 / s0; + if (ratio - DVec3::ONE).abs().max_element() > 1e-4 { + return None; + } + let l = r1 * DMat3::from_diagonal(ratio) * r0.transpose(); + let err = a.iter().zip(b).map(|(p, q)| (l * (*p - t0) + t1).distance(*q)).fold(0.0, f64::max) * 1000.0; + if err > tol_mm { + return None; + } + } + Some(name.clone()) + }); + let Some(attach) = attach else { continue }; + let (ra0, _, ta0) = xf(&b0[&attach]).unwrap(); + let turns: Vec = rb.iter().filter_map(|(_, b1)| xf(b1.get(&attach)?).map(|(r1, ..)| r1 * ra0.transpose())).collect(); + if !turns.iter().any(|r| angle(*r) > 0.05) { + continue; // pure translation: conform / head_box handle it + } + // the CC joint turning exactly as the attach bone does + let best = b0 + .iter() + .filter(|(n, _)| n.starts_with("CC_Base_")) + .filter_map(|(n, x0)| { + let (r0, ..) = xf(x0)?; + let errs: Vec = rb + .iter() + .zip(&turns) + .filter_map(|((_, b1), turn)| xf(b1.get(n)?).map(|(r1, ..)| angle((r1 * r0.transpose()).transpose() * *turn))) + .collect(); + (!errs.is_empty()).then(|| (errs.iter().cloned().fold(0.0, f64::max), n.clone())) + }) + .min_by(|a, b| a.0.total_cmp(&b.0)); + if let Some((err, joint)) = best { + if err < 0.05 { + out.insert(key(k), format!("bone:{joint}@{:.5},{:.5},{:.5}", ta0.x, ta0.z, -ta0.y)); + } + } + } + Ok(out) +} + +/// {mesh name: mode} for a part's meshes made from rigid source layers +/// (meshes are named `_` and `__layer`). +pub fn part_follow(mesh_names: &[String], part_id: &str, modes: &HashMap) -> BTreeMap { + let mut out = BTreeMap::new(); + for m in mesh_names { + let Some(src) = m.strip_prefix(&format!("{part_id}_")) else { continue }; + let src = match src.rfind("_layer") { + Some(i) if src[i + 6..].chars().all(|c| c.is_ascii_digit()) && i + 6 < src.len() => &src[..i], + _ => src, + }; + if let Some(mode) = modes.get(&key(src)) { + if mode == "head_box" || mode.starts_with("bone:") { + out.insert(m.clone(), mode.clone()); + } + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_like_python() { + assert_eq!(key("Viking Beard"), "viking_beard"); + assert_eq!(key("Brows_Hair.002"), "brows_hair"); + assert_eq!(key("--Side part wavy--"), "side_part_wavy"); + } + + /// The Viking group (CC5's own exports): the braid rides the jaw, the + /// own-hair bangs move with the head box. Skipped where the data is absent. + #[test] + fn viking_follow_matches_python() { + let dir = Path::new(r"D:\cc5_editor\smart_hair\Viking"); + if !dir.join("bones_base.json").exists() { + return; + } + let bones = bone_modes(dir, 0.1).unwrap(); + assert_eq!(bones.get("viking_beard").map(String::as_str), Some("bone:CC_Base_JawRoot@0.00158,1.54476,0.08019")); + let layers = layer_modes(dir, 0.05).unwrap(); + assert_eq!(layers.get("bang").map(String::as_str), Some("head_box")); + assert_eq!(layers.get("real_hair").map(String::as_str), Some("head_box")); + assert_eq!(layers.get("hair_base").map(String::as_str), Some("surface")); + let names: Vec = ["sh_viking_Viking_Beard", "sh_viking_Viking_Beard_layer1", "sh_viking_Chinstrap_Thick"].map(String::from).to_vec(); + let mut modes = layers; + modes.extend(bones); + let f = part_follow(&names, "sh_viking", &modes); + assert_eq!(f.len(), 2); + } +} diff --git a/crates/cc5-hair/src/glb_out.rs b/crates/cc5-hair/src/glb_out.rs new file mode 100644 index 0000000..b6c8301 --- /dev/null +++ b/crates/cc5-hair/src/glb_out.rs @@ -0,0 +1,415 @@ +//! Write a [`BuiltPart`] as a library part GLB: the head's armature and skin +//! (joints re-bound by name in the editor), one skinned mesh per part mesh, +//! attributes quantized (u8 colours / weights, u16 UVs and indices, primitives +//! of at most 65 536 vertices). +//! +//! Strands are written as centre lines (glTF LINES, one vertex per strand +//! point: POSITION, TEXCOORD_0 = (strand ID, t), COLOR_0, TEXCOORD_1.x = the +//! half-width / [`HALF_WIDTH_MAX`], skin) that the editor draws as camera-facing +//! ribbons on the GPU; their materials carry `extras.cc5.ribbons`. With +//! [`Output::triangles`], as fixed triangle ribbons instead (hair_build.py's +//! `make_strand_object`; for comparison). + +use std::path::Path; + +use anyhow::{anyhow, Result}; +use glam::{Vec2, Vec3}; +use serde_json::{json, Value}; + +use crate::part::{BuiltPart, Geometry}; +use crate::source::Source; +use crate::{Strand, HEAD_CENTER}; + +/// Largest strand half-width a LINES part stores (metres; TEXCOORD_1.x is the +/// half-width over this, as u16). +pub const HALF_WIDTH_MAX: f32 = 0.002; + +/// How strands are written. +#[derive(Clone, Copy, Default)] +pub struct Output { + /// Fixed triangle ribbons (2 vertices per point) instead of centre lines. + pub triangles: bool, +} + +/// A strand's width at t (0 root .. 1 tip): tapered towards the tip, faded in at the root. +fn width_at(s: &Strand, tt: f32, taper: f32, root_fade: f32) -> f32 { + let mut w = s.width * (1.0 - taper * tt * tt); + if root_fade > 0.0 { + let x = (tt / root_fade).clamp(0.0, 1.0); + w *= x * x * (3.0 - 2.0 * x); + } + w +} + +/// Strand centre lines: one vertex per point; UV = (strand ID, t from root 0 +/// to tip 1); colour = texture colour × shade; segments between consecutive points. +pub struct Lines { + pub pos: Vec, + pub uv: Vec, + pub col: Vec<[f32; 4]>, + pub half_width: Vec, + pub skin: Vec<[(u16, f32); 4]>, + pub segs: Vec<[u32; 2]>, +} + +pub fn lines(strands: &[Strand], skin: &[[(u16, f32); 4]], taper: f32, root_fade: f32) -> Lines { + let n: usize = strands.iter().map(|s| s.pts.len()).sum(); + let mut l = Lines { + pos: Vec::with_capacity(n), + uv: Vec::with_capacity(n), + col: Vec::with_capacity(n), + half_width: Vec::with_capacity(n), + skin: Vec::with_capacity(n), + segs: Vec::with_capacity(n), + }; + for (s, sk) in strands.iter().zip(skin) { + let m = s.pts.len(); + let o = l.pos.len() as u32; + for i in 0..m { + let tt = i as f32 / (m - 1) as f32; + l.pos.push(s.pts[i]); + l.uv.push(Vec2::new(0.02 + 0.96 * s.id, tt)); + let c = (s.rgb[i] * s.shade).clamp(Vec3::ZERO, Vec3::ONE); + l.col.push([c.x, c.y, c.z, 1.0]); + l.half_width.push(0.5 * width_at(s, tt, taper, root_fade)); + l.skin.push(*sk); + } + l.segs.extend((0..m as u32 - 1).map(|k| [o + k, o + k + 1])); + } + l +} + +/// Ribbon vertices of strands (hair_build.py `make_strand_object`): two rows per +/// strand across its width, facing out from the head; UV = (strand ID, t from +/// root 0 to tip 1); colour = texture colour × shade. +pub struct Ribbons { + pub pos: Vec, + pub nrm: Vec, + pub uv: Vec, + pub col: Vec<[f32; 4]>, + pub skin: Vec<[(u16, f32); 4]>, + pub tris: Vec<[u32; 3]>, +} + +pub fn ribbons(strands: &[Strand], skin: &[[(u16, f32); 4]], taper: f32, root_fade: f32) -> Ribbons { + let n: usize = strands.iter().map(|s| 2 * s.pts.len()).sum(); + let mut r = Ribbons { pos: Vec::with_capacity(n), nrm: Vec::with_capacity(n), uv: Vec::with_capacity(n), col: Vec::with_capacity(n), skin: Vec::with_capacity(n), tris: Vec::new() }; + for (s, sk) in strands.iter().zip(skin) { + let m = s.pts.len(); + let o = r.pos.len() as u32; + let p = &s.pts; + let d: Vec = (0..m) + .map(|i| { + let g = if i == 0 { p[1] - p[0] } else if i == m - 1 { p[m - 1] - p[m - 2] } else { (p[i + 1] - p[i - 1]) / 2.0 }; + g.normalize_or_zero() + }) + .collect(); + for sgn in [-1.0f32, 1.0] { + for i in 0..m { + let tt = i as f32 / (m - 1) as f32; + let w = width_at(s, tt, taper, root_fade); + // across the strand, facing out from the head + let mut side = d[i].cross(p[i] - HEAD_CENTER); + if side.length() < 1e-9 { + side = d[i].cross(Vec3::X); + } + let side = side.normalize_or_zero(); + r.pos.push(p[i] + side * (0.5 * sgn * w)); + r.nrm.push(d[i].cross(side).normalize_or_zero()); + r.uv.push(Vec2::new(0.02 + 0.96 * s.id, tt)); + let c = (s.rgb[i] * s.shade).clamp(Vec3::ZERO, Vec3::ONE); + r.col.push([c.x, c.y, c.z, 1.0]); + r.skin.push(*sk); + } + } + let (a, b) = (o, o + m as u32); + for k in 0..m as u32 - 1 { + r.tris.push([a + k, a + k + 1, b + k + 1]); + r.tris.push([a + k, b + k + 1, b + k]); + } + } + r +} + +/// GLB under construction: JSON arrays and the binary chunk. +struct Doc { + json: Value, + bin: Vec, +} + +impl Doc { + fn push(&mut self, key: &str, v: Value) -> usize { + let a = self.json[key].as_array_mut().unwrap(); + a.push(v); + a.len() - 1 + } + + fn view(&mut self, bytes: &[u8], target: Option) -> usize { + while self.bin.len() % 4 != 0 { + self.bin.push(0); + } + let off = self.bin.len(); + self.bin.extend_from_slice(bytes); + let mut v = json!({"buffer": 0, "byteOffset": off, "byteLength": bytes.len()}); + if let Some(t) = target { + v["target"] = json!(t); + } + self.push("bufferViews", v) + } + + fn accessor(&mut self, bytes: &[u8], target: Option, ctype: u32, count: usize, ty: &str, normalized: bool, minmax: Option<(Vec3, Vec3)>) -> usize { + let view = self.view(bytes, target); + let mut a = json!({"bufferView": view, "componentType": ctype, "count": count, "type": ty}); + if normalized { + a["normalized"] = json!(true); + } + if let Some((lo, hi)) = minmax { + a["min"] = json!(lo.to_array()); + a["max"] = json!(hi.to_array()); + } + self.push("accessors", a) + } +} + +fn f32_bytes(v: impl IntoIterator) -> Vec { + v.into_iter().flat_map(f32::to_le_bytes).collect() +} + +/// Weights as u8 summing to 255 per vertex, with u8 joint indices. +fn skin_bytes(skin: &[[(u16, f32); 4]]) -> (Vec, Vec) { + let mut j = Vec::with_capacity(skin.len() * 4); + let mut w = Vec::with_capacity(skin.len() * 4); + for s in skin { + let mut q: [i32; 4] = s.map(|e| (e.1 * 255.0).round() as i32); + q[0] += 255 - q.iter().sum::(); + for k in 0..4 { + j.push(s[k].0 as u8); + w.push(q[k].clamp(0, 255) as u8); + } + } + (j, w) +} + +/// One primitive's attributes from vertex range `vr` of a mesh's arrays. +#[allow(clippy::too_many_arguments)] +fn primitive(doc: &mut Doc, pos: &[Vec3], nrm: &[Vec3], uv: &[Vec2], col: Option<&[[f32; 4]]>, skin: &[[(u16, f32); 4]], tris: &[[u32; 3]], material: usize) -> Value { + let (lo, hi) = pos.iter().fold((Vec3::MAX, Vec3::MIN), |(lo, hi), p| (lo.min(*p), hi.max(*p))); + let n = pos.len(); + let p = doc.accessor(&f32_bytes(pos.iter().flat_map(|v| v.to_array())), Some(34962), 5126, n, "VEC3", false, Some((lo, hi))); + let nr = doc.accessor(&f32_bytes(nrm.iter().flat_map(|v| v.to_array())), Some(34962), 5126, n, "VEC3", false, None); + let in_unit = uv.iter().all(|u| u.x >= 0.0 && u.x <= 1.0 && u.y >= 0.0 && u.y <= 1.0); + let t = if in_unit { + let b: Vec = uv.iter().flat_map(|u| [u.x, u.y]).flat_map(|x| ((x * 65535.0).round() as u16).to_le_bytes()).collect(); + doc.accessor(&b, Some(34962), 5123, n, "VEC2", true, None) + } else { + doc.accessor(&f32_bytes(uv.iter().flat_map(|u| u.to_array())), Some(34962), 5126, n, "VEC2", false, None) + }; + let (jb, wb) = skin_bytes(skin); + let j = doc.accessor(&jb, Some(34962), 5121, n, "VEC4", false, None); + let w = doc.accessor(&wb, Some(34962), 5121, n, "VEC4", true, None); + let mut attrs = json!({"POSITION": p, "NORMAL": nr, "TEXCOORD_0": t, "JOINTS_0": j, "WEIGHTS_0": w}); + if let Some(col) = col { + let b: Vec = col.iter().flat_map(|c| c.map(|x| (x.clamp(0.0, 1.0) * 255.0).round() as u8)).collect(); + attrs["COLOR_0"] = json!(doc.accessor(&b, Some(34962), 5121, n, "VEC4", true, None)); + } + let ib: Vec = tris.iter().flatten().flat_map(|&i| (i as u16).to_le_bytes()).collect(); + let idx = doc.accessor(&ib, Some(34963), 5123, tris.len() * 3, "SCALAR", false, None); + json!({"attributes": attrs, "indices": idx, "material": material, "mode": 4}) +} + +/// A LINES primitive of strand centre lines (a vertex range of [`Lines`]). +#[allow(clippy::too_many_arguments)] +fn line_primitive(doc: &mut Doc, pos: &[Vec3], uv: &[Vec2], col: &[[f32; 4]], half: &[f32], skin: &[[(u16, f32); 4]], segs: &[[u32; 2]], material: usize) -> Value { + let (lo, hi) = pos.iter().fold((Vec3::MAX, Vec3::MIN), |(lo, hi), p| (lo.min(*p), hi.max(*p))); + let n = pos.len(); + let unorm16 = |x: f32| ((x.clamp(0.0, 1.0) * 65535.0).round() as u16).to_le_bytes(); + let p = doc.accessor(&f32_bytes(pos.iter().flat_map(|v| v.to_array())), Some(34962), 5126, n, "VEC3", false, Some((lo, hi))); + let t0: Vec = uv.iter().flat_map(|u| [u.x, u.y]).flat_map(unorm16).collect(); + let t0 = doc.accessor(&t0, Some(34962), 5123, n, "VEC2", true, None); + let t1: Vec = half.iter().flat_map(|&h| [h / HALF_WIDTH_MAX, 0.0]).flat_map(unorm16).collect(); + let t1 = doc.accessor(&t1, Some(34962), 5123, n, "VEC2", true, None); + let c: Vec = col.iter().flat_map(|c| c.map(|x| (x.clamp(0.0, 1.0) * 255.0).round() as u8)).collect(); + let c = doc.accessor(&c, Some(34962), 5121, n, "VEC4", true, None); + let (jb, wb) = skin_bytes(skin); + let j = doc.accessor(&jb, Some(34962), 5121, n, "VEC4", false, None); + let w = doc.accessor(&wb, Some(34962), 5121, n, "VEC4", true, None); + let ib: Vec = segs.iter().flatten().flat_map(|&i| (i as u16).to_le_bytes()).collect(); + let idx = doc.accessor(&ib, Some(34963), 5123, segs.len() * 2, "SCALAR", false, None); + json!({"attributes": {"POSITION": p, "TEXCOORD_0": t0, "TEXCOORD_1": t1, "COLOR_0": c, "JOINTS_0": j, "WEIGHTS_0": w}, + "indices": idx, "material": material, "mode": 1}) +} + +/// Split triangles (or segments) into primitives whose vertex ranges fit u16 +/// indices. They must reference increasing vertex ranges (ribbons, lines and +/// cards are so laid out; one never spans more than 65 536 vertices). +fn windows(tris: &[[u32; N]], nverts: usize) -> Vec<(usize, usize, Vec<[u32; N]>)> { + let mut out = Vec::new(); + let (mut lo, mut cur): (u32, Vec<[u32; N]>) = (0, Vec::new()); + let mut hi = 0u32; + for t in tris { + let (tl, th) = (*t.iter().min().unwrap(), *t.iter().max().unwrap()); + if !cur.is_empty() && (th.max(hi) - lo.min(tl) >= 65535) { + out.push((lo as usize, hi as usize + 1, std::mem::take(&mut cur))); + } + if cur.is_empty() { + lo = tl; + hi = th; + } + lo = lo.min(tl); + hi = hi.max(th); + cur.push(*t); + } + if !cur.is_empty() { + out.push((lo as usize, hi as usize + 1, cur)); + } + if out.is_empty() && nverts > 0 { + out.push((0, nverts.min(65535), Vec::new())); + } + out.into_iter().map(|(a, b, ts)| (a, b, ts.into_iter().map(|t| t.map(|i| i - a as u32)).collect())).collect() +} + +/// Write `part` to `path`, with `head`'s armature (nodes + skin). +pub fn write(part: &BuiltPart, head: &Source, path: &Path, out: Output) -> Result<()> { + write_with(part, &part.materials, head, path, out) +} + +/// [`write`] with other materials (same count and order: e.g. edited colours). +pub fn write_with(part: &BuiltPart, materials: &[crate::part::PartMaterial], head: &Source, path: &Path, out: Output) -> Result<()> { + // materials of strand meshes drawn as GPU ribbons + let line_mats: std::collections::HashSet = if out.triangles { + Default::default() + } else { + part.meshes.iter().filter(|m| matches!(m.geom, Geometry::Strands { .. })).map(|m| m.material).collect() + }; + let mut doc = Doc { + json: json!({"asset": {"version": "2.0", "generator": "cc5-hair"}, "scene": 0, "scenes": [{"nodes": []}], + "nodes": [], "meshes": [], "materials": [], "accessors": [], "bufferViews": [], "buffers": [], + "skins": [], "images": [], "textures": [], "samplers": []}), + bin: Vec::new(), + }; + // the head's armature: every node that is not a mesh node, same hierarchy and transforms + let skin = head.doc.skins().next().ok_or_else(|| anyhow!("{}: no skin", head.path.display()))?; + let mesh_nodes: std::collections::HashSet = head.doc.nodes().filter(|n| n.mesh().is_some()).map(|n| n.index()).collect(); + let mut remap = std::collections::HashMap::new(); + for n in head.doc.nodes() { + if !mesh_nodes.contains(&n.index()) { + remap.insert(n.index(), remap.len()); + } + } + let mut nodes = vec![Value::Null; remap.len()]; + for n in head.doc.nodes() { + let Some(&i) = remap.get(&n.index()) else { continue }; + let (t, r, s) = n.transform().decomposed(); + let mut v = json!({"name": n.name().unwrap_or(""), "translation": t, "rotation": r, "scale": s}); + let kids: Vec = n.children().filter_map(|c| remap.get(&c.index()).copied()).collect(); + if !kids.is_empty() { + v["children"] = json!(kids); + } + nodes[i] = v; + } + let roots: Vec = head.doc.scenes().next().map(|s| s.nodes().filter_map(|n| remap.get(&n.index()).copied()).collect()).unwrap_or_default(); + let armature = *roots.first().ok_or_else(|| anyhow!("head: no armature root"))?; + doc.json["nodes"] = Value::Array(nodes); + let ibm = skin.reader(|b| (b.index() == 0).then_some(head.blob.as_slice())).read_inverse_bind_matrices().ok_or_else(|| anyhow!("head skin: no inverse binds"))?; + let ibm_bytes = f32_bytes(ibm.flat_map(|m| m.into_iter().flatten())); + let nj = skin.joints().count(); + let ibm_acc = doc.accessor(&ibm_bytes, None, 5126, nj, "MAT4", false, None); + let joints: Vec = skin.joints().map(|j| remap[&j.index()]).collect(); + doc.push("skins", json!({"joints": joints, "inverseBindMatrices": ibm_acc})); + doc.json["scenes"][0]["nodes"] = json!(roots); + doc.push("samplers", json!({"magFilter": 9729, "minFilter": 9987, "wrapS": 10497, "wrapT": 10497})); + // materials + for (mi, m) in materials.iter().enumerate() { + let mut v = json!({"name": m.name, "doubleSided": true, + "pbrMetallicRoughness": {"metallicFactor": 0.0, "roughnessFactor": 0.5, "baseColorFactor": m.base_factor}}); + if m.blend { + v["alphaMode"] = json!("BLEND"); // one soft layer on the skin, like CC5's scalp + } else { + v["alphaMode"] = json!("MASK"); + v["alphaCutoff"] = json!(0.35); + } + if let Some((bytes, mime)) = &m.base_image { + let view = doc.view(bytes, None); + let img = doc.push("images", json!({"bufferView": view, "mimeType": mime})); + let tex = doc.push("textures", json!({"source": img, "sampler": 0})); + v["pbrMetallicRoughness"]["baseColorTexture"] = json!({"index": tex}); + } + if let Some(c) = &m.cc5 { + let mut c = c.clone(); + if line_mats.contains(&mi) { + c["ribbons"] = json!(true); + } + v["extras"] = json!({"cc5": c}); + } + doc.push("materials", v); + } + // meshes, each on a node under the armature + let mut mesh_node_ids = Vec::new(); + for pm in &part.meshes { + if let (Geometry::Strands { strands, skin, taper, root_fade }, false) = (&pm.geom, out.triangles) { + let l = lines(strands, skin, *taper, *root_fade); + let mut prims = Vec::new(); + for (a, b, segs) in windows(&l.segs, l.pos.len()) { + prims.push(line_primitive(&mut doc, &l.pos[a..b], &l.uv[a..b], &l.col[a..b], &l.half_width[a..b], &l.skin[a..b], &segs, pm.material)); + } + let mi = doc.push("meshes", json!({"name": pm.name, "primitives": prims})); + mesh_node_ids.push(doc.push("nodes", json!({"name": pm.name, "mesh": mi, "skin": 0}))); + continue; + } + let (pos, nrm, uv, col, skin, tris) = match &pm.geom { + Geometry::Strands { strands, skin, taper, root_fade } => { + let r = ribbons(strands, skin, *taper, *root_fade); + (r.pos, r.nrm, r.uv, Some(r.col), r.skin, r.tris) + } + Geometry::Cards { pos, nrm, uv, tris, skin } => (pos.clone(), nrm.clone(), uv.clone(), None, skin.clone(), tris.clone()), + }; + let mut prims = Vec::new(); + for (a, b, ts) in windows(&tris, pos.len()) { + prims.push(primitive(&mut doc, &pos[a..b], &nrm[a..b], &uv[a..b], col.as_ref().map(|c| &c[a..b]), &skin[a..b], &ts, pm.material)); + } + let mi = doc.push("meshes", json!({"name": pm.name, "primitives": prims})); + mesh_node_ids.push(doc.push("nodes", json!({"name": pm.name, "mesh": mi, "skin": 0}))); + } + let arm = &mut doc.json["nodes"][armature]; + let mut kids: Vec = arm.get("children").and_then(|c| serde_json::from_value(c.clone()).ok()).unwrap_or_default(); + kids.extend(mesh_node_ids); + arm["children"] = json!(kids); + for k in ["images", "textures"] { + if doc.json[k].as_array().is_some_and(|a| a.is_empty()) { + doc.json.as_object_mut().unwrap().remove(k); + } + } + while doc.bin.len() % 4 != 0 { + doc.bin.push(0); + } + doc.json["buffers"] = json!([{"byteLength": doc.bin.len()}]); + let mut js = serde_json::to_vec(&doc.json)?; + while js.len() % 4 != 0 { + js.push(b' '); + } + let total = 12 + 8 + js.len() + 8 + doc.bin.len(); + let mut out = Vec::with_capacity(total); + out.extend_from_slice(&0x4654_6C67u32.to_le_bytes()); + out.extend_from_slice(&2u32.to_le_bytes()); + out.extend_from_slice(&(total as u32).to_le_bytes()); + out.extend_from_slice(&(js.len() as u32).to_le_bytes()); + out.extend_from_slice(&0x4E4F_534Au32.to_le_bytes()); + out.extend_from_slice(&js); + out.extend_from_slice(&(doc.bin.len() as u32).to_le_bytes()); + out.extend_from_slice(&0x004E_4942u32.to_le_bytes()); + out.extend_from_slice(&doc.bin); + std::fs::write(path, out)?; + Ok(()) +} + +/// Copy the part's CC5 map files into the library. +pub fn copy_files(part: &BuiltPart, lib: &Path) -> Result<()> { + for (rel, src) in &part.files { + let dst = lib.join(rel); + if let Some(d) = dst.parent() { + std::fs::create_dir_all(d)?; + } + std::fs::copy(src, &dst)?; + } + Ok(()) +} diff --git a/crates/cc5-hair/src/head.rs b/crates/cc5-hair/src/head.rs new file mode 100644 index 0000000..180f9e4 --- /dev/null +++ b/crates/cc5-hair/src/head.rs @@ -0,0 +1,116 @@ +//! Nearest-point queries on the body (and on a source's scalp layers), and the +//! body's skin weights at a point (hair_build.py `Head`). + +use anyhow::{anyhow, Result}; +use glam::Vec3; +use parry3d::na::Point3; +use parry3d::query::PointQueryWithLocation; +use parry3d::shape::TriMesh; + +use crate::source::Source; + +/// A triangle surface with nearest-point queries. +pub struct Surface { + pub pos: Vec, + pub tris: Vec<[u32; 3]>, + mesh: TriMesh, +} + +/// The nearest surface point to a query point. +#[derive(Debug, Clone, Copy)] +pub struct Hit { + pub point: Vec3, + /// Unit face normal (by winding). + pub normal: Vec3, + pub dist: f32, + pub tri: u32, +} + +impl Surface { + pub fn new(pos: Vec, tris: Vec<[u32; 3]>) -> Option { + if tris.is_empty() { + return None; + } + let mesh = TriMesh::new(pos.iter().map(|p| Point3::new(p.x, p.y, p.z)).collect(), tris.clone()); + Some(Self { pos, tris, mesh }) + } + + pub fn nearest(&self, p: Vec3) -> Hit { + let (proj, (tri, _)) = self.mesh.project_local_point_and_get_location(&Point3::new(p.x, p.y, p.z), false); + let point = Vec3::new(proj.point.x, proj.point.y, proj.point.z); + let [a, b, c] = self.tris[tri as usize].map(|i| self.pos[i as usize]); + let normal = (b - a).cross(c - a).normalize_or_zero(); + Hit { point, normal, dist: point.distance(p), tri } + } +} + +/// The body (`CC_Base_Body`) of the head GLB: nearest point and skin weights. +pub struct Body { + pub surface: Surface, + /// Per vertex: (joint index into `joints`, weight) above 1e-4. + weights: Vec>, + pub joints: Vec, +} + +impl Body { + pub fn from_source(src: &Source) -> Result { + let obj = src.object("CC_Base_Body").ok_or_else(|| anyhow!("{}: no CC_Base_Body", src.path.display()))?; + let (mut pos, mut tris, mut weights) = (Vec::new(), Vec::new(), Vec::new()); + for p in &obj.prims { + let o = pos.len() as u32; + pos.extend(&p.pos); + tris.extend(p.tris.iter().map(|t| t.map(|i| i + o))); + weights.extend(p.joints.iter().zip(&p.weights).map(|(j, w)| { + (0..4).filter(|&k| w[k] > 1e-4).map(|k| (j[k], w[k])).collect::>() + })); + } + let surface = Surface::new(pos, tris).ok_or_else(|| anyhow!("CC_Base_Body has no triangles"))?; + Ok(Self { surface, weights, joints: src.skin_joints() }) + } + + pub fn nearest(&self, p: Vec3) -> Hit { + self.surface.nearest(p) + } + + /// The body's weights at the surface point nearest `p` (barycentric blend of + /// the triangle's vertices), top 4, normalised. + pub fn skin_at(&self, p: Vec3) -> [(u16, f32); 4] { + let h = self.nearest(p); + let t = self.surface.tris[h.tri as usize]; + let [a, b, c] = t.map(|i| self.surface.pos[i as usize]); + let bary = barycentric(h.point, a, b, c); + let mut acc: Vec<(u16, f32)> = Vec::new(); + for (vi, bw) in t.iter().zip(bary) { + for &(j, w) in &self.weights[*vi as usize] { + match acc.iter_mut().find(|e| e.0 == j) { + Some(e) => e.1 += bw.max(0.0) * w, + None => acc.push((j, bw.max(0.0) * w)), + } + } + } + acc.sort_by(|x, y| y.1.total_cmp(&x.1)); + acc.truncate(4); + let s: f32 = acc.iter().map(|e| e.1).sum::().max(1e-12); + let mut out = [(0u16, 0.0f32); 4]; + for (o, e) in out.iter_mut().zip(&acc) { + *o = (e.0, e.1 / s); + } + if acc.is_empty() { + out[0].1 = 1.0; + } + out + } +} + +/// Barycentric weights of `p` (on the triangle's plane) in triangle `a b c`. +pub fn barycentric(p: Vec3, a: Vec3, b: Vec3, c: Vec3) -> [f32; 3] { + let (v0, v1, v2) = (b - a, c - a, p - a); + let (d00, d01, d11, d20, d21) = (v0.dot(v0), v0.dot(v1), v1.dot(v1), v2.dot(v0), v2.dot(v1)); + let den = d00 * d11 - d01 * d01; + if den.abs() < 1e-20 { + return [1.0, 0.0, 0.0]; + } + let v = (d11 * d20 - d01 * d21) / den; + let w = (d00 * d21 - d01 * d20) / den; + [1.0 - v - w, v, w] +} diff --git a/crates/cc5-hair/src/lib.rs b/crates/cc5-hair/src/lib.rs new file mode 100644 index 0000000..449ba46 --- /dev/null +++ b/crates/cc5-hair/src/lib.rs @@ -0,0 +1,124 @@ +//! Convert Character Creator 5 hair into strand hair parts for cc-render's +//! avatar editor, in Rust (the port of hair_build.py's "convert" path, so the +//! editor can rebuild hair live). +//! +//! Stages (each a plain function, so callers can cache their outputs): +//! [`source::Source`] (a CC5 character GLB or FBX export) -> [`cards::cards_to_strands`] +//! (strands following CC5's cards) -> [`shape::finish`] (shape edits, push off +//! the head) -> [`shade::depth_shade`] -> [`part::build`] (meshes, materials, +//! catalog entry) -> [`glb_out::write`] (a library part GLB). +//! +//! Frame: glTF (metres, Y up), the CC5 GLBs' mesh space. + +pub mod cards; +pub mod check; +pub mod follow; +pub mod glb_out; +pub mod head; +pub mod part; +pub mod regions; +pub mod settings; +pub mod shade; +pub mod shape; +pub mod source; + +use glam::Vec3; +use rand::{rngs::StdRng, Rng as _, SeedableRng}; + +/// Between the eyes of the CC base character (glTF frame). +/// A CC5 Smart Hair group's export in `dir`: `character.glb` (prepared), else +/// the raw `character.fbx`. +pub fn group_source(dir: &std::path::Path) -> Option { + ["character.glb", "character.fbx"].iter().map(|f| dir.join(f)).find(|p| p.exists()) +} + +pub const HEAD_CENTER: Vec3 = Vec3::new(0.0, 1.64, 0.005); + +/// One strand: a polyline from the root, with per-point linear colour. +#[derive(Debug, Clone)] +pub struct Strand { + pub pts: Vec, + /// Random 0..1 (the ID map's value: per-strand variation in the shader). + pub id: f32, + /// Ribbon width at the root (m). + pub width: f32, + /// Brightness factor. + pub shade: f32, + /// Linear RGB per point. + pub rgb: Vec, +} + +/// Seeded random numbers (Python's `random.Random` calls the port needs). +pub struct Rng(StdRng); + +impl Rng { + pub fn new(seed: u64) -> Self { + Self(StdRng::seed_from_u64(seed)) + } + /// Uniform in 0..1. + pub fn f(&mut self) -> f64 { + self.0.gen::() + } + pub fn uniform(&mut self, a: f64, b: f64) -> f64 { + a + (b - a) * self.f() + } + /// Normal deviate (Box-Muller). + pub fn gauss(&mut self, mu: f64, sigma: f64) -> f64 { + let u1 = self.f().max(1e-300); + let u2 = self.f(); + mu + sigma * (-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos() + } + pub fn below(&mut self, n: usize) -> usize { + self.0.gen_range(0..n) + } +} + +/// sRGB-encoded 0..1 -> linear. +pub fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { + c / 12.92 + } else { + ((c + 0.055) / 1.055).powf(2.4) + } +} + +/// `n` points evenly spaced along a polyline. +pub fn uniform_polyline(pts: &[Vec3], n: usize) -> Vec { + if pts.len() < 2 || n < 2 { + return vec![pts.first().copied().unwrap_or(Vec3::ZERO); n.max(1)]; + } + let mut s = vec![0.0f32]; + for w in pts.windows(2) { + s.push(s.last().unwrap() + w[0].distance(w[1])); + } + let total = *s.last().unwrap(); + let mut out = Vec::with_capacity(n); + let mut j = 0; + for i in 0..n { + let t = total * i as f32 / (n - 1) as f32; + while j + 1 < s.len() - 1 && s[j + 1] < t { + j += 1; + } + let seg = s[j + 1] - s[j]; + let f = if seg > 0.0 { ((t - s[j]) / seg).clamp(0.0, 1.0) } else { 0.0 }; + out.push(pts[j].lerp(pts[j + 1], f)); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn polyline_resamples_evenly() { + let pts = [Vec3::ZERO, Vec3::new(1.0, 0.0, 0.0), Vec3::new(1.0, 3.0, 0.0)]; + let out = uniform_polyline(&pts, 5); + assert_eq!(out.len(), 5); + assert!(out[0].distance(Vec3::ZERO) < 1e-6); + assert!(out[4].distance(Vec3::new(1.0, 3.0, 0.0)) < 1e-5); + // 4 units of length: one point per unit + assert!(out[1].distance(Vec3::new(1.0, 0.0, 0.0)) < 1e-5); + assert!(out[2].distance(Vec3::new(1.0, 1.0, 0.0)) < 1e-5); + } +} diff --git a/crates/cc5-hair/src/main.rs b/crates/cc5-hair/src/main.rs new file mode 100644 index 0000000..c1211b6 --- /dev/null +++ b/crates/cc5-hair/src/main.rs @@ -0,0 +1,206 @@ +//! `cc5-hair`: build hair library parts for cc-render's avatar editor. +//! +//! cc5-hair bake --head --lib [--settings ] [--style ] [--ribbons] +//! cc5-hair smart-hair --src --head --lib +//! --config [--only [,...]] [--ribbons] +//! cc5-hair regions --lib +//! cc5-hair check --src [ ...] +//! +//! `bake`: one part from a spec (e.g. specs/cc5_converted.json, the character's +//! own hair). `smart-hair`: every CC5 Smart Hair group export (`/character.glb` +//! or `.fbx`) with the per-kind / per-group settings of smart_hair.json; catalog.json +//! and styles.json ("CC5 ") are updated. Strands are written as centre lines +//! the editor expands into ribbons on the GPU; `--ribbons`: as triangle ribbons. +//! `--style`: also a style of that one part. `regions`: the library's +//! regions.json (conform anchors) from the reference character. `check`: the +//! conform against CC5's own reshaped hair (the groups' shape_*.obj). + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use anyhow::{anyhow, bail, Context, Result}; +use cc5_hair::head::Body; +use cc5_hair::settings::Spec; +use cc5_hair::source::Source; +use cc5_hair::{glb_out, part}; +use serde_json::{json, Value}; + +fn arg(args: &[String], name: &str) -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned()) +} + +fn need(args: &[String], name: &str) -> Result { + arg(args, name).map(PathBuf::from).ok_or_else(|| anyhow!("missing {name}")) +} + +fn read_json(p: &Path) -> Result { + serde_json::from_str(&std::fs::read_to_string(p).with_context(|| format!("reading {}", p.display()))?).with_context(|| format!("parsing {}", p.display())) +} + +/// Merge `over` into `base` (objects: key by key, one level). +fn merge(base: &mut Value, over: &Value) { + if let (Some(b), Some(o)) = (base.as_object_mut(), over.as_object()) { + for (k, v) in o { + b.insert(k.clone(), v.clone()); + } + } +} + +struct Library { + dir: PathBuf, + catalog: Value, + styles: Value, + output: glb_out::Output, +} + +impl Library { + fn open(dir: &Path) -> Result { + let cat = dir.join("catalog.json"); + let sty = dir.join("styles.json"); + Ok(Self { + dir: dir.to_path_buf(), + catalog: if cat.exists() { read_json(&cat)? } else { json!({"parts": []}) }, + styles: if sty.exists() { read_json(&sty)? } else { json!({"styles": []}) }, + output: glb_out::Output::default(), + }) + } + + fn put_part(&mut self, entry: Value) { + let parts = self.catalog["parts"].as_array_mut().unwrap(); + parts.retain(|p| p["id"] != entry["id"]); + parts.push(entry); + } + + fn put_style(&mut self, name: &str, parts: Vec) { + let styles = self.styles["styles"].as_array_mut().unwrap(); + styles.retain(|s| s["name"] != name); + styles.push(json!({"name": name, "parts": parts})); + } + + fn save(&self) -> Result<()> { + std::fs::write(self.dir.join("catalog.json"), serde_json::to_string_pretty(&self.catalog)?)?; + std::fs::write(self.dir.join("styles.json"), serde_json::to_string_pretty(&self.styles)?)?; + Ok(()) + } +} + +fn bake_one(spec: &Spec, head: &Source, body: &Body, lib: &mut Library) -> Result<()> { + let t = Instant::now(); + let built = part::build(spec, head, body, &lib.dir)?; + let t_build = t.elapsed().as_secs_f32(); + glb_out::write(&built, head, &lib.dir.join(format!("{}.glb", spec.id)), lib.output)?; + glb_out::copy_files(&built, &lib.dir)?; + let size = std::fs::metadata(lib.dir.join(format!("{}.glb", spec.id)))?.len(); + println!( + "{}: {} strands/cards, {} verts, {:.0} MB (build {:.1} s, total {:.1} s){}", + spec.id, + built.entry["cards"], + built.entry["verts"], + size as f64 / 1048576.0, + t_build, + t.elapsed().as_secs_f32(), + built.entry.get("follow").map(|f| format!(" follow {f}")).unwrap_or_default() + ); + lib.put_part(built.entry); + Ok(()) +} + +fn main() -> Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let cmd = args.first().cloned().unwrap_or_default(); + match cmd.as_str() { + "regions" => { + let src = PathBuf::from(args.get(1).ok_or_else(|| anyhow!("regions --lib "))?); + let lib_dir = need(&args, "--lib")?; + std::fs::create_dir_all(&lib_dir)?; + let r = cc5_hair::regions::regions(&cc5_hair::regions::body_points(&src)?); + std::fs::write(lib_dir.join("regions.json"), serde_json::to_string(&r.to_json())?)?; + println!("regions.json: scalp {}, face {}, head_box {}, head {} anchors", r.scalp.len(), r.face.len(), r.head_box.len(), r.head.len()); + return Ok(()); + } + "check" => { + let src = need(&args, "--src")?; + let groups: Vec<&String> = args[1..].iter().filter(|a| !a.starts_with("--") && **a != src.to_string_lossy()).collect(); + anyhow::ensure!(!groups.is_empty(), "check --src [ ...]"); + for g in groups { + cc5_hair::check::check(&src, g)?; + } + return Ok(()); + } + _ => {} + } + let head_path = need(&args, "--head")?; + let lib_dir = need(&args, "--lib")?; + std::fs::create_dir_all(&lib_dir)?; + let t = Instant::now(); + let head = Source::load(&head_path)?; + let body = Body::from_source(&head)?; + println!("head {} loaded in {:.1} s", head_path.display(), t.elapsed().as_secs_f32()); + let mut lib = Library::open(&lib_dir)?; + lib.output.triangles = args.iter().any(|a| a == "--ribbons"); + match cmd.as_str() { + "bake" => { + let spec_path = PathBuf::from(args.get(1).ok_or_else(|| anyhow!("bake "))?); + let mut v = read_json(&spec_path)?; + if let Some(s) = arg(&args, "--settings") { + let over = read_json(Path::new(&s))?; + merge(&mut v["convert"], &over); + } + let spec: Spec = serde_json::from_value(v)?; + bake_one(&spec, &head, &body, &mut lib)?; + if let Some(name) = arg(&args, "--style") { + lib.put_style(&name, vec![spec.id.clone()]); + } + } + "smart-hair" => { + let src = need(&args, "--src")?; + let cfg = read_json(&need(&args, "--config")?)?; + let only: Option> = arg(&args, "--only").map(|s| s.split(',').map(|x| x.trim().to_string()).collect()); + let beards: Vec = serde_json::from_value(cfg["beards"].clone())?; + let mut groups: Vec<_> = std::fs::read_dir(&src)?.filter_map(|e| e.ok()).map(|e| e.path()).filter(|p| cc5_hair::group_source(p).is_some()).collect(); + groups.sort(); + for gdir in groups { + let g = gdir.file_name().unwrap().to_string_lossy().to_string(); + if only.as_ref().is_some_and(|o| !o.contains(&g)) { + continue; + } + let kind = if beards.contains(&g) { "beard" } else if g.starts_with("Male_") || g.starts_with("Female_") { "brows" } else { "hair" }; + let k = &cfg["kinds"][kind]; + let mut conv = k["convert"].clone(); + conv["source_glb"] = json!(cc5_hair::group_source(&gdir).unwrap()); + if gdir.join("shape_base.obj").exists() { + conv["shape_dir"] = json!(gdir); // which layers CC5 moves with the head box / a bone + } + if let Some(o) = cfg["groups"].get(&g) { + merge(&mut conv, o); + } + let mut slug = String::new(); + for ch in g.to_lowercase().chars() { + if ch.is_ascii_alphanumeric() { + slug.push(ch); + } else if !slug.ends_with('_') { + slug.push('_'); + } + } + let pid = format!("sh_{}", slug.trim_matches('_')); + let spec = Spec { + id: pid.clone(), + slot: k["slot"].as_str().unwrap_or("Base").into(), + kind: k["kind"].as_str().unwrap_or("scalp").into(), + atlas: "strands".into(), + seed: cfg["seed"].as_u64().unwrap_or(101), + vars: Default::default(), + convert: conv, + }; + bake_one(&spec, &head, &body, &mut lib).with_context(|| g.clone())?; + // beards / brows on the converted own hair, as in CC5 + let parts = if kind == "hair" { vec![pid] } else { vec!["cc5_converted".to_string(), pid] }; + lib.put_style(&format!("CC5 {g}"), parts); + } + } + _ => bail!("usage: cc5-hair bake | smart-hair --src --config (--head --lib ) | regions | check"), + } + lib.save()?; + println!("done in {:.1} s", t.elapsed().as_secs_f32()); + Ok(()) +} diff --git a/crates/cc5-hair/src/part.rs b/crates/cc5-hair/src/part.rs new file mode 100644 index 0000000..9e72441 --- /dev/null +++ b/crates/cc5-hair/src/part.rs @@ -0,0 +1,356 @@ +//! Build one library part from a spec (hair_build.py `build`, "convert" path): +//! strands for the source's hair cards, CC5's own cards for `keep_cards` +//! layers (brows) and the source's scalp layers, materials with `extras.cc5`, +//! the catalog entry. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, bail, Context, Result}; +use glam::{Vec2, Vec3}; +use serde_json::{json, Map, Value}; + +use crate::cards::{cards_to_strands, Tex}; +use crate::follow; +use crate::head::{Body, Surface}; +use crate::settings::{Convert, Spec}; +use crate::source::Source; +use crate::{shade, shape, Strand}; + +/// A built part: meshes, materials and the catalog entry. +pub struct BuiltPart { + pub id: String, + pub meshes: Vec, + pub materials: Vec, + pub entry: Value, + /// CC5 map files the part's materials use ({lib-relative path: source file}). + pub files: Vec<(String, PathBuf)>, +} + +pub struct PartMesh { + pub name: String, + pub material: usize, + pub geom: Geometry, +} + +pub enum Geometry { + /// Strands; skin: each strand rides the body's weights at its root. + Strands { strands: Vec, skin: Vec<[(u16, f32); 4]>, taper: f32, root_fade: f32 }, + /// CC5's own cards (kept layers): per-vertex skin from the body. + Cards { pos: Vec, nrm: Vec, uv: Vec, tris: Vec<[u32; 3]>, skin: Vec<[(u16, f32); 4]> }, +} + +pub struct PartMaterial { + pub name: String, + /// Alpha-blended soft layer on the skin (scalp, brow base), else alpha-tested hair. + pub blend: bool, + pub cc5: Option, + pub base_factor: [f32; 4], + /// Encoded base colour image (bytes, MIME type). + pub base_image: Option<(Vec, String)>, +} + +/// Hair, beard, brow ... materials: CC5 names its hair materials after them. +fn is_hair_material(name: &str) -> bool { + let n = name.to_lowercase(); + ["hair", "beard", "brow", "mustache", "moustache", "stubble", "goatee", "sideburn", "bang"].iter().any(|w| n.contains(w)) +} + +/// The material holding the hair cards: the hair-named one with most triangles. +fn main_material(src: &Source, obj: &crate::source::Object) -> Option { + let mut counts: HashMap = HashMap::new(); + for p in &obj.prims { + if let Some(m) = p.material { + *counts.entry(m).or_default() += p.tris.len(); + } + } + counts + .into_iter() + .filter(|(m, _)| { + let n = src.materials[*m].name.to_lowercase(); + is_hair_material(&n) && !["scalp", "base_", "color_", "clap", "cap"].iter().any(|w| n.contains(w)) + }) + .max_by_key(|(m, c)| (*c, usize::MAX - *m)) + .map(|(m, _)| m) +} + +/// Area-weighted vertex normals. +fn smooth_normals(pos: &[Vec3], tris: &[[u32; 3]]) -> Vec { + let mut n = vec![Vec3::ZERO; pos.len()]; + for t in tris { + let [a, b, c] = t.map(|i| pos[i as usize]); + let f = (b - a).cross(c - a); + for &i in t { + n[i as usize] += f; + } + } + n.into_iter().map(|v| v.normalize_or(Vec3::Y)).collect() +} + +fn base_name(n: &str) -> &str { + n.split('.').next().unwrap_or(n) +} + +/// RLHair maps for strand meshes (UV = (ID, t)): root map = v, ID map = u, a +/// flat flow map along +v. Written once into `lib/atlas/strandmaps/`. +pub fn strand_maps(lib: &Path) -> Result { + let d = lib.join("atlas").join("strandmaps"); + std::fs::create_dir_all(&d)?; + let root = d.join("root.png"); + if !root.exists() { + image::RgbImage::from_fn(4, 256, |_, y| image::Rgb([y as u8; 3])).save(&root)?; + image::RgbImage::from_fn(256, 4, |x, _| image::Rgb([x as u8; 3])).save(d.join("id.png"))?; + image::RgbImage::from_fn(4, 4, |_, _| image::Rgb([128, 0, 128])).save(d.join("flow.png"))?; + } + Ok(json!({"Hair Root Map": "atlas/strandmaps/root.png", "Hair ID Map": "atlas/strandmaps/id.png", "Hair Flow Map": "atlas/strandmaps/flow.png"})) +} + +/// Linear RGB a white strand mask is tinted to without CC5's shader (mean of +/// root and tip colour), or None for coloured maps. +fn mask_tint(vars: &Map) -> Option<[f32; 3]> { + let on = vars.get("UseRootTipColor").map_or(false, |v| v.as_f64().map_or(v.as_bool().unwrap_or(false), |x| x != 0.0)); + let mode = vars.get("RootTipBlendMode").and_then(|v| v.as_f64()); + if !on || !matches!(mode, Some(m) if m == 1.0 || m == 2.0) { + return None; + } + let col = |k: &str| -> [f64; 3] { + let a = vars.get(k).and_then(|v| v.as_array()); + std::array::from_fn(|i| a.and_then(|a| a.get(i)).and_then(|x| x.as_f64()).unwrap_or(255.0)) + }; + let (rc, tc) = (col("RootColor"), col("TipColor")); + Some(std::array::from_fn(|i| (crate::srgb_to_linear((rc[i] / 255.0) as f32) + crate::srgb_to_linear((tc[i] / 255.0) as f32)) / 2.0)) +} + +/// RLHair variables of strand parts without CC5 settings (a library without +/// `atlas//atlas.json`): a mid brown with lighter tips. +pub fn default_vars() -> Value { + json!({ + "TangentVectorColor": [255.0, 0.0, 0.0], "TangentMapFlipGreen": 1.0, "Diffuse Strength": 1.0, + "Hair Roughness Map Strength": 0.5, "Hair Specular Map Strength": 0.3, "Specular Strength": 0.8, + "Secondary Specular Strength": 1.0, "Transmission Strength": 0.15, "VertexColorStrength": 0.875, + "ActiveChangeHairColor": 1.0, "BaseColorMapStrength": 1.0, "UseRootTipColor": 1.0, + "RootColor": [48.0, 28.0, 18.0], "TipColor": [120.0, 80.0, 52.0], "RootTipBlendMode": 2.0, + "RootColorStrength": 1.0, "TipColorStrength": 1.0, "InvertRootTip": 0.0, + "_1st Dye Color": [150.0, 105.0, 70.0], "_1st Dye Strength": 0.35, + "_1st Dye Distribution from Grayscale": [25.5, 76.5, 127.5], "_1st BlendMode": 0.0, + "Mask 1st Dye by RootMap": 1.0, "Invert 1st Dye RootMap Mask": 1.0, "_2nd Dye Strength": 0.0 + }) +} + +fn merge_vars(base: Option<&Value>, over: &Map) -> Value { + let mut v = base.and_then(|v| v.as_object()).cloned().unwrap_or_default(); + for (k, x) in over { + v.insert(k.clone(), x.clone()); + } + Value::Object(v) +} + +/// Build `spec` with `head` (the reference character: body, skeleton) into a +/// part for library `lib` (its `atlas//atlas.json`, if any, gives the +/// default RLHair variables; else [`default_vars`]). +pub fn build(spec: &Spec, head: &Source, body: &Body, lib: &Path) -> Result { + build_with(spec, head, body, lib, None) +} + +/// [`build`] with the group's source already loaded (`group`: the spec's +/// `source_glb`; its decoded images are reused across rebuilds). +pub fn build_with(spec: &Spec, head: &Source, body: &Body, lib: &Path, group: Option<&Source>) -> Result { + let base: Convert = Convert::for_source(&spec.convert, "")?; + // the source: another export of the character (a Smart Hair group), or the head itself + let owned: Option = match (&base.source_glb, group) { + (Some(sg), None) => Some(Source::load(sg)?), + _ => None, + }; + let src: &Source = match (&base.source_glb, group) { + (Some(_), Some(g)) => g, + (Some(_), None) => owned.as_ref().unwrap(), + (None, _) => head, + }; + let sources: Vec = if base.source_glb.is_some() { + let mut names: Vec = src + .objects + .iter() + .filter(|o| !o.name.starts_with("CC_Base")) + .filter(|o| o.prims.iter().any(|p| p.material.is_some_and(|m| is_hair_material(&src.materials[m].name)))) + .filter(|o| !base.exclude.iter().any(|x| x == base_name(&o.name))) + .filter(|o| base.include.as_ref().map_or(true, |inc| o.name.to_lowercase().contains(&inc.to_lowercase()))) + .map(|o| o.name.clone()) + .collect(); + names.sort(); + names + } else { + base.sources.clone() + }; + let from_group = base.source_glb.is_some(); + // the CC5 material settings: the source's own, or the character's (the head's + // own when it has them (an FBX head with its sidecar JSON), else + // `../character.glb`, else `.cc5mat.glb` beside it) + let head_has_cc5 = head.materials.iter().any(|m| m.cc5.is_some()); + let mat_glb: Option = base.material_glb.clone().or_else(|| base.source_glb.clone()).or_else(|| { + if head_has_cc5 { + return Some(head.path.clone()); + } + let char_glb = head.path.parent()?.join("..").join("character.glb"); + let cc5mat = head.path.with_extension("cc5mat.glb"); + [char_glb, cc5mat].into_iter().find(|p| p.exists()) + }); + let mat_src_owned; + let mat_src: Option<&Source> = match &mat_glb { + Some(p) if *p == src.path => Some(src), + Some(p) if *p == head.path => Some(head), + Some(p) => { + mat_src_owned = Source::load(p)?; + Some(&mat_src_owned) + } + None => None, + }; + let cc5: HashMap = mat_src + .map(|m| m.materials.iter().filter_map(|x| x.cc5.clone().map(|c| (base_name(&x.name).to_string(), c))).collect()) + .unwrap_or_default(); + let base_dir = mat_glb.as_ref().and_then(|p| p.parent().map(Path::to_path_buf)).unwrap_or_default(); + let atlas_json = lib.join("atlas").join(&spec.atlas).join("atlas.json"); + let atlas: Value = if atlas_json.exists() { + serde_json::from_str(&std::fs::read_to_string(&atlas_json)?).with_context(|| format!("{}", atlas_json.display()))? + } else { + json!({"vars": default_vars()}) + }; + let maps = strand_maps(lib)?; + let default_vars = merge_vars(atlas.get("vars"), &spec.vars); + let default_cc5 = json!({"shader": "RLHair", "srgb": [], "vars": default_vars, "maps": maps}); + let tint = default_vars.as_object().and_then(mask_tint); + let factor = |tint: Option<[f32; 3]>| tint.map_or([1.0; 4], |t| [t[0], t[1], t[2], 1.0]); + + let mut meshes = Vec::new(); + let mut materials: Vec = Vec::new(); + let mut files = Vec::new(); + let (mut cards, mut verts) = (0usize, 0usize); + let mut seed = spec.seed; + for name in &sources { + let c = Convert::for_source(&spec.convert, name)?; + let obj = src.object(name).ok_or_else(|| anyhow!("{}: no object {name}", src.path.display()))?; + let main = main_material(src, obj); + let sname = base_name(name); + seed = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15).wrapping_add(sname.bytes().fold(0u64, |h, b| h.wrapping_mul(131).wrapping_add(b as u64))); + if !c.keep_cards { + let Some(main) = main else { bail!("{name}: no hair material") }; + let mname = base_name(&src.materials[main].name).to_string(); + // the hair material's triangles (one or more primitives) + let (mut pos, mut uv, mut tris) = (Vec::new(), Vec::new(), Vec::new()); + let (mut spos, mut stris) = (Vec::new(), Vec::new()); + for p in &obj.prims { + if p.material == Some(main) { + let o = pos.len() as u32; + pos.extend(&p.pos); + uv.extend(&p.uv); + tris.extend(p.tris.iter().map(|t| t.map(|i| i + o))); + } else { + let o = spos.len() as u32; + spos.extend(&p.pos); + stris.extend(p.tris.iter().map(|t| t.map(|i| i + o))); + } + } + let img = match src.materials[main].base_image { + Some(i) => Some(Tex::new(src.image_rgba(i)?)), + None => None, + }; + let flow = cc5 + .get(&mname) + .and_then(|v| v.get("maps")?.get("Hair Flow Map")?.as_str().map(|f| base_dir.join(f))) + .filter(|p| p.exists()) + .map(|p| src.file_rgba(&p).map(Tex::new)) + .transpose()?; + let scalp = Surface::new(spos, stris); + let mut strands = cards_to_strands(&pos, &uv, &tris, img.as_ref(), flow.as_ref(), &c, body, seed); + shape::finish(&mut strands, &c.shape, body, scalp.as_ref(), seed); + if let Some([ao_min, gamma]) = c.depth_shade { + shade::depth_shade(&mut strands, body, ao_min as f32, gamma as f32); + } + let skin = strands.iter().map(|s| body.skin_at(s.pts[0])).collect(); + let mesh_name = format!("{}_{}", spec.id, sname); + let cc5_mat = match cc5.get(&mname) { + Some(v) => { + let mut v = v.clone(); + let vars = merge_vars(v.get("vars"), &spec.vars); + v["maps"] = maps.clone(); + v["srgb"] = json!([]); + v["vars"] = vars; + v + } + None => default_cc5.clone(), + }; + materials.push(PartMaterial { name: format!("{mesh_name}_Hair"), blend: false, cc5: Some(cc5_mat), base_factor: factor(tint), base_image: None }); + cards += strands.len(); + verts += strands.iter().map(|s| 2 * s.pts.len()).sum::(); + meshes.push(PartMesh { name: mesh_name, material: materials.len() - 1, geom: Geometry::Strands { strands, skin, taper: c.taper as f32, root_fade: c.root_fade as f32 } }); + } + // kept layers: every layer (keep_cards: brows), or the source's other layers (scalp, caps) + if c.keep_cards || c.source_scalp { + for (li, p) in obj.prims.iter().enumerate() { + let Some(m) = p.material else { continue }; + let mname = base_name(&src.materials[m].name).to_string(); + let is_main = Some(m) == main; + if !c.keep_cards && (is_main || (!from_group && !mname.to_lowercase().contains("scalp"))) { + continue; + } + let lname = format!("{}_{}_layer{}", spec.id, sname, li); + let skin = p.pos.iter().map(|q| body.skin_at(*q)).collect(); + let image = match src.materials[m].base_image { + Some(i) => Some(src.image_bytes(i).map(|(b, t)| (b.to_vec(), t))?), + None => None, + }; + let mat = if c.keep_cards && is_main && cc5.contains_key(&mname) { + // the hair cards keep CC5's RLHair shader and their own maps + let mut v = cc5[&mname].clone(); + let mut own = Map::new(); + if let Some(ms) = v.get("maps").and_then(|m| m.as_object()) { + for (slot, f) in ms { + let Some(f) = f.as_str() else { continue }; + let file = base_dir.join(f); + if file.exists() { + let rel = format!("cc5maps/{}/{}", spec.id, Path::new(f).file_name().unwrap().to_string_lossy()); + files.push((rel.clone(), file)); + own.insert(slot.clone(), Value::String(rel)); + } + } + } + v["maps"] = Value::Object(own); + v["vars"] = merge_vars(v.get("vars"), &spec.vars); + PartMaterial { name: format!("{lname}_Hair"), blend: false, cc5: Some(v), base_factor: factor(tint), base_image: image } + } else { + PartMaterial { name: format!("{lname}_Scalp"), blend: true, cc5: None, base_factor: src.materials[m].base_factor, base_image: image } + }; + materials.push(mat); + if c.keep_cards { + cards += p.tris.len(); // (strand parts count strands only, as hair_build.py) + } + verts += p.pos.len(); + meshes.push(PartMesh { + name: lname, + material: materials.len() - 1, + // smooth normals from the cards' own faces, as hair_build.py's rebuilt + // layers had (the hair shader's diffuse reads them) + geom: Geometry::Cards { pos: p.pos.clone(), nrm: smooth_normals(&p.pos, &p.tris), uv: p.uv.clone(), tris: p.tris.clone(), skin }, + }); + } + } + } + let mut entry = json!({"id": spec.id, "slot": spec.slot, "kind": spec.kind, "file": format!("{}.glb", spec.id), + "atlas": spec.atlas, "cards": cards, "verts": verts}); + if !base.follow.is_empty() || base.shape_dir.is_some() { + let mut modes = HashMap::new(); + if let Some(d) = &base.shape_dir { + modes.extend(follow::layer_modes(d, 0.05)?); + modes.extend(follow::bone_modes(d, 0.1)?); + } + modes.extend(base.follow.iter().map(|(k, v)| (follow::key(k), v.clone()))); + let names: Vec = meshes.iter().map(|m| m.name.clone()).collect(); + let f = follow::part_follow(&names, &spec.id, &modes); + if !f.is_empty() { + entry["follow"] = serde_json::to_value(f)?; + } + } + if base.keep_cards && spec.kind == "brow" { + entry["painted_layers"] = json!(true); + } + Ok(BuiltPart { id: spec.id.clone(), meshes, materials, entry, files }) +} diff --git a/crates/cc5-hair/src/regions.rs b/crates/cc5-hair/src/regions.rs new file mode 100644 index 0000000..52c5af5 --- /dev/null +++ b/crates/cc5-hair/src/regions.rs @@ -0,0 +1,113 @@ +//! Anchor regions for hair conforming, as CC_Base_Body vertex indices (the +//! library's `regions.json`; was hair_regions.py). +//! +//! CC characters share the CC_Base_Body topology, so the regions, picked once on +//! the reference character in bind pose, hold for every character shaped from +//! the same pack. Positions in Blender's frame (metres, Z up, face to -Y): +//! scalp the hairline region widened by 10 degrees, without the ears; +//! face the head below the hairline, ears and front of the neck: anchors +//! for beards, moustaches and brows; +//! head_box the head without ears and nose: its bounding box is what CC5 +//! fits rigid hair layers to (conform "head_box"); +//! head the whole head: scalp hair's conform anchors. +//! Indices are mesh-local (control-point order: the pack's CC_Base_Body slice). + +use std::path::Path; + +use anyhow::{anyhow, Context, Result}; +use glam::DVec3; +use serde_json::{json, Value}; + +const CENTER: DVec3 = DVec3::new(0.0, -0.005, 1.64); +/// (azimuth from the front, elevation) of the hairline, degrees. +const HAIRLINE: [(f64, f64); 8] = [(0.0, 33.0), (30.0, 30.0), (55.0, 20.0), (75.0, 9.0), (100.0, 6.0), (125.0, -8.0), (150.0, -30.0), (180.0, -42.0)]; + +fn hairline(az: f64) -> f64 { + if az <= HAIRLINE[0].0 { + return HAIRLINE[0].1; + } + for w in HAIRLINE.windows(2) { + let ((a0, e0), (a1, e1)) = (w[0], w[1]); + if az <= a1 { + return e0 + (e1 - e0) * (az - a0) / (a1 - a0); + } + } + HAIRLINE[HAIRLINE.len() - 1].1 +} + +pub struct Regions { + pub scalp: Vec, + pub face: Vec, + pub head_box: Vec, + pub head: Vec, +} + +/// The regions of CC_Base_Body points `p` (Blender frame). +pub fn regions(p: &[DVec3]) -> Regions { + let margin = 10.0; + let mut r = Regions { scalp: vec![], face: vec![], head_box: vec![], head: vec![] }; + for (i, &q) in p.iter().enumerate() { + let d = q - CENTER; + let dist = d.length(); + let el = (d.z / dist).clamp(-1.0, 1.0).asin().to_degrees(); + let az = d.x.atan2(-d.y).abs().to_degrees(); + let thr = hairline(az); + let ear = q.x.abs() > 0.071 && q.z < 1.675 && q.y.abs() < 0.03; + let i = i as u32; + if el > thr - margin && dist < 0.16 && !ear { + r.scalp.push(i); + } + if dist < 0.17 && q.z > 1.47 && el <= thr + 2.0 && !(q.y > 0.03 && q.z < 1.60) { + r.face.push(i); + } + let nose = q.y < -0.09 && q.z < 1.64; + let ears = q.x.abs() > 0.065 && q.z < 1.68; + if dist < 0.11 && !nose && !ears { + r.head_box.push(i); + } + if dist < 0.17 { + r.head.push(i); + } + } + r +} + +impl Regions { + pub fn to_json(&self) -> Value { + json!({"mesh": "CC_Base_Body", "scalp": self.scalp, "face": self.face, "head_box": self.head_box, "head": self.head}) + } +} + +/// CC_Base_Body's bind positions (Blender frame) from the reference character: +/// a CC5 FBX export, or its `cc5-fbx --dump` JSON (with `.verts.f32`). +pub fn body_points(path: &Path) -> Result> { + if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("fbx")) { + let c = cc5_fbx::Character::load(path)?; + let m = c.mesh("CC_Base_Body").ok_or_else(|| anyhow!("{}: no CC_Base_Body", path.display()))?; + // glTF (Y up) -> Blender (Z up) + return Ok(m.points.iter().map(|p| DVec3::new(p.x as f64, -p.z as f64, p.y as f64)).collect()); + } + let d: Value = serde_json::from_slice(&std::fs::read(path).with_context(|| format!("reading {}", path.display()))?)?; + let v = std::fs::read(format!("{}.verts.f32", path.display()))?; + let f: Vec = v.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + let mut off = 0; + for n in d["mesh_order"].as_array().ok_or_else(|| anyhow!("{}: no mesh_order", path.display()))? { + let n = n.as_str().unwrap_or(""); + let count = d["meshes"][n].as_u64().unwrap_or(0) as usize; + if n == "CC_Base_Body" { + return Ok(f[3 * off..3 * (off + count)].chunks_exact(3).map(|c| DVec3::new(c[0] as f64, c[1] as f64, c[2] as f64)).collect()); + } + off += count; + } + Err(anyhow!("{}: no CC_Base_Body", path.display())) +} + +#[cfg(test)] +mod tests { + #[test] + fn hairline_interpolates() { + assert_eq!(super::hairline(0.0), 33.0); + assert_eq!(super::hairline(15.0), 31.5); + assert_eq!(super::hairline(180.0), -42.0); + } +} diff --git a/crates/cc5-hair/src/settings.rs b/crates/cc5-hair/src/settings.rs new file mode 100644 index 0000000..6121a97 --- /dev/null +++ b/crates/cc5-hair/src/settings.rs @@ -0,0 +1,163 @@ +//! Part spec and conversion settings (the JSON keys of hair_build.py's specs: +//! `specs/cc5_converted.json`, convert_smart_hair.py), with the same defaults. + +use std::collections::HashMap; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// A library part to build. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Spec { + pub id: String, + pub slot: String, + #[serde(default = "scalp")] + pub kind: String, + #[serde(default = "strands")] + pub atlas: String, + #[serde(default = "one")] + pub seed: u64, + /// RLHair variables over the source material's (root/tip colour, dyes, ...). + #[serde(default)] + pub vars: serde_json::Map, + /// Conversion settings, as JSON (merged with `per_source` per source layer). + pub convert: Value, +} + +fn scalp() -> String { + "scalp".into() +} +fn strands() -> String { + "strands".into() +} +fn one() -> u64 { + 1 +} + +/// Conversion settings for one source layer (`convert` + its `per_source`). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct Convert { + /// Hair meshes from another export of the character (a Smart Hair group). + pub source_glb: Option, + /// GLB with the CC5 material settings (default: the source GLB). + pub material_glb: Option, + /// Source objects (without `source_glb`: objects of the head GLB). + pub sources: Vec, + /// Case-insensitive substring an object name must contain. + pub include: Option, + pub exclude: Vec, + pub per_source: HashMap, + /// Points per strand. + pub points: usize, + pub strands_per_cm2: f64, + /// Spread of strands off the card plane (m). + pub thickness: f64, + /// Strand ribbon width range (m). + pub width: [f64; 2], + pub taper: f64, + /// Per-strand brightness range. + pub shade: [f64; 2], + /// A texture column becomes a strand when its alpha (max or mean) clears a + /// random threshold in this range. + pub accept: [f64; 2], + pub accept_on: String, + pub opacity_power: f64, + pub clumps_per_cm: f64, + pub clump_spread: f64, + pub clump_shade: [f64; 2], + /// [darkest factor, curve] for hair deep in the volume. + pub depth_shade: Option<[f64; 2]>, + pub root_fade: f64, + pub start_at_hair: bool, + pub min_tries: usize, + /// Cards shorter than this × their width become short patch strands. + pub patch_aspect: f64, + pub patch_density: f64, + pub patch_length: f64, + /// Keep CC5's cards and textures (brows), every material layer. + pub keep_cards: bool, + /// Keep the source's other material layers (scalp, caps) as CC5 made them. + pub source_scalp: bool, + /// {source layer: "head_box" | "bone:..."} over what `shape_dir` detects. + pub follow: HashMap, + /// The group's CC5 shape OBJs / bone dumps (follow detection). + pub shape_dir: Option, + /// Shape edits (applied after conversion; all off by default). + pub shape: ShapeEdits, +} + +impl Default for Convert { + fn default() -> Self { + Self { + source_glb: None, + material_glb: None, + sources: Vec::new(), + include: None, + exclude: Vec::new(), + per_source: HashMap::new(), + points: 14, + strands_per_cm2: 9.0, + thickness: 0.003, + width: [0.00008, 0.00013], + taper: 0.6, + shade: [0.82, 1.08], + accept: [0.05, 0.6], + accept_on: "mean".into(), + opacity_power: 0.5, + clumps_per_cm: 0.0, + clump_spread: 0.22, + clump_shade: [0.75, 1.15], + depth_shade: None, + root_fade: 0.0, + start_at_hair: false, + min_tries: 2, + patch_aspect: 2.5, + patch_density: 40.0, + patch_length: 0.018, + keep_cards: false, + source_scalp: true, + follow: HashMap::new(), + shape_dir: None, + shape: ShapeEdits::default(), + } + } +} + +/// Edits on the converted strands (root -> tip, t in 0..1). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ShapeEdits { + /// Arc length scale (1 = as converted; < 1 trims the tips, > 1 extends them). + pub length: f64, + /// Comb direction (world, any length) and strength (0..1, grows towards the tip). + pub comb: [f64; 3], + pub comb_strength: f64, + /// Droop along world down (m at the tip of a 10 cm strand). + pub gravity: f64, + /// Helix radius (m) and period (m along the strand). + pub curl_radius: f64, + pub curl_period: f64, + /// Random offsets growing towards the tip (m). + pub frizz: f64, +} + +impl Convert { + /// The settings for one source layer: `base` with that layer's + /// `per_source` entry merged over it. + pub fn for_source(base: &Value, source: &str) -> anyhow::Result { + let mut v = base.clone(); + if let (Some(obj), Some(over)) = (v.as_object_mut(), base.get("per_source").and_then(|p| p.get(source)).and_then(|o| o.as_object())) { + for (k, x) in over { + obj.insert(k.clone(), x.clone()); + } + } + // ShapeEdits defaults are 0; length means "as converted" at 1 + let mut c: Self = serde_json::from_value(v)?; + if c.shape.length == 0.0 { + c.shape.length = 1.0; + } + Ok(c) + } +} diff --git a/crates/cc5-hair/src/shade.rs b/crates/cc5-hair/src/shade.rs new file mode 100644 index 0000000..fbb2bf2 --- /dev/null +++ b/crates/cc5-hair/src/shade.rs @@ -0,0 +1,46 @@ +//! Darken strand points by their depth in the hair (hair_build.py `depth_shade`): +//! distance from the head relative to the outermost hair over the same patch +//! of scalp (cells of `cell` m by the nearest head point). Hair underneath and +//! at the roots reads darker, the outer surface keeps its colour: the depth CC5 +//! paints into its cards. Thin hair (hairlines, partings: outermost hair under +//! `thick_ref`) is darkened in proportion, so hairlines stay light. + +use std::collections::HashMap; + +use glam::IVec3; +use rayon::prelude::*; + +use crate::head::Body; +use crate::Strand; + +pub fn depth_shade(strands: &mut [Strand], body: &Body, ao_min: f32, gamma: f32) { + const CELL: f32 = 0.015; + const THICK_REF: f32 = 0.012; + let per: Vec> = strands + .par_iter() + .map(|s| { + s.pts + .iter() + .map(|p| { + let h = body.nearest(*p); + (h.dist, (h.point / CELL).floor().as_ivec3()) + }) + .collect() + }) + .collect(); + let mut outer: HashMap = HashMap::new(); + for pts in &per { + for &(d, k) in pts { + let e = outer.entry(k).or_insert(0.0); + *e = e.max(d); + } + } + strands.par_iter_mut().zip(&per).for_each(|(s, pts)| { + for (c, &(d, k)) in s.rgb.iter_mut().zip(pts) { + let big_d = outer[&k].max(1e-4); + let ao = ao_min + (1.0 - ao_min) * (d / big_d).clamp(0.0, 1.0).powf(gamma); + let ao = 1.0 - (1.0 - ao) * (big_d / THICK_REF).clamp(0.0, 1.0); + *c *= ao; + } + }); +} diff --git a/crates/cc5-hair/src/shape.rs b/crates/cc5-hair/src/shape.rs new file mode 100644 index 0000000..b99afa3 --- /dev/null +++ b/crates/cc5-hair/src/shape.rs @@ -0,0 +1,173 @@ +//! Shape edits on converted strands (length, comb, gravity, curl, frizz), then +//! the push off the head and the source's scalp layers (hair_build.py's +//! per-point fix in `cards_to_strands`). + +use glam::Vec3; +use rayon::prelude::*; + +use crate::head::{Body, Surface}; +use crate::settings::ShapeEdits; +use crate::{uniform_polyline, Rng, Strand, HEAD_CENTER}; + +/// Apply `edits` (no-ops at their defaults), then keep every point outside the +/// body (0.8 mm) and above the scalp layers (0.6 mm). +pub fn finish(strands: &mut [Strand], edits: &ShapeEdits, body: &Body, scalp: Option<&Surface>, seed: u64) { + let active = *edits != ShapeEdits { length: 1.0, ..Default::default() }; + strands.par_iter_mut().enumerate().for_each(|(i, s)| { + if active { + let mut rng = Rng::new(seed ^ (i as u64).wrapping_mul(0xd1b5_4a32_d192_ed03)); + edit(s, edits, &mut rng); + } + for q in s.pts.iter_mut() { + *q = push(*q, body, scalp); + } + }); +} + +fn push(mut q: Vec3, body: &Body, scalp: Option<&Surface>) -> Vec3 { + let h = body.nearest(q); + if (q - h.point).dot(h.normal) < 0.0 || h.dist < 0.0008 { + q = h.point + h.normal * 0.0008; + } + if let Some(s) = scalp { + let h = s.nearest(q); + if h.dist < 0.004 { + let n = if h.normal.dot(h.point - HEAD_CENTER) < 0.0 { -h.normal } else { h.normal }; + if (q - h.point).dot(n) < 0.0006 { + q = h.point + n * 0.0006; + } + } + } + q +} + +fn edit(s: &mut Strand, e: &ShapeEdits, rng: &mut Rng) { + let n = s.pts.len(); + if n < 2 { + return; + } + let len0: f32 = s.pts.windows(2).map(|w| w[0].distance(w[1])).sum(); + // length: trim the tip, or extend along the last segment + if (e.length - 1.0).abs() > 1e-6 && len0 > 0.0 { + let target = len0 * e.length.max(0.05) as f32; + let mut pts = s.pts.clone(); + if target > len0 { + let d = (pts[n - 1] - pts[n - 2]).normalize_or_zero(); + pts.push(pts[n - 1] + d * (target - len0)); + } else { + pts = cut(&pts, target); + } + s.pts = uniform_polyline(&pts, n); + } + let len: f32 = s.pts.windows(2).map(|w| w[0].distance(w[1])).sum(); + let t = |i: usize| i as f32 / (n - 1) as f32; + // comb: segment directions blend towards the comb direction, more at the tip + let comb = Vec3::new(e.comb[0] as f32, e.comb[1] as f32, e.comb[2] as f32).normalize_or_zero(); + if e.comb_strength > 0.0 && comb != Vec3::ZERO { + let mut out = vec![s.pts[0]]; + for i in 1..n { + let seg = s.pts[i] - s.pts[i - 1]; + let w = (e.comb_strength as f32 * t(i).powf(1.5)).clamp(0.0, 1.0); + let d = seg.normalize_or_zero().lerp(comb, w).normalize_or_zero(); + out.push(out[i - 1] + d * seg.length()); + } + s.pts = out; + } + // gravity: droop along world down (glTF: -Y), scaled by the strand's length + if e.gravity != 0.0 { + let k = e.gravity as f32 * (len / 0.1); + for (i, p) in s.pts.iter_mut().enumerate() { + p.y -= k * t(i) * t(i); + } + } + // curl: a helix around the strand, fading in from the root + if e.curl_radius > 0.0 && e.curl_period > 0.0 { + let base = s.pts.clone(); + let phase = rng.uniform(0.0, std::f64::consts::TAU) as f32; + let mut arc = 0.0f32; + for i in 0..n { + if i > 0 { + arc += base[i].distance(base[i - 1]); + } + let tan = (base[(i + 1).min(n - 1)] - base[i.saturating_sub(1)]).normalize_or_zero(); + let side = tan.cross(if tan.y.abs() < 0.9 { Vec3::Y } else { Vec3::X }).normalize_or_zero(); + let up = tan.cross(side); + let th = phase + std::f32::consts::TAU * arc / e.curl_period as f32; + let fade = smoothstep(t(i) / 0.2); + s.pts[i] = base[i] + (side * th.cos() + up * th.sin()) * e.curl_radius as f32 * fade; + } + } + // frizz: seeded offsets growing towards the tip + if e.frizz > 0.0 { + for (i, p) in s.pts.iter_mut().enumerate() { + let g = Vec3::from(std::array::from_fn(|_| rng.gauss(0.0, 1.0) as f32)); + *p += g * e.frizz as f32 * t(i) * t(i); + } + } + s.pts = uniform_polyline(&s.pts, n); +} + +/// The polyline up to arc length `target`. +fn cut(pts: &[Vec3], target: f32) -> Vec { + let mut out = vec![pts[0]]; + let mut acc = 0.0; + for w in pts.windows(2) { + let l = w[0].distance(w[1]); + if acc + l >= target { + let f = if l > 0.0 { (target - acc) / l } else { 0.0 }; + out.push(w[0].lerp(w[1], f)); + return out; + } + acc += l; + out.push(w[1]); + } + out +} + +fn smoothstep(x: f32) -> f32 { + let x = x.clamp(0.0, 1.0); + x * x * (3.0 - 2.0 * x) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn strand() -> Strand { + let pts: Vec = (0..11).map(|i| Vec3::new(0.0, 0.0, i as f32 * 0.01)).collect(); + Strand { rgb: vec![Vec3::ONE; pts.len()], pts, id: 0.5, width: 1e-4, shade: 1.0 } + } + + fn length(s: &Strand) -> f32 { + s.pts.windows(2).map(|w| w[0].distance(w[1])).sum() + } + + #[test] + fn length_trims_and_extends() { + let mut rng = Rng::new(1); + let mut s = strand(); + edit(&mut s, &ShapeEdits { length: 0.5, ..Default::default() }, &mut rng); + assert!((length(&s) - 0.05).abs() < 1e-4, "{}", length(&s)); + let mut s = strand(); + edit(&mut s, &ShapeEdits { length: 1.5, ..Default::default() }, &mut rng); + assert!((length(&s) - 0.15).abs() < 1e-4, "{}", length(&s)); + assert_eq!(s.pts.len(), 11); + } + + #[test] + fn gravity_pulls_tips_down() { + let mut s = strand(); + edit(&mut s, &ShapeEdits { length: 1.0, gravity: 0.02, ..Default::default() }, &mut Rng::new(1)); + assert!(s.pts[0].y.abs() < 1e-6); + assert!(s.pts[10].y < -0.015, "{:?}", s.pts[10]); + } + + #[test] + fn curl_keeps_its_radius() { + let mut s = strand(); + edit(&mut s, &ShapeEdits { length: 1.0, curl_radius: 0.003, curl_period: 0.02, ..Default::default() }, &mut Rng::new(1)); + // past the fade-in, points sit ~3 mm off the original axis + let r = (s.pts[8].x * s.pts[8].x + s.pts[8].y * s.pts[8].y).sqrt(); + assert!((r - 0.003).abs() < 0.0008, "{r}"); + } +} diff --git a/crates/cc5-hair/src/source.rs b/crates/cc5-hair/src/source.rs new file mode 100644 index 0000000..9fb8ce0 --- /dev/null +++ b/crates/cc5-hair/src/source.rs @@ -0,0 +1,168 @@ +//! A character GLB (or CC5 FBX export) as the converter reads it: mesh objects (one per mesh node, +//! primitives per material), materials with their CC5 settings +//! (`extras.cc5`, glb_cc5_materials.py) and embedded images, and the skin. +//! +//! Positions are the accessors' (metres, Y up: the CC5 GLBs keep their meshes in +//! world space and scale only the armature). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use anyhow::{anyhow, bail, Context, Result}; +use glam::{Mat4, Vec2, Vec3}; +use serde_json::Value; + +pub struct Source { + pub path: PathBuf, + pub doc: gltf::Document, + pub blob: Vec, + pub objects: Vec, + pub materials: Vec, + /// Decoded images (embedded by index, files by path), kept for rebuilds. + images: Mutex>>, +} + +/// A mesh node (Blender's "object"). +pub struct Object { + pub name: String, + pub prims: Vec, +} + +pub struct Prim { + pub material: Option, + pub pos: Vec, + pub nrm: Vec, + pub uv: Vec, + pub tris: Vec<[u32; 3]>, + /// Indices into the skin's joint list. + pub joints: Vec<[u16; 4]>, + pub weights: Vec<[f32; 4]>, +} + +pub struct Material { + pub name: String, + /// `extras.cc5`: {shader, maps, srgb, vars}. + pub cc5: Option, + pub base_image: Option, + pub base_factor: [f32; 4], +} + +impl Source { + /// A GLB, or a CC5 FBX export (converted in memory by cc5-fbx; CC5's + /// material settings from the sidecar `.json`, maps written to + /// `.cc5maps/`). + pub fn load(path: &Path) -> Result { + let is_fbx = path.extension().is_some_and(|e| e.eq_ignore_ascii_case("fbx")); + let bytes = if is_fbx { + let json = path.with_extension("json"); + let opts = cc5_fbx::GlbOptions { cc5_json: json.exists().then_some(json), maps_dir: Some(path.with_extension("cc5maps")) }; + cc5_fbx::Character::load(path)?.to_glb(&opts)? + } else { + std::fs::read(path).with_context(|| format!("reading {}", path.display()))? + }; + Self::from_bytes(path, bytes) + } + + /// A GLB already in memory (`path`: where it came from; relative map paths + /// in `extras.cc5` resolve against its folder). + pub fn from_bytes(path: &Path, bytes: Vec) -> Result { + let g = gltf::Gltf::from_slice(&bytes).with_context(|| format!("parsing {}", path.display()))?; + let blob = g.blob.clone().ok_or_else(|| anyhow!("{}: no binary chunk", path.display()))?; + let doc = g.document; + let materials = doc + .materials() + .map(|m| Material { + name: m.name().unwrap_or("").to_string(), + cc5: m.extras().as_ref().and_then(|r| serde_json::from_str::(r.get()).ok()).and_then(|v| v.get("cc5").cloned()), + base_image: m.pbr_metallic_roughness().base_color_texture().map(|t| t.texture().source().index()), + base_factor: m.pbr_metallic_roughness().base_color_factor(), + }) + .collect(); + let mut objects = Vec::new(); + for node in doc.nodes() { + let Some(mesh) = node.mesh() else { continue }; + let name = node.name().or(mesh.name()).unwrap_or("").to_string(); + let mut prims = Vec::new(); + for p in mesh.primitives() { + if p.mode() != gltf::mesh::Mode::Triangles { + continue; + } + let r = p.reader(|b| (b.index() == 0).then_some(blob.as_slice())); + let pos: Vec = r.read_positions().ok_or_else(|| anyhow!("{name}: no positions"))?.map(Vec3::from).collect(); + let n = pos.len(); + let nrm = r.read_normals().map(|i| i.map(Vec3::from).collect()).unwrap_or_else(|| vec![Vec3::Y; n]); + let uv = r.read_tex_coords(0).map(|i| i.into_f32().map(Vec2::from).collect()).unwrap_or_else(|| vec![Vec2::ZERO; n]); + let idx: Vec = r.read_indices().map(|i| i.into_u32().collect()).unwrap_or_else(|| (0..n as u32).collect()); + let tris = idx.chunks_exact(3).map(|t| [t[0], t[1], t[2]]).collect(); + let joints = r.read_joints(0).map(|i| i.into_u16().collect()).unwrap_or_else(|| vec![[0; 4]; n]); + let weights = r.read_weights(0).map(|i| i.into_f32().collect()).unwrap_or_else(|| vec![[1.0, 0.0, 0.0, 0.0]; n]); + prims.push(Prim { material: p.material().index(), pos, nrm, uv, tris, joints, weights }); + } + objects.push(Object { name, prims }); + } + Ok(Self { path: path.to_path_buf(), doc, blob, objects, materials, images: Mutex::new(HashMap::new()) }) + } + + pub fn object(&self, name: &str) -> Option<&Object> { + self.objects.iter().find(|o| o.name == name) + } + + /// An embedded image's encoded bytes and MIME type. + pub fn image_bytes(&self, index: usize) -> Result<(&[u8], String)> { + let img = self.doc.images().nth(index).ok_or_else(|| anyhow!("no image {index}"))?; + match img.source() { + gltf::image::Source::View { view, mime_type } => { + let o = view.offset(); + Ok((&self.blob[o..o + view.length()], mime_type.to_string())) + } + gltf::image::Source::Uri { uri, .. } => bail!("image {index}: external {uri} (expected embedded)"), + } + } + + /// An embedded image, decoded (cached). + pub fn image_rgba(&self, index: usize) -> Result> { + let k = format!("#{index}"); + if let Some(i) = self.images.lock().unwrap().get(&k) { + return Ok(i.clone()); + } + let (b, _) = self.image_bytes(index)?; + let img = Arc::new(image::load_from_memory(b)?.to_rgba8()); + self.images.lock().unwrap().insert(k, img.clone()); + Ok(img) + } + + /// An image file (e.g. a CC5 flow map next to the GLB), decoded (cached). + pub fn file_rgba(&self, path: &Path) -> Result> { + let k = path.to_string_lossy().to_string(); + if let Some(i) = self.images.lock().unwrap().get(&k) { + return Ok(i.clone()); + } + let img = Arc::new(image::open(path).with_context(|| format!("reading {}", path.display()))?.to_rgba8()); + self.images.lock().unwrap().insert(k, img.clone()); + Ok(img) + } + + /// Joint node names of the first skin, in skin order. + pub fn skin_joints(&self) -> Vec { + self.doc.skins().next().map(|s| s.joints().map(|j| j.name().unwrap_or("").to_string()).collect()).unwrap_or_default() + } + + /// World matrix of every node (parents first). + pub fn world_matrices(&self) -> Vec { + let mut out = vec![Mat4::IDENTITY; self.doc.nodes().len()]; + fn walk(n: gltf::Node, parent: Mat4, out: &mut [Mat4]) { + let m = parent * Mat4::from_cols_array_2d(&n.transform().matrix()); + out[n.index()] = m; + for c in n.children() { + walk(c, m, out); + } + } + for scene in self.doc.scenes() { + for n in scene.nodes() { + walk(n, Mat4::IDENTITY, &mut out); + } + } + out + } +} diff --git a/crates/cc5-hair/tests/roundtrip.rs b/crates/cc5-hair/tests/roundtrip.rs new file mode 100644 index 0000000..6c20897 --- /dev/null +++ b/crates/cc5-hair/tests/roundtrip.rs @@ -0,0 +1,146 @@ +//! A synthetic character (two joints, a skinned CC_Base_Body quad) through the +//! body's skin lookup and the part writer, read back with the `gltf` crate. + +use cc5_hair::glb_out::{self, Output, HALF_WIDTH_MAX}; +use cc5_hair::head::Body; +use cc5_hair::part::{BuiltPart, Geometry, PartMaterial, PartMesh}; +use cc5_hair::source::Source; +use cc5_hair::Strand; +use glam::Vec3; +use serde_json::json; + +fn f32s(v: &[f32]) -> Vec { + v.iter().flat_map(|x| x.to_le_bytes()).collect() +} + +/// A GLB: joints "Root" and "Head" (skin), and a 1 m quad named CC_Base_Body in +/// the y = 1.6 plane whose left edge is skinned to Root, right edge to Head. +fn head_glb() -> Vec { + let pos = [-0.5f32, 1.1, 0.0, 0.5, 1.1, 0.0, 0.5, 2.1, 0.0, -0.5, 2.1, 0.0]; + let joints: [u8; 16] = [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]; + let weights = [1.0f32, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]; + let idx: [u16; 6] = [0, 1, 2, 0, 2, 3]; + let ibm: Vec = [glam::Mat4::IDENTITY, glam::Mat4::from_translation(Vec3::new(0.0, -1.6, 0.0))].iter().flat_map(|m| m.to_cols_array()).collect(); + let mut bin = Vec::new(); + let mut views = Vec::new(); + for part in [f32s(&pos), joints.to_vec(), f32s(&weights), idx.iter().flat_map(|i| i.to_le_bytes()).collect::>(), f32s(&ibm)] { + while bin.len() % 4 != 0 { + bin.push(0); + } + views.push(json!({"buffer": 0, "byteOffset": bin.len(), "byteLength": part.len()})); + bin.extend(part); + } + while bin.len() % 4 != 0 { + bin.push(0); + } + let doc = json!({ + "asset": {"version": "2.0"}, "scene": 0, "scenes": [{"nodes": [3]}], + "nodes": [{"name": "Root", "children": [1]}, {"name": "Head", "translation": [0.0, 1.6, 0.0]}, + {"name": "CC_Base_Body", "mesh": 0, "skin": 0}, {"name": "Armature", "children": [0, 2]}], + "meshes": [{"name": "CC_Base_Body", "primitives": [{"attributes": {"POSITION": 0, "JOINTS_0": 1, "WEIGHTS_0": 2}, "indices": 3}]}], + "skins": [{"joints": [0, 1], "inverseBindMatrices": 4}], + "accessors": [ + {"bufferView": 0, "componentType": 5126, "count": 4, "type": "VEC3", "min": [-0.5, 1.1, 0.0], "max": [0.5, 2.1, 0.0]}, + {"bufferView": 1, "componentType": 5121, "count": 4, "type": "VEC4"}, + {"bufferView": 2, "componentType": 5126, "count": 4, "type": "VEC4"}, + {"bufferView": 3, "componentType": 5123, "count": 6, "type": "SCALAR"}, + {"bufferView": 4, "componentType": 5126, "count": 2, "type": "MAT4"}], + "bufferViews": views, "buffers": [{"byteLength": bin.len()}] + }); + let mut js = serde_json::to_vec(&doc).unwrap(); + while js.len() % 4 != 0 { + js.push(b' '); + } + let mut out = Vec::new(); + out.extend_from_slice(b"glTF"); + out.extend_from_slice(&2u32.to_le_bytes()); + out.extend_from_slice(&((12 + 8 + js.len() + 8 + bin.len()) as u32).to_le_bytes()); + out.extend_from_slice(&(js.len() as u32).to_le_bytes()); + out.extend_from_slice(b"JSON"); + out.extend_from_slice(&js); + out.extend_from_slice(&(bin.len() as u32).to_le_bytes()); + out.extend_from_slice(b"BIN\0"); + out.extend_from_slice(&bin); + out +} + +#[test] +fn skin_at_weights_sum_to_one() { + let head = Source::from_bytes(std::path::Path::new("head.glb"), head_glb()).unwrap(); + let body = Body::from_source(&head).unwrap(); + for p in [Vec3::new(-0.4, 1.6, 0.1), Vec3::new(0.0, 1.6, -0.2), Vec3::new(0.3, 2.0, 0.05), Vec3::new(2.0, 3.0, 1.0)] { + let s = body.skin_at(p); + let sum: f32 = s.iter().map(|e| e.1).sum(); + assert!((sum - 1.0).abs() < 1e-5, "{p}: {s:?}"); + assert!(s.iter().all(|e| e.1 >= 0.0)); + } + // the left edge rides Root alone, the middle mixes both joints + assert_eq!(body.skin_at(Vec3::new(-0.5, 1.6, 0.0))[0], (0, 1.0)); + let mid = body.skin_at(Vec3::new(0.0, 1.6, 0.0)); + assert!(mid[0].1 < 0.99 && mid[1].1 > 0.01, "{mid:?}"); +} + +fn part(head: &Source, body: &Body) -> BuiltPart { + let strands: Vec = (0..3) + .map(|k| { + let x = -0.2 + 0.2 * k as f32; + let pts: Vec = (0..5).map(|i| Vec3::new(x, 2.0 - 0.1 * i as f32, 0.05)).collect(); + Strand { rgb: vec![Vec3::new(0.3, 0.2, 0.1); pts.len()], pts, id: 0.25 * k as f32, width: 1e-4, shade: 1.0 } + }) + .collect(); + let skin = strands.iter().map(|s| body.skin_at(s.pts[0])).collect(); + let _ = head; + BuiltPart { + id: "test".into(), + meshes: vec![PartMesh { name: "test_strands".into(), material: 0, geom: Geometry::Strands { strands, skin, taper: 0.6, root_fade: 0.1 } }], + materials: vec![PartMaterial { name: "test_Hair".into(), blend: false, cc5: Some(json!({"shader": "RLHair", "maps": {}, "srgb": [], "vars": {}})), base_factor: [1.0; 4], base_image: None }], + entry: json!({"id": "test"}), + files: vec![], + } +} + +#[test] +fn part_glb_round_trips_through_gltf() { + let head = Source::from_bytes(std::path::Path::new("head.glb"), head_glb()).unwrap(); + let body = Body::from_source(&head).unwrap(); + let built = part(&head, &body); + let dir = std::env::temp_dir().join(format!("cc5_hair_roundtrip_{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for triangles in [false, true] { + let path = dir.join(format!("test_{triangles}.glb")); + glb_out::write(&built, &head, &path, Output { triangles }).unwrap(); + let g = gltf::Gltf::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let blob = g.blob.clone().unwrap(); + let doc = g.document; + // the head's skeleton and skin came along, joints by name + let skin = doc.skins().next().unwrap(); + let names: Vec<_> = skin.joints().map(|j| j.name().unwrap_or("").to_string()).collect(); + assert_eq!(names, ["Root", "Head"]); + let mesh = doc.meshes().find(|m| m.name() == Some("test_strands")).unwrap(); + let prim = mesh.primitives().next().unwrap(); + let r = prim.reader(|b| (b.index() == 0).then_some(blob.as_slice())); + let pos: Vec<[f32; 3]> = r.read_positions().unwrap().collect(); + let extras: serde_json::Value = serde_json::from_str(prim.material().extras().as_ref().unwrap().get()).unwrap(); + if triangles { + assert_eq!(prim.mode(), gltf::mesh::Mode::Triangles); + assert_eq!(pos.len(), 3 * 5 * 2); // two rows per strand + assert!(extras["cc5"].get("ribbons").is_none()); + } else { + assert_eq!(prim.mode(), gltf::mesh::Mode::Lines); + assert_eq!(pos.len(), 3 * 5); // one vertex per point + let segs: Vec = r.read_indices().unwrap().into_u32().collect(); + assert_eq!(segs.len(), 3 * 4 * 2); + assert!(segs.chunks(2).all(|s| s[1] == s[0] + 1)); + assert!((pos[0][0] + 0.2).abs() < 1e-6 && (pos[0][1] - 2.0).abs() < 1e-6); + // half-width over HALF_WIDTH_MAX in TEXCOORD_1.x: root fade starts at 0, tapers toward the tip + let w: Vec = r.read_tex_coords(1).unwrap().into_f32().map(|t| t[0] * HALF_WIDTH_MAX).collect(); + assert!(w[0] < 1e-7 && w[1] > 1e-5 && w[1] <= 5e-5 + 1e-7, "{w:?}"); + assert_eq!(extras["cc5"]["ribbons"], json!(true)); + } + // weights stay normalised after quantization + for wt in r.read_weights(0).unwrap().into_f32() { + assert!((wt.iter().sum::() - 1.0).abs() < 0.01); + } + } + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/crates/cc5-shape/Cargo.toml b/crates/cc5-shape/Cargo.toml new file mode 100644 index 0000000..d90b94c --- /dev/null +++ b/crates/cc5-shape/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "cc5-shape" +version = "0.1.0" +edition = "2021" +publish = false +description = "Apply Character Creator 5 shaping sliders to a character GLB without CC5 (Rust core + optional CPython bindings)." + +[lib] +name = "cc5_shape" +crate-type = ["rlib", "cdylib"] + +[features] +# CPython extension module `cc5_shape` (build with maturin; see pyproject.toml). +python = ["dep:pyo3"] + +[dependencies] +anyhow = "1" +# 0.29 = the glam Bevy 0.15 uses, so cc-render's types unify with ours. +glam = "0.29" +# parallel conform bind (millions of hair vertices); already in Bevy's tree +rayon = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +pyo3 = { version = "0.26", features = ["abi3-py310", "extension-module"], optional = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/cc5-shape/README.md b/crates/cc5-shape/README.md new file mode 100644 index 0000000..27096df --- /dev/null +++ b/crates/cc5-shape/README.md @@ -0,0 +1,98 @@ +# cc5-shape + +Apply Character Creator 5 shaping sliders (body type, height, face features — the +Morphs panel) to a character GLB without CC5. One Rust implementation, usable from +Rust and from CPython. + +Inputs, produced once per character by +`remotemedia-sdk/scripts/avatars/cc5_editor/build_cc5_editor_pack.ps1`: + +- `character.glb` — the character's reference export; every vertex carries a + `_CCIDX` attribute (its CC5 vertex index); +- `pack/` — `pack.json` + `pack.bin`, what each slider does to that character. + +See `../../docs/design/avatar-editor.md` for how the pack is captured and how +accurate it is. + +## Rust + +```toml +# from another repository +cc5-shape = { git = "https://github.com/RemoteMedia-SDK/cc-render", package = "cc5-shape" } +``` + +```rust +let pack = cc5_shape::ShapePack::load("pack".as_ref())?; +let weights = pack.weights_from([("Character Height", 0.5), ("Body Fat A", 0.3)])?; +let delta = pack.evaluate(&weights); // per-vertex / per-joint deltas, glTF frame (m) +let shaped = cc5_shape::shape_glb(&std::fs::read("character.glb")?, &pack, &delta)?; +std::fs::write("shaped.glb", shaped)?; +``` + +Sliders are found by id or by display name (an error if a name is ambiguous — +use the id then); values are range-checked. + +Facial expressions (CC5's ~400 morph targets; Bevy loads at most 64 per mesh) +come as an expression pack (`expressions.json` + `expressions.bin`, built by +`cc5-fbx expressions` from the reference FBX) in the same vertex space, for +CPU morphing on top of the shape: + +```rust +let expr = cc5_shape::ExpressionPack::load("expressions".as_ref())?; +let mut w = expr.weights_from([("Eye_Blink_L", 1.0), ("Jaw_Open", 0.3)])?; +let arkit = cc5_shape::ArkitMap::load("character.arkit_map.json".as_ref(), &expr)?; +w.extend(arkit.weights([("mouthSmileLeft", 0.6)])?); +expr.add_to(&w, &mut delta.verts); // layer on a ShapeDelta (or expr.evaluate(&w)) +``` + +HD characters: CC5 renders HD characters subdivided (level 1 = 4x faces, level 2 = +16x) with sculpted detail. `build_hd_map.py` (or `build_cc5_editor_pack.ps1 -HdLevel 1`) +writes a map from the pack's level-0 space to CC5's HD FBX export, so the same pack +shapes the HD mesh: + +```rust +let hd = cc5_shape::SubdivMap::load("hd/map".as_ref())?; +hd.check(&pack)?; +let hd_delta = hd.expand(&pack.evaluate(&weights)); +let shaped = cc5_shape::shape_glb_hd(&std::fs::read("hd/character_hd.glb")?, &pack, &hd, &hd_delta)?; +``` + +Exact for sliders that only reshape the base mesh (p99 within 0.02 mm of CC5's own +HD export). HD presets and large body sliders also change the HD level directly, +which a level-0 pack cannot see: p99 up to ~0.8 mm, max ~3 mm. + +## Python + +```sh +cd crates/cc5-shape +maturin build --release # wheel: target/wheels/cc5_shape-*-cp310-abi3-*.whl +pip install ../../target/wheels/cc5_shape-*.whl +``` + +```python +import cc5_shape + +pack = cc5_shape.Pack("pack") +w = pack.weights({"Character Height": 0.5, "Body Fat A": 0.3}) +shaped = pack.shape_glb(open("character.glb", "rb").read(), w) + +verts, joints = pack.evaluate(w) # bytes: f32 xyz per vertex; {joint: (x, y, z)} +# numpy.frombuffer(verts, "=1.5,<2"] +build-backend = "maturin" + +[tool.maturin] +features = ["python"] +module-name = "cc5_shape" +python-source = "python" diff --git a/crates/cc5-shape/python/cc5_shape/__init__.py b/crates/cc5-shape/python/cc5_shape/__init__.py new file mode 100644 index 0000000..13deec7 --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/__init__.py @@ -0,0 +1,6 @@ +"""CC5 shaping sliders outside CC5 (Rust core; see Pack), CC5 facial expressions (Expressions), +CC5 HD (subdivided) meshes (SubdivMap) and uncaptured meshes such as library hair +following the body (Conform).""" +from .cc5_shape import ArkitMap, Conform, Expressions, Pack, SubdivMap + +__all__ = ["ArkitMap", "Conform", "Expressions", "Pack", "SubdivMap"] diff --git a/crates/cc5-shape/python/cc5_shape/__init__.pyi b/crates/cc5-shape/python/cc5_shape/__init__.pyi new file mode 100644 index 0000000..1b3fdea --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/__init__.pyi @@ -0,0 +1,84 @@ +from os import PathLike +from typing import TypedDict + +class Slider(TypedDict): + id: str + name: str + category: str + min: float + max: float + base: float + +class Pack: + """A CC5 slider pack (`pack.json` + `pack.bin`).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def vertex_count(self) -> int: ... + @property + def joints(self) -> list[str]: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: ... + @property + def sliders(self) -> list[Slider]: ... + def categories(self) -> list[tuple[str, list[int]]]: ... + def find(self, key: str) -> int: ... + def base_weights(self) -> list[float]: ... + def weights(self, set: dict[str, float] | None = None) -> list[float]: ... + def evaluate(self, weights: list[float]) -> tuple[bytes, dict[str, tuple[float, float, float]]]: ... + def recipe(self, weights: list[float]) -> str: ... + def weights_from_recipe(self, recipe: str) -> list[float]: ... + def state(self, i: int, which: str) -> tuple[float, bytes, bytes, bytes, bytes] | None: ... + def shape_glb(self, glb: bytes, weights: list[float], hd: SubdivMap | None = None) -> bytes: + """Shaped copy of a GLB carrying `_CCIDX`; with `hd`, of the CC5 HD export.""" + +class Expressions: + """A CC5 expression pack (`expressions.json` + `expressions.bin`): facial morphs + as sparse vertex deltas in the slider pack's global vertex space (glTF frame, m).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def vertex_count(self) -> int: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: ... + @property + def names(self) -> list[str]: ... + def find(self, name: str) -> int: ... + def meshes_of(self, name: str) -> list[str]: ... + def deltas(self, name: str) -> tuple[bytes, bytes]: ... + def evaluate(self, weights: dict[str, float]) -> bytes: ... + def __len__(self) -> int: ... + +class ArkitMap: + """ARKit-52 blendshape weights -> expression weights (runtime ARKit map JSON).""" + def __init__(self, path: str | PathLike[str], expressions: Expressions) -> None: ... + @property + def arkit_names(self) -> list[str]: ... + @property + def missing(self) -> list[str]: ... + def weights(self, arkit: dict[str, float]) -> dict[str, float]: ... + +class SubdivMap: + """Slider pack (CC5 subdivision 0) -> CC5 HD mesh map (`hd_map.json` + `hd_map.bin`).""" + def __init__(self, dir: str | PathLike[str]) -> None: ... + @property + def level(self) -> int: ... + @property + def vertex_count(self) -> int: ... + @property + def meshes(self) -> list[tuple[str, int, int]]: + """`[(mesh name, HD offset, HD count)]`.""" + def expand(self, verts: bytes) -> bytes: + """Base-space f32 xyz per vertex -> HD-space f32 xyz per vertex.""" + +class Conform: + """Bind an uncaptured mesh (hair cards, a beard) to body anchor vertices so it + follows the body's shape. `body` / `asset`: f32 xyz bytes of bind positions in + one space; `anchors`: indices into `body` (the head for scalp hair, face for beards); + `mode`: "scalp" (wide falloff + whole-head similarity), "skin" or "head_box" + (map the anchors' bounding box: hair layers CC5 moves rigidly).""" + + def __init__(self, body: bytes, anchors: list[int], asset: bytes, mode: str = "scalp", k: int | None = None, + sigma_min: float | None = None, sigma_scale: float | None = None, + similarity: bool | None = None) -> None: ... + def apply(self, body_delta: bytes) -> bytes: + """Body vertex deltas (f32 xyz bytes) -> asset vertex deltas (f32 xyz bytes).""" + def __len__(self) -> int: ... diff --git a/crates/cc5-shape/python/cc5_shape/fit.py b/crates/cc5-shape/python/cc5_shape/fit.py new file mode 100644 index 0000000..cfd106b --- /dev/null +++ b/crates/cc5-shape/python/cc5_shape/fit.py @@ -0,0 +1,426 @@ +"""Conform a character to a target shape with its CC5 sliders (needs numpy + scipy). + +The pack's linear model, shape(w) = base + sum_i t_i * delta_i, turns "make our +avatar look like this mesh" into bounded least squares, iterated with closest-point +matching because the target (a scan, another character, an SMPL body) has its own +topology: + + for each iteration: + pair our vertices with the target surface (and target samples with our + nearest vertices, so the shape cannot shrink into part of the target) + solve min |A t - r|^2 + reg |t|^2 with every t_i in [0, 1] + +Each slider has up to two captured sides (towards max / towards min), so it starts as +two columns; afterwards every slider keeps its dominant side and the fit is solved +again with one column per slider, so the result is a valid slider weight vector that +`Pack.shape_glb` and the editor accept as is. + + import cc5_shape + from cc5_shape import fit + pack = cc5_shape.Pack("pack") + base = fit.base_mesh(pack, open("character.glb", "rb").read()) + target_v, target_f = fit.load_mesh("target.glb") # or .obj + result = fit.fit(pack, base, target_v, target_f, meshes=["CC_Base_Body"], + categories=["Actor/Body/", "Actor/HD Body/"]) + shaped = pack.shape_glb(open("character.glb", "rb").read(), result.weights) + +Positions are in the glTF frame (Y-up, meters). The target should stand in roughly the +same pose as the character's bind pose (CC5 A-pose). +""" +from __future__ import annotations + +import json +import struct +from dataclasses import dataclass, field + +import numpy as np + +_DT = {5120: np.int8, 5121: np.uint8, 5122: np.int16, 5123: np.uint16, 5125: np.uint32, 5126: np.float32} +_N = {"SCALAR": 1, "VEC2": 2, "VEC3": 3, "VEC4": 4, "MAT4": 16} + + +def _glb(data: bytes): + if data[:4] != b"glTF": + raise ValueError("not a GLB file") + n = struct.unpack_from(" 1 else out + return g, acc + + +def _node_worlds(g): + nodes = g.get("nodes", []) + parent = {c: i for i, n in enumerate(nodes) for c in n.get("children", [])} + + def local(n): + if "matrix" in n: + return np.array(n["matrix"], float).reshape(4, 4).T + x, y, z, w = n.get("rotation", [0, 0, 0, 1]) + R = np.array([[1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)], + [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)], + [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)]]) + M = np.eye(4) + M[:3, :3] = R * np.array(n.get("scale", [1, 1, 1])) + M[:3, 3] = n.get("translation", [0, 0, 0]) + return M + W = {} + + def world(i): + if i not in W: + W[i] = (world(parent[i]) if i in parent else np.eye(4)) @ local(nodes[i]) + return W[i] + return [world(i) for i in range(len(nodes))] + + +@dataclass +class Base: + """Our character's bind mesh in the pack's global vertex space.""" + verts: np.ndarray # (vertex_count, 3) + faces: np.ndarray # (F, 3) global vertex indices + mesh_of: np.ndarray # (vertex_count,) index into names + names: list[str] + + def mask(self, meshes=None) -> np.ndarray: + if not meshes: + return np.ones(len(self.verts), bool) + ids = [self.names.index(m) for m in meshes] + return np.isin(self.mesh_of, ids) + + +def base_mesh(pack, glb: bytes) -> Base: + """Bind mesh of the character GLB (with `_CCIDX`), in the pack's vertex space.""" + g, acc = _glb(glb) + offsets = {name: off for name, off, _ in pack.meshes} + names = [name for name, _, _ in pack.meshes] + verts = np.full((pack.vertex_count, 3), np.nan) + mesh_of = np.zeros(pack.vertex_count, int) + for name, off, cnt in pack.meshes: + mesh_of[off:off + cnt] = names.index(name) + faces = [] + for m in g["meshes"]: + if m.get("name") not in offsets: + continue + off = offsets[m["name"]] + for p in m["primitives"]: + at = p["attributes"] + if "_CCIDX" not in at: + raise ValueError("mesh %s has no _CCIDX attribute" % m["name"]) + gi = off + acc(at["_CCIDX"]).astype(np.int64) + verts[gi] = acc(at["POSITION"]) + if "indices" in p: + faces.append(gi[acc(p["indices"]).astype(np.int64)].reshape(-1, 3)) + if np.isnan(verts).any(): + raise ValueError("%d pack vertices are not in the GLB" % int(np.isnan(verts[:, 0]).sum())) + return Base(verts, np.concatenate(faces) if faces else np.zeros((0, 3), int), mesh_of, names) + + +def load_mesh(path: str, meshes=None): + """(vertices (V,3), faces (F,3)) of a .glb (all mesh primitives, node transforms + applied, Y-up meters) or .obj file. `meshes`: only glTF meshes with these names.""" + if path.lower().endswith(".obj"): + v, f = [], [] + for line in open(path, "rb"): + if line.startswith(b"v "): + v.append([float(x) for x in line.split()[1:4]]) + elif line.startswith(b"f "): + idx = [int(t.split(b"/")[0]) for t in line.split()[1:]] + idx = [i - 1 if i > 0 else len(v) + i for i in idx] + f.extend([idx[0], idx[k], idx[k + 1]] for k in range(1, len(idx) - 1)) + return np.array(v, float), np.array(f, int).reshape(-1, 3) + g, acc = _glb(open(path, "rb").read()) + worlds = _node_worlds(g) + vs, fs, n = [], [], 0 + for i, node in enumerate(g.get("nodes", [])): + if "mesh" not in node: + continue + m = g["meshes"][node["mesh"]] + if meshes and m.get("name") not in meshes: + continue + # skinned meshes are drawn in bind space; others through their node + W = np.eye(4) if "skin" in node else worlds[i] + for p in m["primitives"]: + pos = acc(p["attributes"]["POSITION"]).astype(float) + vs.append(pos @ W[:3, :3].T + W[:3, 3]) + if "indices" in p: + fs.append(acc(p["indices"]).astype(int).reshape(-1, 3) + n) + n += len(pos) + return np.concatenate(vs), (np.concatenate(fs) if fs else np.zeros((0, 3), int)) + + +def sample_surface(v, f, n, seed=0, normals=False): + """n points spread over the triangles' area (the vertices if there are no faces); + with normals=True also their unit normals (from the triangles, or estimated from + the 16 nearest points for a bare point cloud).""" + if len(f) == 0: + pts = np.asarray(v, float) + if not normals: + return pts + from scipy.spatial import cKDTree + _, nb = cKDTree(pts).query(pts, 16) + nrm = np.empty_like(pts) + for i, idx in enumerate(nb): + q = pts[idx] - pts[idx].mean(0) + nrm[i] = np.linalg.svd(q, full_matrices=False)[2][2] + return pts, nrm + rng = np.random.default_rng(seed) + a, b, c = v[f[:, 0]], v[f[:, 1]], v[f[:, 2]] + cr = np.cross(b - a, c - a) + area = np.linalg.norm(cr, axis=1) + tri = rng.choice(len(f), n, p=area / area.sum()) + u, w = rng.random((2, n)) + flip = u + w > 1 + u[flip], w[flip] = 1 - u[flip], 1 - w[flip] + pts = a[tri] + u[:, None] * (b[tri] - a[tri]) + w[:, None] * (c[tri] - a[tri]) + if not normals: + return pts + return pts, cr[tri] / np.maximum(area[tri], 1e-20)[:, None] + + +def vertex_normals(v, f): + """Area-weighted unit vertex normals.""" + nrm = np.zeros_like(v, dtype=float) + cr = np.cross(v[f[:, 1]] - v[f[:, 0]], v[f[:, 2]] - v[f[:, 0]]) + for k in range(3): + np.add.at(nrm, f[:, k], cr) + return nrm / np.maximum(np.linalg.norm(nrm, axis=1), 1e-20)[:, None] + + +@dataclass +class FitResult: + weights: list[float] # full slider weight vector (pack order) + changed: dict[str, float] # slider id -> weight, for sliders that moved + transform: np.ndarray # 4x4 applied to the target (target -> our frame) + history: list[dict] = field(default_factory=list) # per iteration: mean/p95 distance (m) + + def save_recipe(self, pack, path): + """Write the fit as a recipe (open it in the editor, or pack.weights_from_recipe).""" + with open(path, "w", encoding="utf-8") as f: + f.write(pack.recipe(self.weights)) + + +def _columns(pack, sliders, mask): + """Sparse columns over masked vertices: list of (slider index, side, rows, deltas).""" + gidx = np.flatnonzero(mask) + pos = np.full(len(mask), -1) + pos[gidx] = np.arange(len(gidx)) + cols = [] + for i in sliders: + s = pack.sliders[i] + for which in ("max", "min"): + st = pack.state(i, which) + if st is None or abs(st[0] - s["base"]) < 1e-9: + continue + vi = np.frombuffer(st[1], "= 0 + if keep.any(): + cols.append((i, which, st[0], pos[vi[keep]], vd[keep])) + return gidx, cols + + +def _dense(cols, rows, n): + A = np.zeros((n, 3, len(cols))) + for k, (_, _, _, r, d) in enumerate(cols): + A[r, :, k] = d + return A[rows] + + +def _solve(A, r, reg, lo, hi): + """min |A t - r|^2 + reg |t|^2, lo <= t <= hi, via the normal equations (fast for + many rows: the bounded problem is solved on the n x n Cholesky factor).""" + from scipy.optimize import lsq_linear + G = A.T @ A + reg * np.eye(A.shape[1]) + h = A.T @ r + L = np.linalg.cholesky(G) + y = np.linalg.solve(L, h) + return lsq_linear(L.T, y, bounds=(lo, hi), method="bvls").x + + +def target_from_cc_glb(pack, glb: bytes, meshes=("CC_Base_Body",)) -> np.ndarray: + """Per-vertex target positions (pack vertex space, NaN where absent) from another + CC character's GLB with `_CCIDX` (same CC base topology), for `fit(..., matched=...)`.""" + g, acc = _glb(glb) + offsets = {name: off for name, off, _ in pack.meshes} + out = np.full((pack.vertex_count, 3), np.nan) + for m in g["meshes"]: + if m.get("name") in offsets and m.get("name") in meshes: + for p in m["primitives"]: + at = p["attributes"] + if "_CCIDX" in at: + out[offsets[m["name"]] + acc(at["_CCIDX"]).astype(np.int64)] = acc(at["POSITION"]) + return out + + +def fit(pack, base: Base, target_v=None, target_f=None, *, meshes=("CC_Base_Body",), categories=None, + sliders=None, iterations=8, reg=1e-4, align="translate", samples=80000, stride=1, + reverse_weight=1.0, max_dist=0.05, min_normal_dot=0.5, matched=None, landmarks=None, + landmark_weight=100.0, verbose=True) -> FitResult: + """Fit slider weights so `base` matches the target surface. + + meshes our meshes that must match (default: the body skin only) + categories slider category prefixes to use (e.g. ["Actor/Body/"]); or `sliders` + as explicit slider indices; default: every slider + align "translate" (feet on the floor, centred), "similarity" (also scale), + or "none" (target already in our frame) + reg pull towards the character's own design (larger = smaller changes) + stride use every n-th of our vertices in the solve (speed) + max_dist ignore pairs further apart than this (m) + min_normal_dot ignore pairs whose surfaces face different ways (e.g. toe vs toe) + matched (vertex_count, 3) known target position per vertex (NaN = none), + e.g. target_from_cc_glb() for another CC character: solved directly, + point to point; target_v may then be None + landmarks {our global vertex index: target point (target frame)} pinning + features a surface fit lets slide (eye corners, nose tip, chin...), + weighted by landmark_weight + + Pairs are point-to-plane: only the distance along the surface normal counts, so + vertices may slide along the target surface (sampling density does not bias it). + """ + from scipy.spatial import cKDTree + target_v = None if target_v is None else np.asarray(target_v, float) + target_f = np.zeros((0, 3), int) if target_f is None else np.asarray(target_f, int) + mask = base.mask(meshes) + if sliders is None: + sliders = [i for i, s in enumerate(pack.sliders) + if categories is None or s["category"].startswith(tuple(categories))] + ours0 = base.verts[mask] + + T = np.eye(4) + if target_v is None and matched is None: + raise ValueError("give target_v (a surface) and/or matched (per-vertex targets)") + if target_v is None: + target_v = matched[mask][~np.isnan(matched[mask][:, 0])] + surface = False + else: + surface = True + if align in ("translate", "similarity"): + s = 1.0 + if align == "similarity": + s = np.ptp(ours0[:, 1]) / np.ptp(target_v[:, 1]) + c_t = np.array([np.mean(target_v[:, 0]), target_v[:, 1].min(), np.mean(target_v[:, 2])]) + c_o = np.array([np.mean(ours0[:, 0]), ours0[:, 1].min(), np.mean(ours0[:, 2])]) + T[:3, :3] *= s + T[:3, 3] = c_o - s * c_t + elif align != "none": + raise ValueError("align must be translate, similarity or none") + def to_ours(p): + return np.asarray(p, float) @ T[:3, :3].T + T[:3, 3] + if target_v is not None: + tv = to_ours(target_v) + tpts, tnrm = sample_surface(tv, target_f, samples, normals=True) + ttree = cKDTree(tpts) + + gidx, cols = _columns(pack, sliders, mask) + n = len(gidx) + rows = np.arange(0, n, stride) + A3 = _dense(cols, rows, n) # (rows, 3, C) + local = np.full(len(mask), -1) + local[gidx] = np.arange(n) + our_faces = local[base.faces[np.all(mask[base.faces], axis=1)]] + rev_pts, rev_nrm = tpts[::2], tnrm[::2] + history = [] + + def shape(tt, cc): + out = ours0.copy() + for k, (_, _, _, r, d) in enumerate(cc): + if tt[k]: + out[r] += tt[k] * d + return out + + pin_rows, pin_pts = np.zeros(0, int), np.zeros((0, 3)) + if matched is not None: + mm = to_ours(matched[gidx]) + have = ~np.isnan(mm[:, 0]) + pin_rows, pin_pts = np.flatnonzero(have), mm[have] + pin_w = np.ones(len(pin_rows)) + if landmarks: + lr = np.array([local[g] for g in landmarks], int) + if (lr < 0).any(): + raise ValueError("landmark vertices must be in the fitted meshes") + pin_rows = np.concatenate([pin_rows, lr]) + pin_pts = np.vstack([pin_pts, to_ours(np.array(list(landmarks.values()), float))]) + pin_w = np.concatenate([pin_w if matched is not None else np.zeros(0), np.full(len(lr), landmark_weight)]) + A_pin = _dense(cols, pin_rows, n) if len(pin_rows) else None + + def system(cc, tt, A, keep=None): + pins = None + if A_pin is not None: + Ap = A_pin if keep is None else A_pin[:, :, keep] + sw = np.sqrt(pin_w)[:, None, None] + pins = ((Ap * sw).reshape(-1, Ap.shape[2]), ((pin_pts - ours0[pin_rows]) * sw[:, :, 0]).ravel()) + if not surface: + cur = shape(tt, cc) + d = np.linalg.norm(cur[pin_rows] - pin_pts, axis=1) + return pins[0], pins[1], {"mean_m": float(d.mean()), "p95_m": float(np.percentile(d, 95)), "pairs": len(d)} + cur = shape(tt, cc) + onrm = vertex_normals(cur, our_faces) + cr, on = cur[rows], onrm[rows] + # forward: our vertex -> nearest target point, along the target normal + d, j = ttree.query(cr) + ok = (d < max_dist) & (np.abs(np.sum(tnrm[j] * on, 1)) > min_normal_dot) + nf = tnrm[j[ok]] + Af = np.einsum("rk,rkc->rc", nf, A[ok]) + bf = np.sum(nf * (tpts[j[ok]] - ours0[rows][ok]), 1) + # reverse: target point -> our nearest vertex, along our normal + dr, k = cKDTree(cr).query(rev_pts) + okr = (dr < max_dist) & (np.abs(np.sum(rev_nrm * on[k], 1)) > min_normal_dot) + nr = on[k[okr]] + Ar = np.einsum("rk,rkc->rc", nr, A[k[okr]]) * np.sqrt(reverse_weight) + br = np.sum(nr * (rev_pts[okr] - ours0[rows][k[okr]]), 1) * np.sqrt(reverse_weight) + plane = np.abs(np.sum(nf * (tpts[j[ok]] - cr[ok]), 1)) + stats = {"mean_m": float(plane.mean()), "p95_m": float(np.percentile(plane, 95)), + "pairs": int(ok.sum() + okr.sum())} + if pins is not None: + return np.vstack([Af, Ar, pins[0]]), np.concatenate([bf, br, pins[1]]), stats + return np.vstack([Af, Ar]), np.concatenate([bf, br]), stats + + t = np.zeros(len(cols)) + if not surface: + iterations = 1 # point to point with known pairs: linear, one solve + for it in range(iterations): + A, b, st = system(cols, t, A3) + history.append(dict(st, iteration=it)) + if verbose: + print("fit %d: distance mean %.2f mm p95 %.2f mm (%d pairs)" + % (it, st["mean_m"] * 1e3, st["p95_m"] * 1e3, st["pairs"])) + t = _solve(A, b, reg, 0.0, 1.0) + + # one side per slider: keep the dominant side, re-solve with single columns + best = {} + for k, (i, which, w, r, d) in enumerate(cols): + mag = t[k] * np.abs(d).sum() + if i not in best or mag > best[i][0]: + best[i] = (mag, k) + keep = sorted(k for _, k in best.values()) + cols1 = [cols[k] for k in keep] + A31 = A3[:, :, keep] + t1 = t[keep] + for _ in range(3 if surface else 1): + A, b, st = system(cols1, t1, A31, keep) + t1 = _solve(A, b, reg, 0.0, 1.0) + _, _, st = system(cols1, t1, A31, keep) + history.append(dict(st, iteration="final")) + if verbose: + print("fit final: distance mean %.2f mm p95 %.2f mm" % (st["mean_m"] * 1e3, st["p95_m"] * 1e3)) + + weights = pack.base_weights() + changed = {} + for tk, (i, which, w, _, _) in zip(t1, cols1): + if tk > 1e-4: + s = pack.sliders[i] + weights[i] = float(s["base"] + tk * (w - s["base"])) + changed[s["id"]] = weights[i] + return FitResult(weights, changed, T, history) diff --git a/crates/cc5-shape/python/cc5_shape/py.typed b/crates/cc5-shape/python/cc5_shape/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/crates/cc5-shape/src/conform.rs b/crates/cc5-shape/src/conform.rs new file mode 100644 index 0000000..69c230b --- /dev/null +++ b/crates/cc5-shape/src/conform.rs @@ -0,0 +1,447 @@ +//! Make meshes that were not captured from CC5 (hair cards, beards, brows from a +//! hair library) follow the body's shape: `asset_delta = conform.apply(body_delta)`. +//! +//! Each asset vertex is bound at load time to its `k` nearest *anchor* vertices +//! of the body (the scalp for scalp hair, the face for beards and brows) with +//! Gaussian weights `exp(-((d - d_min) / σ)²)`, `σ = max(sigma_min, sigma_scale · d_min)`: +//! verts on the skin follow their spot, hair far from it averages a wider area. +//! With `similarity`, the anchors' move is first fitted by one similarity +//! transform (scale, rotation, translation; Horn's method), applied to the asset +//! as a whole, and only the anchors' residual moves are blended. That keeps long +//! hair in proportion when the head scales or turns. +//! +//! Calibrated against CC5 itself (`cc5-hair check`: the 14 Smart Hair +//! groups with whole recipes applied by CC5): scalp hair that deforms (hair caps, +//! short styles) anchored to the whole head + similarity: p50 0.4 mm, p99 3–4.7 +//! mm; beards and brows on the face: p50 0.3–0.6 mm. +//! +//! [`ConformOptions::HEAD_BOX`]: CC5 moves some hair layers rigidly (Smart Hair +//! bangs, buns, the long outer layers; CC5 itself with whole recipes applied: +//! an exact per-axis scale + translation): it maps the head's bounding box. The +//! box of the head without ears and nose (`regions.json` "head_box") reproduces +//! it to p50 0.5–1.8 mm, p99 ≤ 2.2 mm, where surface conform is 15–20 mm off. +//! Layers lying on the scalp / skin (hair caps, beards, brows) deform with it +//! and keep the surface conform (p50 0.3–2 mm). + +use anyhow::{bail, Result}; +use glam::{DMat3, DVec3, DVec4}; +use rayon::prelude::*; + +#[derive(Debug, Clone, Copy)] +pub struct ConformOptions { + /// Anchors per asset vertex. + pub k: usize, + /// Minimum Gaussian width (m). + pub sigma_min: f32, + /// Gaussian width per metre of distance from the anchors. + pub sigma_scale: f32, + /// Fit and apply one similarity transform of all anchors first. + pub similarity: bool, + /// Only map the anchors' axis-aligned bounding box (per-axis scale + + /// translation); `k` and the Gaussians are unused. + pub box_fit: bool, +} + +impl ConformOptions { + /// Scalp hair (anchors: the "head" region): falloff widening with the + /// distance from the head (roots and a scalp cap stay on their spot, hair + /// further out averages a wider area) + a whole-head similarity. + pub const SCALP: Self = Self { k: 16, sigma_min: 0.005, sigma_scale: 0.5, similarity: true, box_fit: false }; + /// Beards, brows, anything lying on the face: follow the nearest skin. + pub const SKIN: Self = Self { k: 16, sigma_min: 0.003, sigma_scale: 0.5, similarity: false, box_fit: false }; + /// Hair layers CC5 moves rigidly with the head's bounding box (anchors: the + /// "head_box" region). + pub const HEAD_BOX: Self = Self { k: 0, sigma_min: 0.0, sigma_scale: 0.0, similarity: false, box_fit: true }; +} + +pub struct Conform { + opts: ConformOptions, + /// Anchor indices into the body vertex array. + anchors: Vec, + anchor_bind: Vec, + asset_bind: Vec, + /// Per asset vertex: `k` (anchor slot, weight) pairs, flattened. + bind: Vec<(u32, f32)>, +} + +impl Conform { + /// `body`: the body's bind positions; `anchors`: indices into `body`; + /// `asset`: the asset's bind positions, same space as `body`. + pub fn bind(body: &[[f32; 3]], anchors: &[u32], asset: &[[f32; 3]], opts: ConformOptions) -> Result { + if anchors.is_empty() { + bail!("conform: no anchor vertices"); + } + if let Some(&bad) = anchors.iter().find(|&&a| a as usize >= body.len()) { + bail!("conform: anchor {bad} past the body's {} vertices", body.len()); + } + let anchor_bind: Vec = anchors.iter().map(|&a| DVec3::from(body[a as usize].map(f64::from))).collect(); + let asset_bind: Vec = asset.iter().map(|p| DVec3::from(p.map(f64::from))).collect(); + if opts.box_fit { + return Ok(Self { opts: ConformOptions { k: 0, ..opts }, anchors: anchors.to_vec(), anchor_bind, asset_bind, bind: Vec::new() }); + } + let k = opts.k.min(anchors.len()).max(1); + // k nearest anchors per vertex from a uniform grid (exact), in parallel: + // millions of hair vertices against thousands of anchors + let grid = AnchorGrid::new(&anchor_bind); + let mut bind = vec![(0u32, 0f32); asset_bind.len() * k]; + bind.par_chunks_mut(k).zip(asset_bind.par_iter()).for_each_init(Vec::new, |near, (out, &p)| { + grid.nearest(&anchor_bind, p, k, near); + gaussian_weights(&near[..k], opts, out); + }); + Ok(Self { + opts: ConformOptions { k, ..opts }, + anchors: anchors.to_vec(), + anchor_bind, + asset_bind, + bind, + }) + } + + pub fn len(&self) -> usize { + self.asset_bind.len() + } + + pub fn is_empty(&self) -> bool { + self.asset_bind.is_empty() + } + + /// Asset vertex deltas for a body delta (one entry per body vertex, same + /// space as the bind positions). + pub fn apply(&self, body_delta: &[[f32; 3]]) -> Vec<[f32; 3]> { + let d: Vec = self.anchors.iter().map(|&a| DVec3::from(body_delta[a as usize].map(f64::from))).collect(); + if self.opts.box_fit { + let bounds = |pts: &mut dyn Iterator| pts.fold((DVec3::MAX, DVec3::MIN), |(lo, hi), p| (lo.min(p), hi.max(p))); + let (lo0, hi0) = bounds(&mut self.anchor_bind.iter().copied()); + let (lo1, hi1) = bounds(&mut self.anchor_bind.iter().zip(&d).map(|(p, d)| *p + *d)); + let ext = hi0 - lo0; + let s = DVec3::select(ext.cmpgt(DVec3::splat(1e-9)), (hi1 - lo1) / ext, DVec3::ONE); + let t = lo1 - s * lo0; + return self.asset_bind.par_iter().map(|p| (*p * s + t - *p).as_vec3().to_array()).collect(); + } + let (global, residual): (Option, Vec) = if self.opts.similarity && d.iter().any(|v| *v != DVec3::ZERO) { + let moved: Vec = self.anchor_bind.iter().zip(&d).map(|(p, d)| *p + *d).collect(); + let s = Similarity::fit(&self.anchor_bind, &moved); + let r = self.anchor_bind.iter().zip(&d).map(|(p, d)| *d - (s.apply(*p) - *p)).collect(); + (Some(s), r) + } else { + (None, d) + }; + let k = self.opts.k; + self.asset_bind + .par_iter() + .enumerate() + .map(|(v, p)| { + let mut acc = global.as_ref().map_or(DVec3::ZERO, |s| s.apply(*p) - *p); + for &(i, w) in &self.bind[v * k..(v + 1) * k] { + acc += residual[i as usize] * w as f64; + } + acc.as_vec3().to_array() + }) + .collect() + } +} + +/// Weights `exp(-((d - d_min) / σ)²)` (normalised) for the `k` nearest anchors +/// `near` (squared distance, anchor slot), nearest first. +fn gaussian_weights(near: &[(f64, u32)], opts: ConformOptions, out: &mut [(u32, f32)]) { + let d0 = near[0].0.sqrt(); + let sigma = (opts.sigma_min as f64).max(opts.sigma_scale as f64 * d0).max(1e-6); + let mut sum = 0.0; + for (o, &(d2, i)) in out.iter_mut().zip(near) { + let w = (-((d2.sqrt() - d0) / sigma).powi(2)).exp(); + *o = (i, w as f32); + sum += w; + } + for o in out.iter_mut() { + o.1 = (o.1 as f64 / sum) as f32; + } +} + +/// Anchor points bucketed in a uniform grid, for exact k-nearest queries. +struct AnchorGrid { + origin: DVec3, + cell: f64, + dims: [i64; 3], + /// CSR: anchors of cell `c` are `items[starts[c]..starts[c + 1]]`. + starts: Vec, + items: Vec, +} + +impl AnchorGrid { + fn new(pts: &[DVec3]) -> Self { + let (lo, hi) = pts.iter().fold((DVec3::MAX, DVec3::MIN), |(lo, hi), p| (lo.min(*p), hi.max(*p))); + let ext = (hi - lo).max(DVec3::splat(1e-6)); + // about one anchor per cell over the bounding volume (surface anchors + // fill only a shell of it: a few per occupied cell), at most 128 a side + let cell = (ext.x * ext.y * ext.z / pts.len() as f64).cbrt().max(ext.max_element() / 128.0).max(1e-4); + let dims = [0, 1, 2].map(|a| ((ext[a] / cell).floor() as i64 + 1).max(1)); + let n = (dims[0] * dims[1] * dims[2]) as usize; + let index = |p: &DVec3| { + let c = ((*p - lo) / cell).floor(); + let c = [0, 1, 2].map(|a| (c[a] as i64).clamp(0, dims[a] - 1)); + ((c[2] * dims[1] + c[1]) * dims[0] + c[0]) as usize + }; + let mut starts = vec![0u32; n + 1]; + for p in pts { + starts[index(p) + 1] += 1; + } + for c in 0..n { + starts[c + 1] += starts[c]; + } + let mut fill = starts.clone(); + let mut items = vec![0u32; pts.len()]; + for (i, p) in pts.iter().enumerate() { + let c = index(p); + items[fill[c] as usize] = i as u32; + fill[c] += 1; + } + Self { origin: lo, cell, dims, starts, items } + } + + /// The `k` anchors nearest `p` into `out[..k]` (squared distance, index), + /// nearest first. Searches cubic shells of cells around `p`'s cell: after + /// shell `r` every unseen anchor is at least `r` cells away, so the search + /// stops once the k-th distance is within that. + fn nearest(&self, pts: &[DVec3], p: DVec3, k: usize, out: &mut Vec<(f64, u32)>) { + out.clear(); + let c = ((p - self.origin) / self.cell).floor(); + let c = [c.x as i64, c.y as i64, c.z as i64]; + // shells beyond this reach no cell of the grid + let r_max = (0..3).map(|a| (c[a]).max(self.dims[a] - 1 - c[a]).max(0)).max().unwrap_or(0); + let mut r = 0i64; + loop { + for z in c[2] - r..=c[2] + r { + if z < 0 || z >= self.dims[2] { + continue; + } + for y in c[1] - r..=c[1] + r { + if y < 0 || y >= self.dims[1] { + continue; + } + let edge = (z - c[2]).abs() == r || (y - c[1]).abs() == r; + // inside the shell only the two x ends are new + let xs: &mut dyn Iterator = if edge { &mut (c[0] - r..=c[0] + r) } else if r == 0 { &mut std::iter::once(c[0]) } else { &mut [c[0] - r, c[0] + r].into_iter() }; + for x in xs { + if x < 0 || x >= self.dims[0] { + continue; + } + let cell = ((z * self.dims[1] + y) * self.dims[0] + x) as usize; + for &i in &self.items[self.starts[cell] as usize..self.starts[cell + 1] as usize] { + out.push((pts[i as usize].distance_squared(p), i)); + } + } + } + } + if out.len() >= k { + out.select_nth_unstable_by(k - 1, |a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + out.truncate(k); + let reach = r as f64 * self.cell; + if out[k - 1].0 <= reach * reach || r >= r_max { + break; + } + } else if r >= r_max { + break; + } + r += 1; + } + out.sort_unstable_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + } +} + +/// `q ≈ s·R·p + t`. +struct Similarity { + s: f64, + r: DMat3, + t: DVec3, +} + +impl Similarity { + fn apply(&self, p: DVec3) -> DVec3 { + self.s * (self.r * p) + self.t + } + + /// Horn's quaternion method (least squares), scale from the rotated spread. + fn fit(p: &[DVec3], q: &[DVec3]) -> Self { + let n = p.len() as f64; + let (mp, mq) = (p.iter().sum::() / n, q.iter().sum::() / n); + let mut m = [[0.0f64; 3]; 3]; // Σ a bᵀ + let mut aa = 0.0; + for (a, b) in p.iter().zip(q) { + let (a, b) = (*a - mp, *b - mq); + aa += a.length_squared(); + for (i, ai) in a.to_array().iter().enumerate() { + for (j, bj) in b.to_array().iter().enumerate() { + m[i][j] += ai * bj; + } + } + } + let [[sxx, sxy, sxz], [syx, syy, syz], [szx, szy, szz]] = m; + let nmat = [ + [sxx + syy + szz, syz - szy, szx - sxz, sxy - syx], + [syz - szy, sxx - syy - szz, sxy + syx, szx + sxz], + [szx - sxz, sxy + syx, -sxx + syy - szz, syz + szy], + [sxy - syx, szx + sxz, syz + szy, -sxx - syy + szz], + ]; + let qv = max_eigenvector4(nmat); + let rot = glam::DQuat::from_xyzw(qv.y, qv.z, qv.w, qv.x).normalize(); + let r = DMat3::from_quat(rot); + let mut ab = 0.0; + for (a, b) in p.iter().zip(q) { + ab += (*b - mq).dot(r * (*a - mp)); + } + let s = if aa > 0.0 { ab / aa } else { 1.0 }; + Self { s, r, t: mq - s * (r * mp) } + } +} + +/// Eigenvector of the largest eigenvalue of a symmetric 4×4 (cyclic Jacobi). +fn max_eigenvector4(mut a: [[f64; 4]; 4]) -> DVec4 { + let mut v = [[0.0f64; 4]; 4]; + for (i, row) in v.iter_mut().enumerate() { + row[i] = 1.0; + } + for _ in 0..50 { + let off: f64 = (0..4).flat_map(|i| (0..4).filter(move |&j| j != i).map(move |j| (i, j))).map(|(i, j)| a[i][j] * a[i][j]).sum(); + if off < 1e-24 { + break; + } + for p in 0..3 { + for q in p + 1..4 { + if a[p][q].abs() < 1e-300 { + continue; + } + let theta = (a[q][q] - a[p][p]) / (2.0 * a[p][q]); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..4 { + let (akp, akq) = (a[k][p], a[k][q]); + a[k][p] = c * akp - s * akq; + a[k][q] = s * akp + c * akq; + } + for k in 0..4 { + let (apk, aqk) = (a[p][k], a[q][k]); + a[p][k] = c * apk - s * aqk; + a[q][k] = s * apk + c * aqk; + } + for row in v.iter_mut() { + let (vp, vq) = (row[p], row[q]); + row[p] = c * vp - s * vq; + row[q] = s * vp + c * vq; + } + } + } + } + let best = (0..4).max_by(|&i, &j| a[i][i].total_cmp(&a[j][j])).unwrap(); + DVec4::new(v[0][best], v[1][best], v[2][best], v[3][best]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn grid() -> Vec<[f32; 3]> { + let mut g = Vec::new(); + for i in 0..10 { + for j in 0..10 { + g.push([i as f32 * 0.02 - 0.09, j as f32 * 0.02 - 0.09, 0.0]); + } + } + g + } + + fn close(a: [f32; 3], b: [f32; 3], tol: f32) -> bool { + (0..3).all(|i| (a[i] - b[i]).abs() < tol) + } + + #[test] + fn translation_carries_assets_along() { + let body = grid(); + let anchors: Vec = (0..body.len() as u32).collect(); + let asset = [[0.0, 0.0, 0.01], [0.03, -0.02, 0.2]]; + for opts in [ConformOptions::SCALP, ConformOptions::SKIN] { + let c = Conform::bind(&body, &anchors, &asset, opts).unwrap(); + let out = c.apply(&vec![[0.01, -0.02, 0.005]; body.len()]); + assert!(out.iter().all(|d| close(*d, [0.01, -0.02, 0.005], 1e-5)), "{out:?}"); + } + } + + #[test] + fn similarity_scales_far_vertices() { + let body = grid(); + let anchors: Vec = (0..body.len() as u32).collect(); + let asset = [[0.0, 0.0, 0.3]]; + // the body grows 10% about the origin; a vertex 30 cm out moves 3 cm + let delta: Vec<[f32; 3]> = body.iter().map(|p| p.map(|x| x * 0.1)).collect(); + let c = Conform::bind(&body, &anchors, &asset, ConformOptions::SCALP).unwrap(); + // a flat grid fixes no scale along z; the fitted scale applies uniformly + assert!(close(c.apply(&delta)[0], [0.0, 0.0, 0.03], 1e-4), "{:?}", c.apply(&delta)); + // rotation by 90° about z: (0.05, 0, 0.3) -> (0, 0.05, 0.3) + let rot: Vec<[f32; 3]> = body.iter().map(|p| [-p[1] - p[0], p[0] - p[1], 0.0]).collect(); + let c = Conform::bind(&body, &anchors, &[[0.05, 0.0, 0.3]], ConformOptions::SCALP).unwrap(); + assert!(close(c.apply(&rot)[0], [-0.05, 0.05, 0.0], 1e-4), "{:?}", c.apply(&rot)); + } + + #[test] + fn head_box_scales_per_axis() { + let body = grid(); + let anchors: Vec = (0..body.len() as u32).collect(); + // the grid grows 10% along x and moves 1 cm along z + let delta: Vec<[f32; 3]> = body.iter().map(|p| [p[0] * 0.1, 0.0, 0.01]).collect(); + let c = Conform::bind(&body, &anchors, &[[0.2, 0.05, 0.3]], ConformOptions::HEAD_BOX).unwrap(); + assert!(close(c.apply(&delta)[0], [0.02, 0.0, 0.01], 1e-5), "{:?}", c.apply(&delta)); + } + + /// The grid search gives the brute-force nearest anchors and weights. + #[test] + fn grid_bind_matches_brute_force() { + // xorshift: deterministic pseudo-random points without a dependency + let mut s = 0x9e37_79b9_7f4a_7c15u64; + let mut rnd = || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + (s >> 11) as f64 / (1u64 << 53) as f64 + }; + // anchors on a sphere (like a scalp), assets around and far outside it + let body: Vec<[f32; 3]> = (0..3000) + .map(|_| { + let (u, v) = (rnd() * std::f64::consts::TAU, rnd() * 2.0 - 1.0); + let r = (1.0 - v * v).sqrt(); + [(0.1 * r * u.cos()) as f32, (0.1 * r * u.sin()) as f32, (0.1 * v) as f32] + }) + .collect(); + let anchors: Vec = (0..body.len() as u32).collect(); + let asset: Vec<[f32; 3]> = (0..2000).map(|i| { + let f = if i % 10 == 0 { 3.0 } else { 0.3 }; + [((rnd() - 0.5) * f) as f32, ((rnd() - 0.5) * f) as f32, ((rnd() - 0.5) * f) as f32] + }).collect(); + for opts in [ConformOptions::SKIN, ConformOptions::SCALP] { + let c = Conform::bind(&body, &anchors, &asset, opts).unwrap(); + let anchor_bind: Vec = body.iter().map(|p| DVec3::from(p.map(f64::from))).collect(); + for (v, p) in asset.iter().enumerate() { + let p = DVec3::from(p.map(f64::from)); + let mut all: Vec<(f64, u32)> = anchor_bind.iter().enumerate().map(|(i, a)| (a.distance_squared(p), i as u32)).collect(); + all.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + let mut want = vec![(0u32, 0f32); c.opts.k]; + gaussian_weights(&all[..c.opts.k], opts, &mut want); + assert_eq!(&c.bind[v * c.opts.k..(v + 1) * c.opts.k], &want[..], "vertex {v}"); + } + } + } + + #[test] + fn skin_vertices_follow_their_spot() { + let body = grid(); + let anchors: Vec = (0..body.len() as u32).collect(); + // only one body vertex moves; an asset vertex right on it follows, a far one does not + let mut delta = vec![[0.0f32; 3]; body.len()]; + delta[55] = [0.0, 0.0, 0.01]; + let asset = [[body[55][0], body[55][1], 0.001], [body[0][0], body[0][1], 0.001]]; + let c = Conform::bind(&body, &anchors, &asset, ConformOptions::SKIN).unwrap(); + let out = c.apply(&delta); + assert!(out[0][2] > 0.009, "{out:?}"); + assert!(out[1][2].abs() < 1e-6, "{out:?}"); + } +} diff --git a/crates/cc5-shape/src/expr.rs b/crates/cc5-shape/src/expr.rs new file mode 100644 index 0000000..4bb7278 --- /dev/null +++ b/crates/cc5-shape/src/expr.rs @@ -0,0 +1,300 @@ +//! CC5 facial expressions (morph targets) on the CPU: load an expression pack and +//! evaluate a weighted mix into per-vertex deltas. +//! +//! A CC5 HD character has ~400 facial morphs per face mesh; Bevy loads at most 64 +//! morph targets per mesh, so they are kept out of the GLB (`cc5-fbx` drops +//! them) and applied here instead, like the shaping sliders. The pack is built +//! from the reference FBX's morph targets by `cc5-fbx expressions`: +//! +//! - `expressions.json` — `vertex_count` and `meshes` (as in `pack.json`), and per +//! expression its name, the meshes it moves and its delta block; +//! - `expressions.bin` — per block: `u32 index[count]` then `f32 delta[count * 3]`. +//! +//! Deltas are in the slider pack's global vertex space (`global = mesh.offset + +//! _CCIDX`), glTF frame (Y-up, meters), at weight 1. One expression covers every +//! mesh that has a morph of that name (body, brows, lashes, tear line, eye +//! occlusion, eyes, tongue), so a mix is `base + Σ wᵢ·Δᵢ` for the whole head. +//! Morph targets are linear, so this is exact; only nonzero weights cost anything. +//! +//! [`ArkitMap`] resolves ARKit-52 weights (Audio2Face) to expression weights. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +use crate::pack::{add_block, block_deltas, block_indices, PackMesh}; + +#[derive(Debug, Clone, Deserialize)] +pub struct Expression { + pub name: String, + /// Meshes this expression moves (glTF mesh names). + pub meshes: Vec, + /// `[byte_offset, count]` into `expressions.bin`. + pub verts: [u64; 2], +} + +#[derive(Debug, Deserialize)] +struct ExprJson { + version: u32, + vertex_count: usize, + meshes: Vec, + expressions: Vec, +} + +pub struct ExpressionPack { + pub vertex_count: usize, + pub meshes: Vec, + pub expressions: Vec, + by_name: HashMap, + bin: Vec, +} + +impl ExpressionPack { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("expressions.json"); + let pack: ExprJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + if pack.version != 1 { + bail!("unsupported expression pack version {}", pack.version); + } + let bin_path = dir.join("expressions.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let by_name = pack.expressions.iter().enumerate().map(|(i, e)| (e.name.clone(), i)).collect(); + let me = Self { vertex_count: pack.vertex_count, meshes: pack.meshes, expressions: pack.expressions, by_name, bin }; + me.validate()?; + Ok(me) + } + + fn validate(&self) -> Result<()> { + if self.by_name.len() != self.expressions.len() { + bail!("duplicate expression names"); + } + for e in &self.expressions { + let [off, n] = e.verts; + if (off + n * 16) as usize > self.bin.len() { + bail!("{}: block past end of expressions.bin", e.name); + } + if let Some(bad) = block_indices(&self.bin, off, n).find(|&i| i as usize >= self.vertex_count) { + bail!("{}: index {bad} out of range {}", e.name, self.vertex_count); + } + } + Ok(()) + } + + /// Number of expressions. + pub fn len(&self) -> usize { + self.expressions.len() + } + + pub fn is_empty(&self) -> bool { + self.expressions.is_empty() + } + + /// Expression names, in pack order (index = position). + pub fn names(&self) -> impl Iterator + '_ { + self.expressions.iter().map(|e| e.name.as_str()) + } + + /// Index of the expression with this name. + pub fn find(&self, name: &str) -> Option { + self.by_name.get(name).copied() + } + + /// Index-weight pairs from `{name: weight}`; unknown names are an error. + pub fn weights_from<'a>(&self, set: impl IntoIterator) -> Result> { + set.into_iter() + .map(|(k, w)| Ok((self.find(k).with_context(|| format!("no expression named {k:?}"))?, w))) + .collect() + } + + /// Sparse delta of expression `i` at weight 1: global vertex indices and deltas. + pub fn deltas(&self, i: usize) -> (Vec, Vec<[f32; 3]>) { + let [off, n] = self.expressions[i].verts; + (block_indices(&self.bin, off, n).collect(), block_deltas(&self.bin, off, n).collect()) + } + + /// `out[v] += Σ wᵢ·Δᵢ[v]` over `(expression index, weight)` pairs; zero weights + /// are skipped. `out` has one entry per global vertex (e.g. a [`ShapeDelta`]'s + /// `verts`, to layer expressions on a shaped character). + /// + /// [`ShapeDelta`]: crate::pack::ShapeDelta + pub fn add_to(&self, weights: &[(usize, f32)], out: &mut [[f32; 3]]) { + assert_eq!(out.len(), self.vertex_count, "one entry per global vertex"); + for &(i, w) in weights { + if w != 0.0 { + add_block(&self.bin, self.expressions[i].verts, w, out); + } + } + } + + /// Per global vertex delta (glTF frame, meters) for `(expression index, weight)` pairs. + pub fn evaluate(&self, weights: &[(usize, f32)]) -> Vec<[f32; 3]> { + let mut out = vec![[0.0; 3]; self.vertex_count]; + self.add_to(weights, &mut out); + out + } +} + +/// ARKit-52 blendshape names -> expression weights, from a runtime ARKit map +/// (`avatars/*.arkit_map*.json`): +/// `{"mapping": {arkit_name: [{"morph": name, "weight": scale, "meshes": [...]}]}}`. +/// Each ARKit weight drives every listed morph, scaled; `meshes` is ignored since +/// an expression already covers every mesh with that morph. +pub struct ArkitMap { + /// ARKit name -> `(expression index, scale)`. + pub mapping: HashMap>, + /// Morph names the map uses that the expression pack does not have. + pub missing: Vec, +} + +#[derive(Deserialize)] +struct ArkitTarget { + morph: String, + #[serde(default = "one")] + weight: f32, +} + +fn one() -> f32 { + 1.0 +} + +impl ArkitMap { + pub fn load(path: &Path, pack: &ExpressionPack) -> Result { + let v: serde_json::Value = serde_json::from_slice( + &std::fs::read(path).with_context(|| format!("reading {}", path.display()))?, + ) + .with_context(|| format!("parsing {}", path.display()))?; + Self::from_json(&v, pack) + } + + pub fn from_json(v: &serde_json::Value, pack: &ExpressionPack) -> Result { + let raw: HashMap> = + serde_json::from_value(v.get("mapping").context("ARKit map has no \"mapping\"")?.clone())?; + let (mut mapping, mut missing) = (HashMap::new(), Vec::new()); + for (arkit, targets) in raw { + let mut resolved = Vec::new(); + for t in targets { + match pack.find(&t.morph) { + Some(i) => resolved.push((i, t.weight)), + None => missing.push(t.morph), + } + } + mapping.insert(arkit, resolved); + } + missing.sort(); + missing.dedup(); + Ok(Self { mapping, missing }) + } + + /// Expression weights for ARKit weights (summed where morphs are shared); + /// unknown ARKit names are an error. + pub fn weights<'a>(&self, arkit: impl IntoIterator) -> Result> { + let mut acc: Vec<(usize, f32)> = Vec::new(); + for (name, w) in arkit { + let targets = self.mapping.get(name).with_context(|| format!("no ARKit blendshape {name:?} in the map"))?; + for &(i, s) in targets { + match acc.iter_mut().find(|(j, _)| *j == i) { + Some(e) => e.1 += w * s, + None => acc.push((i, w * s)), + } + } + } + Ok(acc) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(bin: &mut Vec, idx: &[u32], d: &[[f32; 3]]) -> [u64; 2] { + let off = bin.len() as u64; + for i in idx { + bin.extend_from_slice(&i.to_le_bytes()); + } + for v in d { + for c in v { + bin.extend_from_slice(&c.to_le_bytes()); + } + } + [off, idx.len() as u64] + } + + fn write_pack(dir: &Path) { + let mut bin = Vec::new(); + let blink = block(&mut bin, &[0, 3], &[[0.0, -0.01, 0.0], [0.0, -0.02, 0.0]]); + let jaw = block(&mut bin, &[1, 3], &[[0.0, -0.1, 0.0], [0.0, 0.0, 0.04]]); + let none = block(&mut bin, &[], &[]); + std::fs::write(dir.join("expressions.bin"), &bin).unwrap(); + let json = serde_json::json!({ + "version": 1, "vertex_count": 4, + "meshes": [{"name": "Body", "offset": 0, "count": 3}, {"name": "Lash", "offset": 3, "count": 1}], + "expressions": [ + {"name": "Eye_Blink_L", "meshes": ["Body", "Lash"], "verts": blink}, + {"name": "Jaw_Open", "meshes": ["Body", "Lash"], "verts": jaw}, + {"name": "V_None", "meshes": [], "verts": none} + ] + }); + std::fs::write(dir.join("expressions.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + } + + #[test] + fn evaluates_sparse_weighted_sum() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ExpressionPack::load(dir.path()).unwrap(); + assert_eq!(p.names().collect::>(), ["Eye_Blink_L", "Jaw_Open", "V_None"]); + assert_eq!(p.find("Jaw_Open"), Some(1)); + assert_eq!(p.find("Nope"), None); + assert_eq!(p.evaluate(&[]), vec![[0.0; 3]; 4]); + + let d = p.evaluate(&[(0, 1.0), (1, 0.5), (2, 1.0)]); + assert_eq!(d[0], [0.0, -0.01, 0.0]); + assert_eq!(d[1], [0.0, -0.05, 0.0]); + assert_eq!(d[2], [0.0; 3]); + assert_eq!(d[3], [0.0, -0.02, 0.02]); + + // Layered on an existing (shape) delta. + let mut out = vec![[1.0, 0.0, 0.0]; 4]; + p.add_to(&p.weights_from([("Jaw_Open", 1.0)]).unwrap(), &mut out); + assert_eq!(out[1], [1.0, -0.1, 0.0]); + assert!(p.weights_from([("Nope", 1.0)]).unwrap_err().to_string().contains("no expression")); + assert_eq!(p.deltas(0), (vec![0, 3], vec![[0.0, -0.01, 0.0], [0.0, -0.02, 0.0]])); + } + + #[test] + fn resolves_arkit_weights() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ExpressionPack::load(dir.path()).unwrap(); + let map = ArkitMap::from_json( + &serde_json::json!({"mapping": { + "eyeBlinkLeft": [{"morph": "Eye_Blink_L", "weight": 1.0, "meshes": ["Body"]}], + "jawOpen": [{"morph": "Jaw_Open", "weight": 0.5}, {"morph": "Eye_Blink_L", "weight": 0.25}], + "mouthClose": [{"morph": "Mouth_Close", "weight": 1.0}] + }}), + &p, + ) + .unwrap(); + assert_eq!(map.missing, ["Mouth_Close"]); + let w = map.weights([("eyeBlinkLeft", 0.5), ("jawOpen", 1.0), ("mouthClose", 1.0)]).unwrap(); + assert_eq!(w, vec![(0, 0.75), (1, 0.5)]); + assert!(map.weights([("tongueOut", 1.0)]).is_err()); + } + + #[test] + fn rejects_out_of_range_index() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let mut json: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("expressions.json")).unwrap()).unwrap(); + json["vertex_count"] = 3.into(); + std::fs::write(dir.path().join("expressions.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + let err = ExpressionPack::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/glb.rs b/crates/cc5-shape/src/glb.rs new file mode 100644 index 0000000..ed8bdd6 --- /dev/null +++ b/crates/cc5-shape/src/glb.rs @@ -0,0 +1,464 @@ +//! Write a shaped copy of a GLB: apply a [`ShapeDelta`] to the bind mesh and skeleton. +//! +//! The GLB must carry the `_CCIDX` vertex attribute (CC5 vertex index per glTF +//! vertex, written by `cc5-fbx`) so seam-split glTF vertices map back to +//! the pack's global vertex space. +//! +//! - Vertex positions: `bind + delta`, with the world-space delta taken into each +//! skinned mesh's bind space (`W(joint) · IBM(joint)`). +//! - Joints: every node's world translation moves by its pack delta, or by its +//! parent's when it has none (helpers ride along). Rotations are unchanged, so +//! a node's local translation changes by `parent_linear⁻¹ · (Δnode − Δparent)`. +//! - Inverse bind matrices: `IBM' = W'⁻¹ · W · IBM`, so the new bind mesh renders +//! exactly as written in the bind pose. +//! - Normals and tangents: turned with the surface ([`crate::normals`]); kept +//! exactly where it did not move. + +use std::collections::HashMap; + +use anyhow::{anyhow, bail, Context, Result}; +use glam::{Mat3, Mat4, Quat, Vec3}; +use serde_json::Value; + +use crate::normals; +use crate::pack::{ShapeDelta, ShapePack}; +use crate::subdiv::SubdivMap; + +const GLB_MAGIC: u32 = 0x4654_6C67; +const CHUNK_JSON: u32 = 0x4E4F_534A; +const CHUNK_BIN: u32 = 0x004E_4942; + +struct Glb { + json: Value, + bin: Vec, +} + +fn parse(glb: &[u8]) -> Result { + let u32_at = |o: usize| -> Result { + Ok(u32::from_le_bytes(glb.get(o..o + 4).ok_or_else(|| anyhow!("truncated GLB"))?.try_into()?)) + }; + if u32_at(0)? != GLB_MAGIC { + bail!("not a GLB file"); + } + let (mut json, mut bin, mut off) = (None, None, 12usize); + while off + 8 <= glb.len() { + let (len, kind) = (u32_at(off)? as usize, u32_at(off + 4)?); + let body = glb.get(off + 8..off + 8 + len).ok_or_else(|| anyhow!("truncated GLB chunk"))?; + match kind { + CHUNK_JSON => json = Some(serde_json::from_slice(body)?), + CHUNK_BIN => bin = Some(body.to_vec()), + _ => {} + } + off += 8 + len; + } + Ok(Glb { json: json.context("GLB has no JSON chunk")?, bin: bin.context("GLB has no BIN chunk")? }) +} + +fn write(glb: &Glb) -> Result> { + let mut json = serde_json::to_vec(&glb.json)?; + while json.len() % 4 != 0 { + json.push(b' '); + } + let mut bin = glb.bin.clone(); + while bin.len() % 4 != 0 { + bin.push(0); + } + let total = 12 + 8 + json.len() + 8 + bin.len(); + let mut out = Vec::with_capacity(total); + for v in [GLB_MAGIC, 2, total as u32, json.len() as u32, CHUNK_JSON] { + out.extend_from_slice(&v.to_le_bytes()); + } + out.extend_from_slice(&json); + out.extend_from_slice(&(bin.len() as u32).to_le_bytes()); + out.extend_from_slice(&CHUNK_BIN.to_le_bytes()); + out.extend_from_slice(&bin); + Ok(out) +} + +/// Byte offset and stride of each element of a float accessor with `comps` components. +fn accessor_layout(json: &Value, acc: usize, comps: usize) -> Result<(usize, usize, usize)> { + let a = &json["accessors"][acc]; + if a["componentType"].as_u64() != Some(5126) { + bail!("accessor {acc} is not float"); + } + let bv = &json["bufferViews"][a["bufferView"].as_u64().context("sparse accessors unsupported")? as usize]; + let start = bv["byteOffset"].as_u64().unwrap_or(0) as usize + a["byteOffset"].as_u64().unwrap_or(0) as usize; + let stride = bv["byteStride"].as_u64().map(|s| s as usize).unwrap_or(4 * comps); + Ok((start, stride, a["count"].as_u64().context("accessor count")? as usize)) +} + +fn read_f32s(g: &Glb, acc: usize) -> Result> { + let (start, stride, count) = accessor_layout(&g.json, acc, N)?; + (0..count) + .map(|i| { + let b = start + i * stride; + let mut v = [0f32; N]; + for (k, x) in v.iter_mut().enumerate() { + let s = g.bin.get(b + 4 * k..b + 4 * k + 4).ok_or_else(|| anyhow!("accessor past BIN"))?; + *x = f32::from_le_bytes(s.try_into()?); + } + Ok(v) + }) + .collect() +} + +fn write_f32s(g: &mut Glb, acc: usize, vals: &[[f32; N]]) -> Result<()> { + let (start, stride, count) = accessor_layout(&g.json, acc, N)?; + if count != vals.len() { + bail!("accessor {acc}: {} values for {count} elements", vals.len()); + } + for (i, v) in vals.iter().enumerate() { + let b = start + i * stride; + for (k, x) in v.iter().enumerate() { + g.bin[b + 4 * k..b + 4 * k + 4].copy_from_slice(&x.to_le_bytes()); + } + } + Ok(()) +} + +fn node_local(node: &Value) -> Mat4 { + if let Some(m) = node["matrix"].as_array() { + let a: Vec = m.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect(); + return Mat4::from_cols_slice(&a); + } + let v = |k: &str, d: &[f32]| -> Vec { + node[k].as_array().map(|a| a.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect()).unwrap_or(d.to_vec()) + }; + let (t, r, s) = (v("translation", &[0.0; 3]), v("rotation", &[0.0, 0.0, 0.0, 1.0]), v("scale", &[1.0; 3])); + Mat4::from_scale_rotation_translation(Vec3::new(s[0], s[1], s[2]), Quat::from_xyzw(r[0], r[1], r[2], r[3]), Vec3::new(t[0], t[1], t[2])) +} + +/// Shaped copy of `glb` for `delta` (from `pack.evaluate`). +pub fn shape_glb(glb: &[u8], pack: &ShapePack, delta: &ShapeDelta) -> Result> { + if delta.verts.len() != pack.vertex_count { + bail!("delta has {} vertices, pack {}", delta.verts.len(), pack.vertex_count); + } + shape(glb, &pack.joints, &|n| pack.mesh_offset(n), delta) +} + +/// Shaped copy of an HD character GLB (CC5 export at `map.level`, `_CCIDX` in +/// CC5's HD vertex order) for an HD delta (`map.expand(&pack.evaluate(w))`). +pub fn shape_glb_hd(glb: &[u8], pack: &ShapePack, map: &SubdivMap, hd_delta: &ShapeDelta) -> Result> { + if hd_delta.verts.len() != map.vertex_count { + bail!("delta has {} vertices, HD map {}", hd_delta.verts.len(), map.vertex_count); + } + shape(glb, &pack.joints, &|n| map.mesh_offset(n), hd_delta) +} + +fn shape(glb: &[u8], joints: &[String], mesh_offset: &dyn Fn(&str) -> Option, delta: &ShapeDelta) -> Result> { + let mut g = parse(glb)?; + let nodes = g.json["nodes"].as_array().cloned().unwrap_or_default(); + let mut parent = vec![None; nodes.len()]; + for (i, n) in nodes.iter().enumerate() { + for c in n["children"].as_array().into_iter().flatten() { + parent[c.as_u64().context("child index")? as usize] = Some(i); + } + } + // Hierarchy order (parents first) and base world matrices. + let mut order: Vec = Vec::with_capacity(nodes.len()); + let mut placed = vec![false; nodes.len()]; + while order.len() < nodes.len() { + let before = order.len(); + for i in 0..nodes.len() { + if !placed[i] && parent[i].map_or(true, |p| placed[p]) { + placed[i] = true; + order.push(i); + } + } + if order.len() == before { + bail!("node hierarchy has a cycle"); + } + } + let local: Vec = nodes.iter().map(node_local).collect(); + let mut world = vec![Mat4::IDENTITY; nodes.len()]; + for &i in &order { + world[i] = parent[i].map_or(local[i], |p| world[p] * local[i]); + } + + // World translation delta per node: its pack delta, else its parent's. + let pack_joint: HashMap<&str, usize> = joints.iter().enumerate().map(|(i, n)| (n.as_str(), i)).collect(); + let mut dworld = vec![Vec3::ZERO; nodes.len()]; + for &i in &order { + let own = nodes[i]["name"].as_str().and_then(|n| pack_joint.get(n)); + dworld[i] = match own { + Some(&j) => Vec3::from(delta.joints[j]), + None => parent[i].map_or(Vec3::ZERO, |p| dworld[p]), + }; + } + for &i in &order { + if dworld[i] == parent[i].map_or(Vec3::ZERO, |p| dworld[p]) { + continue; + } + if nodes[i].get("matrix").is_some() { + bail!("node {i} uses a matrix; only TRS joints are supported"); + } + let plin = parent[i].map_or(Mat3::IDENTITY, |p| Mat3::from_mat4(world[p])); + let d = plin.inverse() * (dworld[i] - parent[i].map_or(Vec3::ZERO, |p| dworld[p])); + let n = &mut g.json["nodes"][i]; + let t = n["translation"].as_array().map(|a| a.iter().map(|x| x.as_f64().unwrap_or(0.0) as f32).collect::>()).unwrap_or(vec![0.0; 3]); + n["translation"] = serde_json::json!([t[0] + d.x, t[1] + d.y, t[2] + d.z]); + } + let world_new = |i: usize| Mat4::from_translation(dworld[i]) * world[i]; + + // Inverse bind matrices; remember each skin's bind-space transform. + let skins = g.json["skins"].as_array().cloned().unwrap_or_default(); + let mut mesh_space = HashMap::new(); + for (s, skin) in skins.iter().enumerate() { + let joints: Vec = skin["joints"].as_array().context("skin joints")?.iter().filter_map(|j| j.as_u64().map(|j| j as usize)).collect(); + let acc = skin["inverseBindMatrices"].as_u64().context("skin without inverseBindMatrices")? as usize; + let ibm: Vec<[f32; 16]> = read_f32s(&g, acc)?; + let base: Vec = ibm.iter().map(Mat4::from_cols_array).collect(); + mesh_space.insert(s, world[joints[0]] * base[0]); + let new: Vec<[f32; 16]> = joints.iter().zip(&base).map(|(&j, b)| (world_new(j).inverse() * world[j] * *b).to_cols_array()).collect(); + write_f32s(&mut g, acc, &new)?; + } + + // Vertex positions. + let mut skin_of_mesh = HashMap::new(); + for n in &nodes { + if let (Some(m), Some(s)) = (n["mesh"].as_u64(), n["skin"].as_u64()) { + skin_of_mesh.insert(m as usize, s as usize); + } + } + struct Prim { + global: Vec, + bind: Vec<[f32; 3]>, + pos: Vec<[f32; 3]>, + tris: Option>, + normal: Option, + tangent: Option, + } + let mut prims = Vec::new(); + let meshes = g.json["meshes"].as_array().cloned().unwrap_or_default(); + for (m, mesh) in meshes.iter().enumerate() { + let name = mesh["name"].as_str().unwrap_or_default(); + let Some(offset) = mesh_offset(name) else { continue }; + let to_mesh = skin_of_mesh.get(&m).and_then(|s| mesh_space.get(s)).map_or(Mat3::IDENTITY, |w| Mat3::from_mat4(w.inverse())); + for (p, prim) in mesh["primitives"].as_array().into_iter().flatten().enumerate() { + let attrs = &prim["attributes"]; + let (Some(pos_acc), Some(idx_acc)) = (attrs["POSITION"].as_u64(), attrs["_CCIDX"].as_u64()) else { + bail!("mesh {name} primitive {p} lacks POSITION or _CCIDX"); + }; + let ccidx: Vec<[f32; 1]> = read_f32s(&g, idx_acc as usize)?; + let bind: Vec<[f32; 3]> = read_f32s(&g, pos_acc as usize)?; + let mut pos = bind.clone(); + let mut global = Vec::with_capacity(pos.len()); + for (v, [ci]) in pos.iter_mut().zip(&ccidx) { + let gi = offset as usize + *ci as usize; + let d = to_mesh * Vec3::from(*delta.verts.get(gi).ok_or_else(|| anyhow!("{name}: _CCIDX {ci} past pack"))?); + *v = (Vec3::from(*v) + d).into(); + global.push(gi as u32); + } + write_f32s(&mut g, pos_acc as usize, &pos)?; + let (lo, hi) = pos.iter().fold((Vec3::splat(f32::MAX), Vec3::splat(f32::MIN)), |(lo, hi), v| (lo.min(Vec3::from(*v)), hi.max(Vec3::from(*v)))); + g.json["accessors"][pos_acc as usize]["min"] = serde_json::json!([lo.x, lo.y, lo.z]); + g.json["accessors"][pos_acc as usize]["max"] = serde_json::json!([hi.x, hi.y, hi.z]); + let tris = match (prim["mode"].as_u64().unwrap_or(4), prim["indices"].as_u64()) { + (4, Some(acc)) => Some(read_indices(&g, acc as usize)?), + (4, None) => Some((0..pos.len() as u32).collect()), + _ => None, // not a triangle list: normals left as they are + }; + let acc = |k: &str| attrs[k].as_u64().map(|a| a as usize); + prims.push(Prim { global, bind, pos, tris, normal: acc("NORMAL"), tangent: acc("TANGENT") }); + } + } + + // Normals (and tangents) turned with the surface; see `normals`. + let surface = |shaped: bool| { + let it = prims.iter().filter_map(|p| Some((&p.global[..], if shaped { &p.pos[..] } else { &p.bind[..] }, &p.tris.as_ref()?[..]))); + normals::welded_normals(delta.verts.len(), it) + }; + let (bind_n, shaped_n) = (surface(false), surface(true)); + for p in prims.iter().filter(|p| p.tris.is_some()) { + let Some(n_acc) = p.normal else { continue }; + let mut n: Vec<[f32; 3]> = read_f32s(&g, n_acc)?; + let mut t: Option> = p.tangent.map(|a| read_f32s(&g, a)).transpose()?; + normals::reorient(&bind_n, &shaped_n, &p.global, &mut n, t.as_deref_mut()); + write_f32s(&mut g, n_acc, &n)?; + if let (Some(a), Some(t)) = (p.tangent, t) { + write_f32s(&mut g, a, &t)?; + } + } + write(&g) +} + +/// Triangle indices (unsigned byte / short / int accessor). +fn read_indices(g: &Glb, acc: usize) -> Result> { + let a = &g.json["accessors"][acc]; + let size = match a["componentType"].as_u64() { + Some(5121) => 1, + Some(5123) => 2, + Some(5125) => 4, + t => bail!("accessor {acc}: index component type {t:?}"), + }; + let bv = &g.json["bufferViews"][a["bufferView"].as_u64().context("sparse accessors unsupported")? as usize]; + let start = bv["byteOffset"].as_u64().unwrap_or(0) as usize + a["byteOffset"].as_u64().unwrap_or(0) as usize; + let count = a["count"].as_u64().context("accessor count")? as usize; + let bytes = g.bin.get(start..start + count * size).ok_or_else(|| anyhow!("accessor {acc} past BIN"))?; + Ok(bytes + .chunks_exact(size) + .map(|c| match size { + 1 => c[0] as u32, + 2 => u16::from_le_bytes([c[0], c[1]]) as u32, + _ => u32::from_le_bytes([c[0], c[1], c[2], c[3]]), + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pack::ShapeDelta; + + /// Two-joint chain (Hip -> Head, Head rotated 90° about Y), one skinned mesh + /// with 3 glTF vertices mapping to 2 CC5 vertices (index 1 seam-split). + fn tiny_glb() -> Vec { + let mut bin = Vec::new(); + let mut push = |vals: &[f32]| { + let off = bin.len(); + for v in vals { + bin.extend_from_slice(&v.to_le_bytes()); + } + off + }; + let pos = push(&[0.0, 1.0, 0.0, 0.0, 1.7, 0.0, 0.0, 1.7, 0.0]); + let cc = push(&[0.0, 1.0, 1.0]); + let hip = Mat4::from_translation(Vec3::new(0.0, 1.0, 0.0)); + let head = hip * Mat4::from_rotation_translation(Quat::from_rotation_y(std::f32::consts::FRAC_PI_2), Vec3::new(0.0, 0.7, 0.0)); + let ibm = push(&[hip.inverse().to_cols_array(), head.inverse().to_cols_array()].concat()); + let json = serde_json::json!({ + "asset": {"version": "2.0"}, + "nodes": [ + {"name": "Hip", "translation": [0.0, 1.0, 0.0], "children": [1]}, + {"name": "Head", "translation": [0.0, 0.7, 0.0], "rotation": [0.0, 0.70710677, 0.0, 0.70710677]}, + {"name": "Body", "mesh": 0, "skin": 0} + ], + "skins": [{"joints": [0, 1], "inverseBindMatrices": 2}], + "meshes": [{"name": "CC_Base_Body", "primitives": [{"attributes": {"POSITION": 0, "_CCIDX": 1}}]}], + "buffers": [{"byteLength": bin.len()}], + "bufferViews": [ + {"buffer": 0, "byteOffset": pos, "byteLength": 36}, + {"buffer": 0, "byteOffset": cc, "byteLength": 12}, + {"buffer": 0, "byteOffset": ibm, "byteLength": 128} + ], + "accessors": [ + {"bufferView": 0, "componentType": 5126, "count": 3, "type": "VEC3"}, + {"bufferView": 1, "componentType": 5126, "count": 3, "type": "SCALAR"}, + {"bufferView": 2, "componentType": 5126, "count": 2, "type": "MAT4"} + ] + }); + write(&Glb { json, bin }).unwrap() + } + + fn pack(dir: &std::path::Path) -> ShapePack { + std::fs::write(dir.join("pack.bin"), []).unwrap(); + let json = serde_json::json!({"version": 1, "vertex_count": 2, + "meshes": [{"name": "CC_Base_Body", "offset": 0, "count": 2}], + "joints": ["Head", "Hip"], "sliders": []}); + std::fs::write(dir.join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + ShapePack::load(dir).unwrap() + } + + #[test] + fn moves_mesh_joints_and_keeps_skin_consistent() { + let dir = tempfile::tempdir().unwrap(); + let pack = pack(dir.path()); + // Hip down 0.1, Head down 0.15 (so the Hip->Head offset shrinks by 0.05); + // vertex 1 (both seam copies) down 0.15. + let delta = ShapeDelta { + verts: vec![[0.0, -0.1, 0.0], [0.0, -0.15, 0.0]], + joints: vec![[0.0, -0.15, 0.0], [0.0, -0.1, 0.0]], + }; + let out = parse(&shape_glb(&tiny_glb(), &pack, &delta).unwrap()).unwrap(); + let t = |i: usize| out.json["nodes"][i]["translation"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap() as f32).collect::>(); + assert!((t(0)[1] - 0.9).abs() < 1e-6, "{:?}", t(0)); + assert!((t(1)[1] - 0.65).abs() < 1e-6, "{:?}", t(1)); + let pos: Vec<[f32; 3]> = read_f32s(&out, 0).unwrap(); + assert!((pos[0][1] - 0.9).abs() < 1e-6 && (pos[1][1] - 1.55).abs() < 1e-6 && (pos[2][1] - 1.55).abs() < 1e-6, "{pos:?}"); + assert!((out.json["accessors"][0]["max"][1].as_f64().unwrap() - 1.55).abs() < 1e-6); + + // Bind renders the written mesh: world(joint) · IBM' is the same for both joints. + let local: Vec = out.json["nodes"].as_array().unwrap().iter().map(node_local).collect(); + let world = [local[0], local[0] * local[1]]; + let ibm: Vec<[f32; 16]> = read_f32s(&out, 2).unwrap(); + let m: Vec = world.iter().zip(&ibm).map(|(w, i)| *w * Mat4::from_cols_array(i)).collect(); + assert!(m[0].abs_diff_eq(m[1], 1e-5) && m[0].abs_diff_eq(Mat4::IDENTITY, 1e-5), "{m:?}"); + } + + /// Unskinned mesh: a unit quad (vertices 0-3, two triangles) and a separate + /// triangle (4-6), all facing +Z, with u16 indices and NORMAL. + fn normals_glb() -> Vec { + let mut bin = Vec::new(); + let pos: [[f32; 3]; 7] = [[0., 0., 0.], [1., 0., 0.], [1., 1., 0.], [0., 1., 0.], [5., 0., 0.], [6., 0., 0.], [5., 1., 0.]]; + for v in pos.iter().flatten().chain([[0f32, 0., 1.]; 7].iter().flatten()).chain((0..7).map(|i| i as f32).collect::>().iter()) { + bin.extend_from_slice(&v.to_le_bytes()); + } + for i in [0u16, 1, 2, 0, 2, 3, 4, 5, 6] { + bin.extend_from_slice(&i.to_le_bytes()); + } + bin.extend_from_slice(&[0, 0]); + let json = serde_json::json!({ + "asset": {"version": "2.0"}, + "nodes": [{"name": "Body", "mesh": 0}], + "meshes": [{"name": "CC_Base_Body", "primitives": [{"attributes": {"POSITION": 0, "NORMAL": 1, "_CCIDX": 2}, "indices": 3}]}], + "buffers": [{"byteLength": bin.len()}], + "bufferViews": [ + {"buffer": 0, "byteOffset": 0, "byteLength": 84}, + {"buffer": 0, "byteOffset": 84, "byteLength": 84}, + {"buffer": 0, "byteOffset": 168, "byteLength": 28}, + {"buffer": 0, "byteOffset": 196, "byteLength": 18} + ], + "accessors": [ + {"bufferView": 0, "componentType": 5126, "count": 7, "type": "VEC3"}, + {"bufferView": 1, "componentType": 5126, "count": 7, "type": "VEC3"}, + {"bufferView": 2, "componentType": 5126, "count": 7, "type": "SCALAR"}, + {"bufferView": 3, "componentType": 5123, "count": 9, "type": "SCALAR"} + ] + }); + write(&Glb { json, bin }).unwrap() + } + + fn normals_pack(dir: &std::path::Path) -> ShapePack { + std::fs::write(dir.join("pack.bin"), []).unwrap(); + let json = serde_json::json!({"version": 1, "vertex_count": 7, + "meshes": [{"name": "CC_Base_Body", "offset": 0, "count": 7}], "joints": [], "sliders": []}); + std::fs::write(dir.join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + ShapePack::load(dir).unwrap() + } + + #[test] + fn normals_follow_the_reshaped_surface() { + let dir = tempfile::tempdir().unwrap(); + let pack = normals_pack(dir.path()); + let src = normals_glb(); + let normals_of = |glb: &[u8]| -> Vec<[f32; 3]> { read_f32s(&parse(glb).unwrap(), 1).unwrap() }; + + let still = ShapeDelta { verts: vec![[0.0; 3]; 7], joints: vec![] }; + assert_eq!(normals_of(&shape_glb(&src, &pack, &still).unwrap()), normals_of(&src)); + + // Lift vertex 2: the quad tilts, the separate triangle does not move. + let mut verts = vec![[0.0; 3]; 7]; + verts[2] = [0.0, 0.0, 0.5]; + let out = shape_glb(&src, &pack, &ShapeDelta { verts, joints: vec![] }).unwrap(); + let n = normals_of(&out); + let pos: Vec<[f32; 3]> = read_f32s(&parse(&out).unwrap(), 0).unwrap(); + let p = |i: usize| Vec3::from(pos[i]); + let (f0, f1) = ((p(1) - p(0)).cross(p(2) - p(0)), (p(2) - p(0)).cross(p(3) - p(0))); + // Authored +Z equals the flat bind's smooth normal, so it becomes the new one. + for (v, want) in [(0, f0 + f1), (1, f0), (2, f0 + f1), (3, f1)] { + assert!(Vec3::from(n[v]).abs_diff_eq(want.normalize(), 1e-6), "vertex {v}: {:?} vs {:?}", n[v], want.normalize()); + } + assert_eq!(&n[4..], &[[0.0, 0.0, 1.0]; 3]); + } + + #[test] + fn rejects_mesh_without_ccidx() { + let dir = tempfile::tempdir().unwrap(); + let pack = pack(dir.path()); + let mut g = parse(&tiny_glb()).unwrap(); + g.json["meshes"][0]["primitives"][0]["attributes"].as_object_mut().unwrap().remove("_CCIDX"); + let delta = ShapeDelta { verts: vec![[0.0; 3]; 2], joints: vec![[0.0; 3]; 2] }; + let err = shape_glb(&write(&g).unwrap(), &pack, &delta).unwrap_err().to_string(); + assert!(err.contains("_CCIDX"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/lib.rs b/crates/cc5-shape/src/lib.rs new file mode 100644 index 0000000..ce59875 --- /dev/null +++ b/crates/cc5-shape/src/lib.rs @@ -0,0 +1,30 @@ +//! Character Creator 5 shaping sliders outside CC5. +//! +//! - [`pack`]: load a slider pack (`pack.json` + `pack.bin`, built from a CC5 +//! capture) and evaluate a slider mix into vertex and joint deltas. +//! - [`glb`]: write those deltas into a character GLB (bind mesh, joints and +//! inverse bind matrices). +//! - [`subdiv`]: drive CC5's HD (subdivided) mesh from the level-0 pack. +//! - [`conform`]: make uncaptured meshes (hair cards, beards from a hair library) +//! follow the body's shape from its anchors (scalp / face). +//! - [`expr`]: facial expressions (CC5's ~400 morph targets, more than Bevy's 64 +//! per mesh) as sparse vertex deltas in the same space, for CPU morphing. +//! - [`normals`]: turn the character's normals with a reshaped surface. +//! +//! With the `python` feature this crate is also the CPython module `cc5_shape`. + +pub mod conform; +pub mod expr; +pub mod glb; +pub mod normals; +pub mod pack; +pub mod subdiv; + +#[cfg(feature = "python")] +mod python; + +pub use conform::{Conform, ConformOptions}; +pub use expr::{ArkitMap, ExpressionPack}; +pub use glb::{shape_glb, shape_glb_hd}; +pub use pack::{ShapeDelta, ShapePack, Slider, StateDeltas}; +pub use subdiv::SubdivMap; diff --git a/crates/cc5-shape/src/normals.rs b/crates/cc5-shape/src/normals.rs new file mode 100644 index 0000000..97b8d7b --- /dev/null +++ b/crates/cc5-shape/src/normals.rs @@ -0,0 +1,122 @@ +//! Vertex normals of a reshaped mesh. +//! +//! A shape delta moves positions only; the character's normals are CC5's, authored +//! for the captured shape. They are carried along with the surface: per vertex, the +//! rotation that takes the bind mesh's smooth normal to the shaped mesh's smooth +//! normal turns the authored normal (and tangent). Smooth normals are area-weighted +//! face normals accumulated per *global* vertex (CC5 vertex index), so seam-split +//! glTF copies of one CC5 vertex get the same rotation and seams stay closed. +//! Where the surface did not move the rotation is the identity and the authored +//! normal is kept bit for bit. A twist of the surface about its own normal is not +//! seen (the arc rotation has none); it does not change the normal. + +use glam::{Quat, Vec3}; + +/// Area-weighted smooth normals (unnormalized) per global vertex of a surface made of +/// several primitives: `(global id per vertex, positions, triangle list indices)`. +/// `count` is the size of the global vertex space. +pub fn welded_normals<'a>( + count: usize, + prims: impl IntoIterator, +) -> Vec { + let mut acc = vec![Vec3::ZERO; count]; + for (global, pos, tris) in prims { + for t in tris.chunks_exact(3) { + let [a, b, c] = [t[0], t[1], t[2]].map(|i| Vec3::from(pos[i as usize])); + let n = (b - a).cross(c - a); // length = twice the triangle's area + for &i in t { + acc[global[i as usize] as usize] += n; + } + } + } + acc +} + +/// Turn each vertex's authored normal (and tangent, `w` kept) by the rotation from its +/// bind smooth normal to its shaped smooth normal (both from [`welded_normals`]). +pub fn reorient( + bind: &[Vec3], + shaped: &[Vec3], + global: &[u32], + normals: &mut [[f32; 3]], + mut tangents: Option<&mut [[f32; 4]]>, +) { + for (v, &g) in global.iter().enumerate() { + let (a, b) = (bind[g as usize], shaped[g as usize]); + if a == b { + continue; + } + let (a, b) = (a.normalize_or_zero(), b.normalize_or_zero()); + if a == Vec3::ZERO || b == Vec3::ZERO { + continue; + } + let q = Quat::from_rotation_arc(a, b); + normals[v] = (q * Vec3::from(normals[v])).normalize_or_zero().into(); + if let Some(t) = tangents.as_deref_mut() { + let r = q * Vec3::new(t[v][0], t[v][1], t[v][2]); + t[v] = [r.x, r.y, r.z, t[v][3]]; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Unit quad in the XY plane as two primitives sharing the diagonal; vertices 1 + /// and 3 seam-split (each primitive has its own copy). + fn quad() -> (Vec<[u32; 3]>, Vec>) { + let p = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 0.0]]; + (vec![[0, 1, 2], [0, 2, 3]], vec![vec![p[0], p[1], p[2]], vec![p[0], p[2], p[3]]]) + } + + fn smooth(pos: &[Vec<[f32; 3]>]) -> Vec { + let (glob, _) = quad(); + welded_normals(4, glob.iter().zip(pos).map(|(g, p)| (&g[..], &p[..], &[0u32, 1, 2][..]))) + } + + #[test] + fn unchanged_surface_keeps_authored_normals_exactly() { + let (glob, pos) = quad(); + let s = smooth(&pos); + let authored = [[0.1f32, 0.2, 0.97], [0.0, 0.0, 1.0], [0.3, 0.0, 0.95]]; + let mut n = authored; + let mut t = [[1.0f32, 0.0, 0.0, -1.0]; 3]; + reorient(&s, &s, &glob[0], &mut n, Some(&mut t)); + assert_eq!(n, authored); + assert_eq!(t, [[1.0, 0.0, 0.0, -1.0]; 3]); + } + + #[test] + fn rotated_surface_turns_normals_and_tangents() { + let (glob, pos) = quad(); + // Rotate the quad 90° about X: +Z normals become -Y. + let r = Quat::from_rotation_x(std::f32::consts::FRAC_PI_2); + let moved: Vec> = pos.iter().map(|p| p.iter().map(|v| (r * Vec3::from(*v)).into()).collect()).collect(); + let (a, b) = (smooth(&pos), smooth(&moved)); + let authored = [0.0f32, 0.6, 0.8]; + let mut n = [authored; 3]; + let mut t = [[1.0f32, 0.0, 0.0, 1.0]; 3]; + reorient(&a, &b, &glob[1], &mut n, Some(&mut t)); + let want = r * Vec3::from(authored); + for v in 0..3 { + assert!(Vec3::from(n[v]).abs_diff_eq(want, 1e-6), "{:?} vs {want:?}", n[v]); + assert!(Vec3::new(t[v][0], t[v][1], t[v][2]).abs_diff_eq(Vec3::X, 1e-6) && t[v][3] == 1.0, "{:?}", t[v]); + } + } + + #[test] + fn seam_copies_share_the_welded_normal() { + let (_, mut pos) = quad(); + // Lift vertex 2 (shared by both primitives) in z: both triangles tilt. + pos[0][2][2] = 0.5; + pos[1][1][2] = 0.5; + let s = smooth(&pos); + // Vertex 0 and 2 belong to both triangles: their normal is the sum of both + // faces, not just the face of one primitive. + let f0 = (Vec3::from(pos[0][1]) - Vec3::from(pos[0][0])).cross(Vec3::from(pos[0][2]) - Vec3::from(pos[0][0])); + let f1 = (Vec3::from(pos[1][1]) - Vec3::from(pos[1][0])).cross(Vec3::from(pos[1][2]) - Vec3::from(pos[1][0])); + assert!(s[0].abs_diff_eq(f0 + f1, 1e-6) && s[2].abs_diff_eq(f0 + f1, 1e-6)); + assert!(s[1].abs_diff_eq(f0, 1e-6) && s[3].abs_diff_eq(f1, 1e-6)); + } +} diff --git a/crates/cc5-shape/src/pack.rs b/crates/cc5-shape/src/pack.rs new file mode 100644 index 0000000..fa56222 --- /dev/null +++ b/crates/cc5-shape/src/pack.rs @@ -0,0 +1,419 @@ +//! CC5 shaping sliders outside CC5: load an editor pack and evaluate a slider mix. +//! +//! The pack is built from a CC5 capture (`cc5_capture_sliders.py`, MODE "bind") +//! by `build_editor_pack.py`: +//! +//! - `pack.json` — slider tree (id, name, category, range, base weight) and, per +//! slider, the captured min/max states as byte ranges into `pack.bin`; +//! - `pack.bin` — per block: `u32 index[count]` then `f32 delta[count * 3]`. +//! +//! Vertex deltas are in one global index space (CC5 bind-pose mesh order, meshes +//! concatenated); a GLB vertex maps to it through its `_CCIDX` attribute: +//! `global = mesh.offset + _CCIDX`. Joint deltas index `joints`. Everything is in +//! the glTF frame (Y-up, meters) and describes the *bind pose*: skeletal stretch +//! (e.g. Character Height) is already baked into the vertex deltas, so applying a +//! mix is `bind + Σ t_i · delta_i` for the mesh and the joints alike. +//! +//! CC5 sliders are linear on each side of their base weight (verified in CC5: a +//! slider at half weight and two sliders combined both reproduce the sum of the +//! captured deltas exactly), so a state captured at `weight` scales by +//! `(w - base) / (weight - base)`. + +use std::collections::HashMap; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct PackMesh { + pub name: String, + pub offset: u32, + pub count: u32, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct PackState { + pub weight: f32, + /// `[byte_offset, count]` into `pack.bin`. + pub verts: [u64; 2], + pub joints: [u64; 2], +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Slider { + pub id: String, + pub name: String, + pub category: String, + pub min: f32, + pub max: f32, + /// Weight the character was captured at (its design value). + pub base: f32, + pub states: HashMap, +} + +#[derive(Debug, Deserialize)] +struct PackJson { + version: u32, + vertex_count: usize, + meshes: Vec, + joints: Vec, + sliders: Vec, +} + +/// One captured slider state, see [`ShapePack::state`]. +#[derive(Debug, Clone, PartialEq)] +pub struct StateDeltas { + pub weight: f32, + pub vert_index: Vec, + pub vert_delta: Vec<[f32; 3]>, + pub joint_index: Vec, + pub joint_delta: Vec<[f32; 3]>, +} + +/// Summed deltas for a slider mix, in the glTF frame (meters). +#[derive(Debug, Clone, PartialEq)] +pub struct ShapeDelta { + /// One entry per global vertex. + pub verts: Vec<[f32; 3]>, + /// One entry per `ShapePack::joints` entry. + pub joints: Vec<[f32; 3]>, +} + +pub struct ShapePack { + pub vertex_count: usize, + pub meshes: Vec, + pub joints: Vec, + pub sliders: Vec, + bin: Vec, +} + +impl ShapePack { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("pack.json"); + let pack: PackJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + if pack.version != 1 { + bail!("unsupported pack version {}", pack.version); + } + let bin_path = dir.join("pack.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let me = Self { + vertex_count: pack.vertex_count, + meshes: pack.meshes, + joints: pack.joints, + sliders: pack.sliders, + bin, + }; + me.validate()?; + Ok(me) + } + + fn validate(&self) -> Result<()> { + for s in &self.sliders { + for (which, st) in &s.states { + for (what, [off, n], limit) in [ + ("verts", st.verts, self.vertex_count), + ("joints", st.joints, self.joints.len()), + ] { + let end = off + n * 16; + if end as usize > self.bin.len() { + bail!("{} {which} {what}: block past end of pack.bin", s.name); + } + if let Some(bad) = self.indices(off, n).find(|&i| i as usize >= limit) { + bail!("{} {which} {what}: index {bad} out of range {limit}", s.name); + } + } + } + } + Ok(()) + } + + /// Global vertex offset of a mesh (by glTF mesh name). + pub fn mesh_offset(&self, name: &str) -> Option { + self.meshes.iter().find(|m| m.name == name).map(|m| m.offset) + } + + /// Base weights: the character exactly as captured. + pub fn base_weights(&self) -> Vec { + self.sliders.iter().map(|s| s.base).collect() + } + + /// Index of the slider with this id, or with this display name if that is unique. + pub fn find(&self, key: &str) -> Result { + if let Some(i) = self.sliders.iter().position(|s| s.id == key) { + return Ok(i); + } + let hits: Vec = (0..self.sliders.len()).filter(|&i| self.sliders[i].name == key).collect(); + match hits.as_slice() { + [i] => Ok(*i), + [] => bail!("no slider named {key:?}"), + _ => bail!( + "{key:?} names {} sliders ({}); use an id", + hits.len(), + hits.iter().map(|&i| self.sliders[i].category.as_str()).collect::>().join(", ") + ), + } + } + + /// Full weight vector: base weights with the given sliders (id or unique name) set. + pub fn weights_from<'a>(&self, set: impl IntoIterator) -> Result> { + let mut w = self.base_weights(); + for (key, value) in set { + let i = self.find(key)?; + let s = &self.sliders[i]; + if !(s.min..=s.max).contains(&value) { + bail!("{key}: {value} outside {}..{}", s.min, s.max); + } + w[i] = value; + } + Ok(w) + } + + /// A recipe for `weights`: the sliders that differ from the captured design, by id. + /// `{"format": "cc5-shape recipe", "version": 1, "sliders": {id: value}}` + pub fn recipe(&self, weights: &[f32]) -> serde_json::Value { + self.recipe_on(&self.base_weights(), weights) + } + + /// A recipe for `weights` made on top of `base` (e.g. a CC5 neutral body): + /// `"base"` holds the sliders where `base` differs from the captured design, + /// `"sliders"` the changes on top of it. Absent when `base` is the captured design. + pub fn recipe_on(&self, base: &[f32], weights: &[f32]) -> serde_json::Value { + let diff = |from: &[f32], to: &[f32]| -> serde_json::Map { + self.sliders + .iter() + .zip(from.iter().zip(to)) + .filter(|(_, (a, b))| (**a - **b).abs() > 1e-6) + .map(|(s, (_, b))| (s.id.clone(), serde_json::json!(b))) + .collect() + }; + let mut r = serde_json::json!({"format": "cc5-shape recipe", "version": 1, "sliders": diff(base, weights)}); + let b = diff(&self.base_weights(), base); + if !b.is_empty() { + r["base"] = serde_json::Value::Object(b); + } + r + } + + /// Weights from a recipe (`recipe()` / `recipe_on()` output: its `"base"`, then its + /// `"sliders"`; or a plain `{id or name: value}` map). Unknown sliders are an + /// error: the recipe was made for a different pack. + pub fn weights_from_recipe(&self, recipe: &serde_json::Value) -> Result> { + let mut set = Vec::new(); + let maps = [recipe.get("base"), Some(recipe.get("sliders").unwrap_or(recipe))]; + for map in maps.into_iter().flatten() { + for (k, v) in map.as_object().context("recipe is not a JSON object")? { + set.push((k.as_str(), v.as_f64().with_context(|| format!("{k}: not a number"))? as f32)); + } + } + self.weights_from(set) + } + + /// Raw captured state `which` ("min" or "max") of slider `i`: its weight, and the + /// sparse vertex and joint deltas at that weight (relative to the base weight). + /// For building solvers on top of the linear model (e.g. fitting to a target). + pub fn state(&self, i: usize, which: &str) -> Option { + let st = self.sliders.get(i)?.states.get(which)?; + let [vo, vn] = st.verts; + let [jo, jn] = st.joints; + Some(StateDeltas { + weight: st.weight, + vert_index: self.indices(vo, vn).collect(), + vert_delta: self.deltas(vo, vn).collect(), + joint_index: self.indices(jo, jn).collect(), + joint_delta: self.deltas(jo, jn).collect(), + }) + } + + fn indices(&self, off: u64, n: u64) -> impl Iterator + '_ { + block_indices(&self.bin, off, n) + } + + fn deltas(&self, off: u64, n: u64) -> impl Iterator + '_ { + block_deltas(&self.bin, off, n) + } + + fn add_block(&self, block: [u64; 2], t: f32, out: &mut [[f32; 3]]) { + add_block(&self.bin, block, t, out); + } + + /// Captured state and scale for slider `i` at weight `w`, if it moves at all. + fn state_for(&self, i: usize, w: f32) -> Option<(&PackState, f32)> { + let s = &self.sliders[i]; + let d = w - s.base; + if d.abs() < 1e-6 { + return None; + } + let st = s.states.get(if d > 0.0 { "max" } else { "min" })?; + let span = st.weight - s.base; + (span.abs() > 1e-9).then(|| (st, d / span)) + } + + /// Sum of deltas for `weights` (one per slider, in slider order). + pub fn evaluate(&self, weights: &[f32]) -> ShapeDelta { + assert_eq!(weights.len(), self.sliders.len(), "one weight per slider"); + let mut out = ShapeDelta { + verts: vec![[0.0; 3]; self.vertex_count], + joints: vec![[0.0; 3]; self.joints.len()], + }; + for (i, &w) in weights.iter().enumerate() { + if let Some((st, t)) = self.state_for(i, w) { + self.add_block(st.verts, t, &mut out.verts); + self.add_block(st.joints, t, &mut out.joints); + } + } + out + } + + /// Sliders grouped by category path, sorted, for building a UI tree. + pub fn categories(&self) -> Vec<(String, Vec)> { + let mut by: HashMap<&str, Vec> = HashMap::new(); + for (i, s) in self.sliders.iter().enumerate() { + by.entry(s.category.as_str()).or_default().push(i); + } + let mut out: Vec<(String, Vec)> = by + .into_iter() + .map(|(c, mut v)| { + v.sort_by(|&a, &b| self.sliders[a].name.cmp(&self.sliders[b].name)); + (c.to_string(), v) + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } +} + +// Sparse delta blocks (`pack.bin`, `expressions.bin`): `u32 index[n]` then +// `f32 delta[n * 3]`, starting at byte `off`. Bounds are checked at load. + +pub(crate) fn block_indices(bin: &[u8], off: u64, n: u64) -> impl Iterator + '_ { + let start = off as usize; + (0..n as usize).map(move |i| { + let b = start + 4 * i; + u32::from_le_bytes(bin[b..b + 4].try_into().unwrap()) + }) +} + +pub(crate) fn block_deltas(bin: &[u8], off: u64, n: u64) -> impl Iterator + '_ { + let start = (off + 4 * n) as usize; + (0..n as usize).map(move |i| { + let b = start + 12 * i; + let f = |k: usize| f32::from_le_bytes(bin[b + 4 * k..b + 4 * k + 4].try_into().unwrap()); + [f(0), f(1), f(2)] + }) +} + +/// `out[index] += t * delta` for one block. +pub(crate) fn add_block(bin: &[u8], [off, n]: [u64; 2], t: f32, out: &mut [[f32; 3]]) { + for (i, d) in block_indices(bin, off, n).zip(block_deltas(bin, off, n)) { + let o = &mut out[i as usize]; + o[0] += t * d[0]; + o[1] += t * d[1]; + o[2] += t * d[2]; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn block(bin: &mut Vec, idx: &[u32], d: &[[f32; 3]]) -> [u64; 2] { + let off = bin.len() as u64; + for i in idx { + bin.extend_from_slice(&i.to_le_bytes()); + } + for v in d { + for c in v { + bin.extend_from_slice(&c.to_le_bytes()); + } + } + [off, idx.len() as u64] + } + + fn write_pack(dir: &Path) { + let mut bin = Vec::new(); + let h_max_v = block(&mut bin, &[0, 2], &[[0.0, -0.1, 0.0], [0.0, -0.2, 0.0]]); + let h_max_j = block(&mut bin, &[1], &[[0.0, -0.1, 0.0]]); + let h_min_v = block(&mut bin, &[2], &[[0.0, 0.4, 0.0]]); + let h_min_j = block(&mut bin, &[], &[]); + let f_max_v = block(&mut bin, &[1], &[[0.05, 0.0, 0.0]]); + let f_max_j = block(&mut bin, &[], &[]); + std::fs::write(dir.join("pack.bin"), &bin).unwrap(); + let json = serde_json::json!({ + "version": 1, "vertex_count": 3, + "meshes": [{"name": "Body", "offset": 0, "count": 3}], + "joints": ["Hip", "Head"], + "sliders": [ + {"id": "h", "name": "Height", "category": "Body/Full", "min": -1.0, "max": 1.0, "base": 0.0, + "states": {"max": {"weight": 1.0, "verts": h_max_v, "joints": h_max_j}, + "min": {"weight": -1.0, "verts": h_min_v, "joints": h_min_j}}}, + {"id": "f", "name": "Fat", "category": "Body/Full", "min": 0.0, "max": 1.0, "base": 0.0, + "states": {"max": {"weight": 1.0, "verts": f_max_v, "joints": f_max_j}}} + ] + }); + std::fs::write(dir.join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + } + + #[test] + fn evaluates_linear_mix_per_side() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + assert_eq!(p.evaluate(&p.base_weights()).verts, vec![[0.0; 3]; 3]); + + let d = p.evaluate(&[0.5, 1.0]); + assert_eq!(d.verts[0], [0.0, -0.05, 0.0]); + assert_eq!(d.verts[1], [0.05, 0.0, 0.0]); + assert_eq!(d.verts[2], [0.0, -0.1, 0.0]); + assert_eq!(d.joints, vec![[0.0; 3], [0.0, -0.05, 0.0]]); + + // Below base uses the min state, scaled by its own span. + let d = p.evaluate(&[-0.25, 0.0]); + assert_eq!(d.verts[2], [0.0, 0.1, 0.0]); + assert_eq!(p.categories(), vec![("Body/Full".to_string(), vec![1, 0])]); + } + + #[test] + fn resolves_weights_by_id_or_unique_name() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + assert_eq!(p.weights_from([("Height", 0.5), ("f", 1.0)]).unwrap(), vec![0.5, 1.0]); + assert!(p.weights_from([("Height", 2.0)]).unwrap_err().to_string().contains("outside")); + assert!(p.find("Nope").unwrap_err().to_string().contains("no slider")); + } + + #[test] + fn recipe_round_trips() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let p = ShapePack::load(dir.path()).unwrap(); + let r = p.recipe(&[0.5, 0.0]); + assert_eq!(r["sliders"], serde_json::json!({"h": 0.5})); + assert_eq!(p.weights_from_recipe(&r).unwrap(), vec![0.5, 0.0]); + assert_eq!(p.weights_from_recipe(&serde_json::json!({"Fat": 1.0})).unwrap(), vec![0.0, 1.0]); + assert!(p.weights_from_recipe(&serde_json::json!({"sliders": {"zz": 1.0}})).is_err()); + // on top of another base: the base is kept apart, and round-trips + let base = vec![0.0, 1.0]; + let r = p.recipe_on(&base, &[0.5, 1.0]); + assert_eq!(r["base"], serde_json::json!({p.sliders[1].id.clone(): 1.0})); + assert_eq!(r["sliders"], serde_json::json!({p.sliders[0].id.clone(): 0.5})); + assert_eq!(p.weights_from_recipe(&r).unwrap(), vec![0.5, 1.0]); + assert!(p.recipe_on(&p.base_weights(), &[0.5, 0.0]).get("base").is_none()); + } + + #[test] + fn rejects_out_of_range_index() { + let dir = tempfile::tempdir().unwrap(); + write_pack(dir.path()); + let mut json: serde_json::Value = + serde_json::from_slice(&std::fs::read(dir.path().join("pack.json")).unwrap()).unwrap(); + json["vertex_count"] = 2.into(); + std::fs::write(dir.path().join("pack.json"), serde_json::to_vec(&json).unwrap()).unwrap(); + let err = ShapePack::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/crates/cc5-shape/src/python.rs b/crates/cc5-shape/src/python.rs new file mode 100644 index 0000000..f5b91ed --- /dev/null +++ b/crates/cc5-shape/src/python.rs @@ -0,0 +1,414 @@ +//! CPython bindings: `import cc5_shape`. +//! +//! ```python +//! import cc5_shape +//! pack = cc5_shape.Pack("path/to/pack") +//! w = pack.weights({"Character Height": 0.5, "Body Fat A": 0.3}) +//! open("shaped.glb", "wb").write(pack.shape_glb(open("character.glb", "rb").read(), w)) +//! verts, joints = pack.evaluate(w) # bytes (f32 xyz per vertex), {joint: (x, y, z)} +//! +//! ex = cc5_shape.Expressions("path/to/expressions") +//! d = ex.evaluate({"Eye_Blink_L": 1.0, "Jaw_Open": 0.3}) # bytes, f32 xyz per vertex +//! arkit = cc5_shape.ArkitMap("beautiful.arkit_map.json", ex) +//! d = ex.evaluate(arkit.weights({"jawOpen": 0.4})) +//! +//! hd = cc5_shape.SubdivMap("path/to/hd_map") # CC5 HD mesh (subdivision 1 or 2) +//! open("shaped_hd.glb", "wb").write(pack.shape_glb(open("character_hd.glb", "rb").read(), w, hd=hd)) +//! ``` +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict}; + +use crate::conform::{Conform, ConformOptions}; +use crate::expr::{ArkitMap, ExpressionPack}; +use crate::pack::ShapePack; +use crate::subdiv::SubdivMap; + +fn err(e: anyhow::Error) -> PyErr { + PyValueError::new_err(format!("{e:#}")) +} + +fn f32_bytes<'py>(py: Python<'py>, v: &[[f32; 3]]) -> Bound<'py, PyBytes> { + let mut bytes = Vec::with_capacity(v.len() * 12); + for p in v { + for c in p { + bytes.extend_from_slice(&c.to_le_bytes()); + } + } + PyBytes::new(py, &bytes) +} + +/// A CC5 slider pack (`pack.json` + `pack.bin`). +#[pyclass(name = "Pack", module = "cc5_shape", frozen)] +struct PyPack { + inner: ShapePack, +} + +impl PyPack { + fn check(&self, weights: &[f32]) -> PyResult<()> { + if weights.len() != self.inner.sliders.len() { + return Err(PyValueError::new_err(format!( + "expected {} weights (one per slider), got {}", + self.inner.sliders.len(), + weights.len() + ))); + } + Ok(()) + } +} + +#[pymethods] +impl PyPack { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: ShapePack::load(&dir).map_err(err)? }) + } + + /// Number of vertices in the pack's global vertex space. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// Joint names, in the order `evaluate` reports them. + #[getter] + fn joints(&self) -> Vec { + self.inner.joints.clone() + } + + /// `[(mesh name, global vertex offset, vertex count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Sliders as dicts: id, name, category, min, max, base (the captured design value). + #[getter] + fn sliders<'py>(&self, py: Python<'py>) -> PyResult>> { + self.inner + .sliders + .iter() + .map(|s| { + let d = PyDict::new(py); + d.set_item("id", &s.id)?; + d.set_item("name", &s.name)?; + d.set_item("category", &s.category)?; + d.set_item("min", s.min)?; + d.set_item("max", s.max)?; + d.set_item("base", s.base)?; + Ok(d) + }) + .collect() + } + + /// Sliders grouped by category: `[(category, [slider index, ...])]`, sorted. + fn categories(&self) -> Vec<(String, Vec)> { + self.inner.categories() + } + + /// Index of a slider by id, or by display name when that is unique. + fn find(&self, key: &str) -> PyResult { + self.inner.find(key).map_err(err) + } + + /// The character as captured: one weight per slider. + fn base_weights(&self) -> Vec { + self.inner.base_weights() + } + + /// Base weights with `{id or name: value}` applied (range-checked). + #[pyo3(signature = (set=None))] + fn weights(&self, set: Option>) -> PyResult> { + let set = set.unwrap_or_default(); + self.inner.weights_from(set.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err) + } + + /// Recipe JSON (str) for a weight vector: only sliders changed from the design, by id. + fn recipe(&self, weights: Vec) -> PyResult { + self.check(&weights)?; + Ok(serde_json::to_string_pretty(&self.inner.recipe(&weights)).unwrap()) + } + + /// Weights from a recipe JSON string (`recipe()` output or a plain {id or name: value}). + fn weights_from_recipe(&self, recipe: &str) -> PyResult> { + let v: serde_json::Value = serde_json::from_str(recipe).map_err(|e| err(e.into()))?; + self.inner.weights_from_recipe(&v).map_err(err) + } + + /// `(vertex deltas, joint deltas)` for a weight vector, in the glTF frame (meters): + /// vertex deltas as little-endian f32 bytes (x, y, z per global vertex; e.g. + /// `numpy.frombuffer(v, "(&self, py: Python<'py>, weights: Vec) -> PyResult<(Bound<'py, PyBytes>, HashMap)> { + self.check(&weights)?; + let d = py.detach(|| self.inner.evaluate(&weights)); + let joints = self + .inner + .joints + .iter() + .zip(&d.joints) + .filter(|(_, j)| j.iter().any(|c| *c != 0.0)) + .map(|(n, j)| (n.clone(), (j[0], j[1], j[2]))) + .collect(); + Ok((f32_bytes(py, &d.verts), joints)) + } + + /// Raw captured state of slider `i` (`which` = "min" or "max"), or None: + /// `(weight, vertex indices u32 bytes, vertex deltas f32 xyz bytes, + /// joint indices u32 bytes, joint deltas f32 xyz bytes)`, glTF frame (m). + fn state<'py>(&self, py: Python<'py>, i: usize, which: &str) -> Option<(f32, Bound<'py, PyBytes>, Bound<'py, PyBytes>, Bound<'py, PyBytes>, Bound<'py, PyBytes>)> { + let s = self.inner.state(i, which)?; + let u = |v: &[u32]| PyBytes::new(py, &v.iter().flat_map(|x| x.to_le_bytes()).collect::>()); + let f = |v: &[[f32; 3]]| PyBytes::new(py, &v.iter().flatten().flat_map(|x| x.to_le_bytes()).collect::>()); + Some((s.weight, u(&s.vert_index), f(&s.vert_delta), u(&s.joint_index), f(&s.joint_delta))) + } + + /// A shaped copy of `glb` (which must carry `_CCIDX`) for a weight vector. + /// With `hd` (a `SubdivMap`), `glb` is the character's CC5 HD export. + #[pyo3(signature = (glb, weights, hd=None))] + fn shape_glb<'py>(&self, py: Python<'py>, glb: &[u8], weights: Vec, hd: Option<&PySubdivMap>) -> PyResult> { + self.check(&weights)?; + if let Some(hd) = hd { + hd.inner.check(&self.inner).map_err(err)?; + } + let out = py.detach(|| { + let d = self.inner.evaluate(&weights); + match hd { + Some(hd) => crate::glb::shape_glb_hd(glb, &self.inner, &hd.inner, &hd.inner.expand(&d)), + None => crate::glb::shape_glb(glb, &self.inner, &d), + } + }); + Ok(PyBytes::new(py, &out.map_err(err)?)) + } + + fn __repr__(&self) -> String { + format!( + "", + self.inner.sliders.len(), + self.inner.vertex_count, + self.inner.joints.len() + ) + } +} + +/// A CC5 expression pack (`expressions.json` + `expressions.bin`): facial morphs +/// as sparse deltas in the slider pack's global vertex space. +#[pyclass(name = "Expressions", module = "cc5_shape", frozen)] +struct PyExpressions { + inner: ExpressionPack, +} + +#[pymethods] +impl PyExpressions { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: ExpressionPack::load(&dir).map_err(err)? }) + } + + /// Number of vertices in the global vertex space. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// `[(mesh name, global vertex offset, vertex count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Expression names, in pack order. + #[getter] + fn names(&self) -> Vec { + self.inner.names().map(str::to_owned).collect() + } + + /// Index of an expression by name. + fn find(&self, name: &str) -> PyResult { + self.inner.find(name).ok_or_else(|| PyValueError::new_err(format!("no expression named {name:?}"))) + } + + /// Meshes expression `name` moves. + fn meshes_of(&self, name: &str) -> PyResult> { + Ok(self.inner.expressions[self.find(name)?].meshes.clone()) + } + + /// Sparse delta of one expression at weight 1: `(u32 indices bytes, f32 xyz bytes)`. + fn deltas<'py>(&self, py: Python<'py>, name: &str) -> PyResult<(Bound<'py, PyBytes>, Bound<'py, PyBytes>)> { + let (idx, d) = self.inner.deltas(self.find(name)?); + let idx: Vec = idx.iter().flat_map(|i| i.to_le_bytes()).collect(); + Ok((PyBytes::new(py, &idx), f32_bytes(py, &d))) + } + + /// Vertex deltas for `{name: weight}` (glTF frame, meters) as little-endian f32 + /// bytes, x, y, z per global vertex. + fn evaluate<'py>(&self, py: Python<'py>, weights: HashMap) -> PyResult> { + let w = self.inner.weights_from(weights.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err)?; + let d = py.detach(|| self.inner.evaluate(&w)); + Ok(f32_bytes(py, &d)) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __repr__(&self) -> String { + format!("", self.inner.len(), self.inner.vertex_count) + } +} + +/// Map from the pack's (level 0) vertex space to CC5's HD mesh (`hd_map.json` + `.bin`). +#[pyclass(name = "SubdivMap", module = "cc5_shape", frozen)] +struct PySubdivMap { + inner: SubdivMap, +} + +#[pymethods] +impl PySubdivMap { + #[new] + fn new(dir: std::path::PathBuf) -> PyResult { + Ok(Self { inner: SubdivMap::load(&dir).map_err(err)? }) + } + + /// CC5 subdivision level (1 or 2). + #[getter] + fn level(&self) -> u32 { + self.inner.level + } + + /// HD global vertex count. + #[getter] + fn vertex_count(&self) -> usize { + self.inner.vertex_count + } + + /// `[(mesh name, HD offset, HD count)]`. + #[getter] + fn meshes(&self) -> Vec<(String, u32, u32)> { + self.inner.meshes.iter().map(|m| (m.name.clone(), m.offset, m.count)).collect() + } + + /// Base-space per-vertex f32 xyz bytes (e.g. `Pack.evaluate`'s vertex deltas) + /// -> HD-space f32 xyz bytes. + fn expand<'py>(&self, py: Python<'py>, verts: &[u8]) -> PyResult> { + if verts.len() != self.inner.base_vertex_count * 12 { + return Err(PyValueError::new_err(format!( + "expected {} bytes ({} base vertices x 3 f32), got {}", + self.inner.base_vertex_count * 12, + self.inner.base_vertex_count, + verts.len() + ))); + } + let v: Vec<[f32; 3]> = verts + .chunks_exact(12) + .map(|c| std::array::from_fn(|k| f32::from_le_bytes(c[4 * k..4 * k + 4].try_into().unwrap()))) + .collect(); + let out = py.detach(|| self.inner.expand_verts(&v)); + Ok(f32_bytes(py, &out)) + } + + fn __repr__(&self) -> String { + format!("", self.inner.level, self.inner.vertex_count, self.inner.base_vertex_count) + } +} + +fn read_f32x3(b: &[u8], what: &str) -> PyResult> { + if b.len() % 12 != 0 { + return Err(PyValueError::new_err(format!("{what}: {} bytes is not a whole number of f32 xyz", b.len()))); + } + Ok(b.chunks_exact(12).map(|c| std::array::from_fn(|k| f32::from_le_bytes(c[4 * k..4 * k + 4].try_into().unwrap()))).collect()) +} + +/// Bind an uncaptured mesh (hair cards, a beard) to body anchor vertices. +/// `body` / `asset`: f32 xyz bytes (bind positions, same space); `anchors`: +/// indices into `body`; `mode`: "scalp", "skin" or "head_box" (see `ConformOptions`). +#[pyclass(name = "Conform", module = "cc5_shape", frozen)] +struct PyConform { + inner: Conform, + body_len: usize, +} + +#[pymethods] +impl PyConform { + #[new] + #[pyo3(signature = (body, anchors, asset, mode="scalp", k=None, sigma_min=None, sigma_scale=None, similarity=None))] + #[allow(clippy::too_many_arguments)] + fn new(py: Python<'_>, body: &[u8], anchors: Vec, asset: &[u8], mode: &str, k: Option, sigma_min: Option, + sigma_scale: Option, similarity: Option) -> PyResult { + let mut o = match mode { + "scalp" => ConformOptions::SCALP, + "skin" => ConformOptions::SKIN, + "head_box" => ConformOptions::HEAD_BOX, + _ => return Err(PyValueError::new_err(format!("mode {mode:?}: expected \"scalp\", \"skin\" or \"head_box\""))), + }; + o.k = k.unwrap_or(o.k); + o.sigma_min = sigma_min.unwrap_or(o.sigma_min); + o.sigma_scale = sigma_scale.unwrap_or(o.sigma_scale); + o.similarity = similarity.unwrap_or(o.similarity); + let (body, asset) = (read_f32x3(body, "body")?, read_f32x3(asset, "asset")?); + let inner = py.detach(|| Conform::bind(&body, &anchors, &asset, o)).map_err(err)?; + Ok(Self { inner, body_len: body.len() }) + } + + /// Body vertex deltas (f32 xyz bytes) -> asset vertex deltas (f32 xyz bytes). + fn apply<'py>(&self, py: Python<'py>, body_delta: &[u8]) -> PyResult> { + let d = read_f32x3(body_delta, "body_delta")?; + if d.len() != self.body_len { + return Err(PyValueError::new_err(format!("body_delta has {} vertices, the body {}", d.len(), self.body_len))); + } + let out = py.detach(|| self.inner.apply(&d)); + Ok(f32_bytes(py, &out)) + } + + fn __len__(&self) -> usize { + self.inner.len() + } +} + +/// ARKit-52 -> expression weights, from a runtime ARKit map JSON. +#[pyclass(name = "ArkitMap", module = "cc5_shape", frozen)] +struct PyArkitMap { + inner: ArkitMap, + names: Vec, +} + +#[pymethods] +impl PyArkitMap { + #[new] + fn new(path: std::path::PathBuf, expressions: &PyExpressions) -> PyResult { + let inner = ArkitMap::load(&path, &expressions.inner).map_err(err)?; + Ok(Self { inner, names: expressions.inner.names().map(str::to_owned).collect() }) + } + + /// ARKit names in the map, sorted. + #[getter] + fn arkit_names(&self) -> Vec { + let mut v: Vec = self.inner.mapping.keys().cloned().collect(); + v.sort(); + v + } + + /// Morph names the map uses that the expression pack lacks. + #[getter] + fn missing(&self) -> Vec { + self.inner.missing.clone() + } + + /// `{expression name: weight}` for `{arkit name: weight}` (for `Expressions.evaluate`). + fn weights(&self, arkit: HashMap) -> PyResult> { + let w = self.inner.weights(arkit.iter().map(|(k, &v)| (k.as_str(), v))).map_err(err)?; + Ok(w.into_iter().map(|(i, v)| (self.names[i].clone(), v)).collect()) + } +} + +#[pymodule] +fn cc5_shape(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/crates/cc5-shape/src/subdiv.rs b/crates/cc5-shape/src/subdiv.rs new file mode 100644 index 0000000..66116c8 --- /dev/null +++ b/crates/cc5-shape/src/subdiv.rs @@ -0,0 +1,212 @@ +//! Drive CC5's HD mesh (subdivision level 1 or 2) from a level-0 slider pack. +//! +//! CC5 HD characters render a Catmull-Clark subdivision of the base mesh plus +//! per-character sculpted detail. Subdivision is linear, so an HD vertex moves +//! by a fixed weighted sum of base-vertex moves: `hd_delta = S · base_delta`. +//! `build_hd_map.py` writes `S` per subdivided mesh, rows in CC5's HD vertex +//! order (= the HD character GLB's `_CCIDX`): +//! +//! - `hd_map.json` — `{"level", "vertex_count", "base_vertex_count", "meshes": +//! [{"name", "offset", "count", "base_offset", "base_count", "stencil": +//! [byte_offset, nnz] | null}]}`; meshes in the pack's order, `offset`s in the +//! HD global vertex space; +//! - `hd_map.bin` — per stencil: `u32 indptr[count + 1]`, `u32 base_index[nnz]` +//! (mesh-local), `f32 weight[nnz]`. +//! +//! Meshes without a stencil (hair, brows, lashes, clothes) map 1:1. The sculpted +//! detail is already in the HD GLB's bind positions. Sliders that only reshape +//! the base mesh are reproduced exactly (p99 ≤ 0.02 mm vs CC5); HD presets and +//! large body sliders also change the HD level directly, which a level-0 pack +//! cannot see (p99 up to ~0.8 mm, max ~3 mm). + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +use crate::pack::{ShapeDelta, ShapePack}; + +#[derive(Debug, Clone, Deserialize)] +pub struct HdMesh { + pub name: String, + pub offset: u32, + pub count: u32, + pub base_offset: u32, + pub base_count: u32, + stencil: Option<[u64; 2]>, +} + +#[derive(Deserialize)] +struct MapJson { + level: u32, + vertex_count: usize, + base_vertex_count: usize, + meshes: Vec, +} + +/// CSR rows = HD vertices of one mesh, columns = its base vertices. +struct Stencil { + indptr: Vec, + indices: Vec, + weights: Vec, +} + +pub struct SubdivMap { + /// CC5 subdivision level of the HD mesh (1 or 2). + pub level: u32, + /// HD global vertex count. + pub vertex_count: usize, + /// The slider pack's (level 0) global vertex count. + pub base_vertex_count: usize, + pub meshes: Vec, + stencils: Vec>, +} + +fn u32s(b: &[u8]) -> Vec { + b.chunks_exact(4).map(|c| u32::from_le_bytes(c.try_into().unwrap())).collect() +} + +impl SubdivMap { + pub fn load(dir: &Path) -> Result { + let json_path = dir.join("hd_map.json"); + let j: MapJson = serde_json::from_slice( + &std::fs::read(&json_path).with_context(|| format!("reading {}", json_path.display()))?, + ) + .with_context(|| format!("parsing {}", json_path.display()))?; + let bin_path = dir.join("hd_map.bin"); + let bin = std::fs::read(&bin_path).with_context(|| format!("reading {}", bin_path.display()))?; + let mut stencils = Vec::with_capacity(j.meshes.len()); + for m in &j.meshes { + let Some([off, nnz]) = m.stencil else { + if m.count != m.base_count { + bail!("{}: {} HD vs {} base vertices but no stencil", m.name, m.count, m.base_count); + } + stencils.push(None); + continue; + }; + let (off, nnz, rows) = (off as usize, nnz as usize, m.count as usize); + let end = off + 4 * (rows + 1) + 8 * nnz; + if end > bin.len() { + bail!("{}: stencil past end of hd_map.bin", m.name); + } + let indptr = u32s(&bin[off..off + 4 * (rows + 1)]); + let i0 = off + 4 * (rows + 1); + let indices = u32s(&bin[i0..i0 + 4 * nnz]); + let weights = bin[i0 + 4 * nnz..end].chunks_exact(4).map(|c| f32::from_le_bytes(c.try_into().unwrap())).collect(); + if indptr.last().copied() != Some(nnz as u32) || indptr.windows(2).any(|w| w[0] > w[1]) { + bail!("{}: malformed stencil row pointers", m.name); + } + if let Some(bad) = indices.iter().find(|&&i| i >= m.base_count) { + bail!("{}: stencil column {bad} out of range {}", m.name, m.base_count); + } + stencils.push(Some(Stencil { indptr, indices, weights })); + } + let end = j.meshes.iter().map(|m| (m.offset + m.count) as usize).max().unwrap_or(0); + if end > j.vertex_count { + bail!("hd_map.json: meshes extend past vertex_count"); + } + Ok(Self { level: j.level, vertex_count: j.vertex_count, base_vertex_count: j.base_vertex_count, meshes: j.meshes, stencils }) + } + + /// Error unless this map was built for `pack`'s vertex space. + pub fn check(&self, pack: &ShapePack) -> Result<()> { + if self.base_vertex_count != pack.vertex_count { + bail!("HD map is for {} base vertices, pack has {}", self.base_vertex_count, pack.vertex_count); + } + for m in &self.meshes { + match pack.meshes.iter().find(|p| p.name == m.name) { + Some(p) if p.offset == m.base_offset && p.count == m.base_count => {} + _ => bail!("HD map mesh {} does not match the pack", m.name), + } + } + Ok(()) + } + + /// HD global vertex offset of a mesh (by glTF mesh name). + pub fn mesh_offset(&self, name: &str) -> Option { + self.meshes.iter().find(|m| m.name == name).map(|m| m.offset) + } + + /// Per-vertex values (deltas) in the base space -> the HD space. + pub fn expand_verts(&self, base: &[[f32; 3]]) -> Vec<[f32; 3]> { + assert_eq!(base.len(), self.base_vertex_count, "one entry per base vertex"); + let mut out = vec![[0.0; 3]; self.vertex_count]; + for (m, st) in self.meshes.iter().zip(&self.stencils) { + let src = &base[m.base_offset as usize..(m.base_offset + m.base_count) as usize]; + let dst = &mut out[m.offset as usize..(m.offset + m.count) as usize]; + match st { + None => dst.copy_from_slice(src), + Some(s) => { + for (r, d) in dst.iter_mut().enumerate() { + let (a, b) = (s.indptr[r] as usize, s.indptr[r + 1] as usize); + let mut acc = [0.0f32; 3]; + for k in a..b { + let (w, v) = (s.weights[k], src[s.indices[k] as usize]); + if v != [0.0; 3] { + acc[0] += w * v[0]; + acc[1] += w * v[1]; + acc[2] += w * v[2]; + } + } + *d = acc; + } + } + } + } + out + } + + /// A base-space [`ShapeDelta`] (from `ShapePack::evaluate`) in the HD space; + /// joints are unchanged. + pub fn expand(&self, d: &ShapeDelta) -> ShapeDelta { + ShapeDelta { verts: self.expand_verts(&d.verts), joints: d.joints.clone() } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Mesh "A" (2 verts) maps 1:1; mesh "B" (2 base verts) subdivides into 3. + fn map(dir: &Path) -> SubdivMap { + let mut bin = Vec::new(); + for v in [0u32, 1, 2, 4] { + bin.extend(v.to_le_bytes()); // indptr: rows {0}, {1}, {0, 1} + } + for v in [0u32, 1, 0, 1] { + bin.extend(v.to_le_bytes()); + } + for w in [1.0f32, 1.0, 0.5, 0.5] { + bin.extend(w.to_le_bytes()); + } + std::fs::write(dir.join("hd_map.bin"), bin).unwrap(); + std::fs::write( + dir.join("hd_map.json"), + r#"{"level": 1, "vertex_count": 5, "base_vertex_count": 4, "meshes": [ + {"name": "A", "offset": 0, "count": 2, "base_offset": 0, "base_count": 2, "stencil": null}, + {"name": "B", "offset": 2, "count": 3, "base_offset": 2, "base_count": 2, "stencil": [0, 4]}]}"#, + ) + .unwrap(); + SubdivMap::load(dir).unwrap() + } + + #[test] + fn expands_identity_and_stencil_meshes() { + let dir = tempfile::tempdir().unwrap(); + let m = map(dir.path()); + let out = m.expand_verts(&[[1.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 4.0, 2.0]]); + assert_eq!(out, vec![[1.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 4.0, 2.0], [0.0, 3.0, 1.0]]); + assert_eq!(m.mesh_offset("B"), Some(2)); + } + + #[test] + fn rejects_out_of_range_columns() { + let dir = tempfile::tempdir().unwrap(); + map(dir.path()); + let mut bin = std::fs::read(dir.path().join("hd_map.bin")).unwrap(); + bin[16..20].copy_from_slice(&7u32.to_le_bytes()); + std::fs::write(dir.path().join("hd_map.bin"), bin).unwrap(); + let err = SubdivMap::load(dir.path()).err().unwrap().to_string(); + assert!(err.contains("out of range"), "{err}"); + } +} diff --git a/crates/cc5-skin/Cargo.toml b/crates/cc5-skin/Cargo.toml new file mode 100644 index 0000000..b473a9e --- /dev/null +++ b/crates/cc5-skin/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "cc5-skin" +version = "0.1.0" +edition = "2021" +publish = false +description = "Character Creator 5 skin customization for cc-render: SkinGen layers and skins captured from CC5, composited in Rust." + +[lib] +name = "cc5_skin" + +[[bin]] +name = "cc5-skin" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +rayon = "1" diff --git a/crates/cc5-skin/src/build.rs b/crates/cc5-skin/src/build.rs new file mode 100644 index 0000000..fe8072e --- /dev/null +++ b/crates/cc5-skin/src/build.rs @@ -0,0 +1,580 @@ +//! `cc5-skin build`: a [`SkinPack`] from a CC5 capture (`cc5_skin_capture.py` with +//! `"export": true`: per variant `tex/_.png` as CC5 exported +//! them, and `character.json`). +//! +//! Layers: the layer on two bases, variant `X` on the character's own skin +//! (`base`) and `X@dark` on the character saved with a contrasting skin (`alt`; +//! CC5's SkinGen bake composites onto its own base skin, so a flat diffuse set +//! in the material never reaches it). Per texel CC5 gives L = blend(B, C, α) on +//! both bases; each [`Blend`] is solved by least squares and the mode with the +//! least error over both bases is kept. Normal: (1−α) = (L1−L2)/(B1−B2) over the +//! channels and C = (L1 − B1·(1−α)) / α; multiply and screen: a per-channel +//! effect (B·M, 1−(1−B)(1−S)), split into the least opacity and a colour. +use crate::{Blend, Layer, LayerMap, LayerPart, Skin, SkinPack}; +use anyhow::{Context, Result}; +use image::RgbaImage; +use rayon::prelude::*; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +pub struct BuildOptions { + pub capture: PathBuf, + pub out: PathBuf, + /// the two clean skin bases layers are solved on (variants), and the suffixes + /// of their layer variants (``) + pub base: String, + pub alt: String, + pub base_suffix: String, + pub alt_suffix: String, + /// layers solved on another second base: id -> (variant, suffix) (a preset + /// over CC5's 15-layer cap on `alt`) + pub alt_for: BTreeMap, + /// held-out check: the layer laid over this image against CC5's `` + pub check: Option<(PathBuf, String)>, + /// the second base as an image (the flat diffuse CC5 got in place of every + /// material's) instead of a variant + pub alt_image: Option, + /// the character's own variant (layers on it validate the solve) + pub character: String, + pub materials: Vec, + /// texels of opacity below this (0..1) are dropped + pub min_alpha: f32, +} + +/// Solve result for one layer on one material. +pub struct Solved { + pub w: u32, + pub h: u32, + /// sRGB colour 0..1 and opacity per texel + pub c: Vec<[f32; 3]>, + pub a: Vec, + /// the main blend mode, and each texel's (parts in other modes) + pub blend: Blend, + pub modes: Vec, + /// how well the solve explains both bases: mean |L − pred| (0..255) + pub err_alt: f32, + /// per texel: that error (0..1), and how much the layer changes it (0 untouched) + pub e: Vec, + pub d: Vec, + /// mean |L1 − B1| over the layer's texels (0..255): how visible it is + pub size: f32, +} + +fn rgb(path: &Path) -> Result<(u32, u32, Vec<[f32; 3]>)> { + let img = image::open(path).with_context(|| format!("{}", path.display()))?.to_rgb8(); + let (w, h) = img.dimensions(); + Ok((w, h, img.pixels().map(|p| [p[0] as f32 / 255.0, p[1] as f32 / 255.0, p[2] as f32 / 255.0]).collect())) +} + +/// The diffuse map CC5 baked (`/_Diffuse.png`) or exported +/// (`/tex/`) for `material` in a variant. +pub fn diffuse_path(capture: &Path, variant: &str, material: &str) -> Option { + let d = capture.join(variant); + [d.join(format!("{material}_Diffuse.png")), d.join("tex").join(format!("{material}_Diffuse.png"))].into_iter().find(|p| p.is_file()) +} + +/// Colour and opacity of one texel of a layer blended by `blend`, from the layer +/// on two bases; None where the layer leaves the texel as it is. +fn solve_texel(blend: Blend, b1: [f32; 3], l1: [f32; 3], b2: [f32; 3], l2: [f32; 3]) -> Option<([f32; 3], f32)> { + let d1 = (0..3).map(|c| (l1[c] - b1[c]).abs()).fold(0.0f32, f32::max); + if d1 < 1.5 / 255.0 { + return None; + } + // least squares per channel of an effect `e` linear in a known function of + // the base: L = B + e·g(B) for multiply (g = B, e = M − 1) and screen + // (g = 1 − B, e = S), fitted over the two bases + let fit = |g: fn(f32) -> f32| { + [0, 1, 2].map(|c| { + let (g1, g2) = (g(b1[c]), g(b2[c])); + ((l1[c] - b1[c]) * g1 + (l2[c] - b2[c]) * g2) / (g1 * g1 + g2 * g2 + 1e-6) + }) + }; + let (col, a) = match blend { + // not solved: captured layers fit normal / multiply / screen (the editor's + // own layers may use these) + Blend::Overlay | Blend::SoftLight => return None, + Blend::Normal => { + // 1 - alpha, regularized towards "fully covered" where the bases agree + // (then only an opaque layer explains a change) + let (mut num, mut den) = (0.0, 0.0); + for c in 0..3 { + let db = b1[c] - b2[c]; + num += (l1[c] - l2[c]) * db; + den += db * db; + } + let keep = (num / (den + 1e-4)).clamp(0.0, 1.0); + let a = 1.0 - keep; + (a >= 1e-3).then(|| ([0, 1, 2].map(|c| ((l1[c] - b1[c] * keep) / a).clamp(0.0, 1.0)), a))? + } + Blend::Multiply => { + // B·(1 − α + α·C): 1 − α + α·C = 1 + e; least opacity that explains it + let e = fit(|b| b).map(|e| e.clamp(-1.0, 0.0)); + let a = -e.iter().copied().fold(0.0f32, f32::min); + (a >= 1e-3).then(|| (e.map(|e| (1.0 + e / a).clamp(0.0, 1.0)), a))? + } + Blend::Screen => { + // 1 − (1−B)(1 − α·C): α·C = e + let e = fit(|b| 1.0 - b).map(|e| e.clamp(0.0, 1.0)); + let a = e.iter().copied().fold(0.0f32, f32::max); + (a >= 1e-3).then(|| (e.map(|e| (e / a).clamp(0.0, 1.0)), a))? + } + }; + Some((col, a)) +} + +/// Solves a layer blended by `blend`: colour, opacity and how well that explains +/// both bases. +pub fn solve(blend: Blend, b1: &[[f32; 3]], l1: &[[f32; 3]], b2: &[[f32; 3]], l2: &[[f32; 3]], w: u32, h: u32) -> Solved { + let n = b1.len(); + let res: Vec<([f32; 3], f32, f32, f32)> = (0..n) + .into_par_iter() + .map(|i| { + let (b1, l1, b2, l2) = (b1[i], l1[i], b2[i], l2[i]); + let d1 = (0..3).map(|c| (l1[c] - b1[c]).abs()).fold(0.0f32, f32::max); + let Some((col, a)) = solve_texel(blend, b1, l1, b2, l2) else { + // untouched (d1 0), or a change no opacity explains (counted) + let err = if d1 < 1.5 / 255.0 { 0.0 } else { (0..3).map(|c| (l1[c] - b1[c]).abs() + (l2[c] - b2[c]).abs()).sum::() / 6.0 }; + return ([0.0; 3], 0.0, err, if d1 < 1.5 / 255.0 { 0.0 } else { d1 }); + }; + let err = (0..3).map(|c| (blend.apply(b1[c], col[c], a) - l1[c]).abs() + (blend.apply(b2[c], col[c], a) - l2[c]).abs()).sum::() / 6.0; + (col, a, err, d1) + }) + .collect(); + let touched: Vec<&([f32; 3], f32, f32, f32)> = res.iter().filter(|r| r.3 > 0.0).collect(); + let m = touched.len().max(1) as f32; + Solved { + w, + h, + blend, + modes: vec![blend; n], + err_alt: touched.iter().map(|r| r.2).sum::() / m * 255.0, + size: touched.iter().map(|r| r.3).sum::() / m * 255.0, + c: res.iter().map(|r| r.0).collect(), + a: res.iter().map(|r| r.1).collect(), + e: res.iter().map(|r| r.2).collect(), + d: res.iter().map(|r| r.3).collect(), + } +} + +/// One solve from fits of the same layer in each blend mode: the best mode +/// overall, and per texel another mode where it explains the texel clearly better +/// (by over a level and at least twice as well): presets made of layers in +/// several modes (full makeup: its lipstick multiplies, the rest is normal). +pub fn merge(fits: Vec) -> Solved { + let k = (0..fits.len()).min_by(|&a, &b| fits[a].err_alt.total_cmp(&fits[b].err_alt)).unwrap(); + let f = &fits[k]; + let mut out = Solved { w: f.w, h: f.h, blend: f.blend, modes: f.modes.clone(), err_alt: f.err_alt, e: f.e.clone(), d: f.d.clone(), size: f.size, c: f.c.clone(), a: f.a.clone() }; + for i in 0..out.a.len() { + if out.d[i] <= 0.0 { + continue; + } + for f in &fits { + if f.a[i] > 0.0 && f.e[i] < out.e[i] - 1.0 / 255.0 && f.e[i] * 2.0 < out.e[i] { + (out.modes[i], out.c[i], out.a[i], out.e[i]) = (f.blend, f.c[i], f.a[i], f.e[i]); + } + } + } + // parts of a few texels are noise: back to the main mode + for f in &fits { + if f.blend != out.blend && out.modes.iter().filter(|&&m| m == f.blend).count() < 500 { + for i in 0..out.a.len() { + if out.modes[i] == f.blend { + (out.modes[i], out.c[i], out.a[i], out.e[i]) = (fits[k].blend, fits[k].c[i], fits[k].a[i], fits[k].e[i]); + } + } + } + } + let touched: Vec = (0..out.d.len()).filter(|&i| out.d[i] > 0.0).collect(); + out.err_alt = touched.iter().map(|&i| out.e[i]).sum::() / touched.len().max(1) as f32 * 255.0; + out +} + +/// Crop to where the layer shows: RGBA8 in the main blend mode, the parts in +/// other modes (same rect), the rect and the opacity-weighted mean colour. +fn crop(s: &Solved, min_alpha: f32) -> Option<(RgbaImage, Vec<(Blend, RgbaImage)>, [u32; 4], [f32; 3])> { + let (mut x0, mut y0, mut x1, mut y1) = (u32::MAX, u32::MAX, 0, 0); + let mut mean = [0.0f64; 3]; + let mut aw = 0.0f64; + for y in 0..s.h { + for x in 0..s.w { + let i = (y * s.w + x) as usize; + let a = s.a[i]; + if a >= min_alpha { + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x + 1); + y1 = y1.max(y + 1); + for c in 0..3 { + mean[c] += (s.c[i][c] * a) as f64; + } + aw += a as f64; + } + } + } + if x1 <= x0 { + return None; + } + // one texel of margin for bilinear sampling + let (x0, y0, x1, y1) = (x0.saturating_sub(1), y0.saturating_sub(1), (x1 + 1).min(s.w), (y1 + 1).min(s.h)); + let image = |mode: Blend| { + RgbaImage::from_fn(x1 - x0, y1 - y0, |x, y| { + let i = ((y + y0) * s.w + x + x0) as usize; + let a = if s.a[i] >= min_alpha && s.modes[i] == mode { s.a[i] } else { 0.0 }; + let c = s.c[i]; + image::Rgba([(c[0] * 255.0).round() as u8, (c[1] * 255.0).round() as u8, (c[2] * 255.0).round() as u8, (a * 255.0).round() as u8]) + }) + }; + let parts = [Blend::Normal, Blend::Multiply, Blend::Screen] + .into_iter() + .filter(|&b| b != s.blend) + .map(|b| (b, image(b))) + .filter(|(_, img)| img.pixels().any(|p| p[3] > 0)) + .collect(); + let mean = [0, 1, 2].map(|c| (mean[c] / aw.max(1e-9) * 255.0) as f32); + Some((image(s.blend), parts, [x0, y0, x1 - x0, y1 - y0], mean)) +} + +/// CC5's shader variables per material from an export's JSON. +fn export_vars(json: &Path, materials: &[String]) -> BTreeMap { + let mut out = BTreeMap::new(); + let Ok(v) = std::fs::read(json).map_err(anyhow::Error::from).and_then(|b| Ok(serde_json::from_slice::(&b)?)) else { return out }; + fn walk(v: &Value, materials: &[String], out: &mut BTreeMap) { + if let Some(o) = v.as_object() { + for (k, x) in o { + if materials.contains(k) { + if let Some(vars) = x.pointer("/Custom Shader/Variable") { + out.insert(k.clone(), vars.clone()); + } + } else { + walk(x, materials, out); + } + } + } + } + walk(&v, materials, &mut out); + out +} + +fn channel_key(c: &str) -> String { + if c == "roughness" { + "MetallicRoughness".into() + } else { + c.into() + } +} + +/// Held-out check: the layer laid over `opt.check`'s image ([`crate::compose`]) +/// against CC5's texture of the layer on that image: (mean abs error, mean +/// change), 0..255, over the layer's rect; None without that capture. +fn validate(opt: &BuildOptions, id: &str, m: &str, li: &crate::LayerImage, w: u32, h: u32) -> Result> { + let Some((pb, suffix)) = &opt.check else { return Ok(None) }; + let Some(pl) = diffuse_path(&opt.capture, &format!("{id}{suffix}"), m) else { return Ok(None) }; + let base = image::open(&pb)?.to_rgba8(); + let cc5 = image::open(&pl)?.to_rgba8(); + if base.dimensions() != (w, h) || cc5.dimensions() != (w, h) { + return Ok(None); + } + let rect = li.rect; + let mut ours = base.as_raw().clone(); + crate::compose(&mut ours, w, h, &[crate::Apply::new(li)]); + let (mut err, mut size, mut n) = (0.0f64, 0.0f64, 0u64); + for y in rect[1]..rect[1] + rect[3] { + for x in rect[0]..rect[0] + rect[2] { + let i = ((y * w + x) * 4) as usize; + for c in 0..3 { + err += (ours[i + c] as f64 - cc5.as_raw()[i + c] as f64).abs(); + size += (base.as_raw()[i + c] as f64 - cc5.as_raw()[i + c] as f64).abs(); + } + n += 3; + } + } + Ok(Some(((err / n.max(1) as f64) as f32, (size / n.max(1) as f64) as f32))) +} + +/// The layer's roughness: CC5's baked roughness of the layer on the first base +/// against that base's, with the opacity solved from the diffuse: +/// R = (L - B·(1-α)) / α. None when the layer leaves roughness alone. +fn layer_roughness(opt: &BuildOptions, id: &str, m: &str, s: &Solved, rect: [u32; 4]) -> Result> { + let path = |v: &str| opt.capture.join(v).join(format!("{m}_Roughness.png")); + let (pb, pl) = (path(&opt.base), path(&format!("{id}{}", opt.base_suffix))); + if !pb.is_file() || !pl.is_file() { + return Ok(None); + } + let b = image::open(&pb)?.to_luma8(); + let l = image::open(&pl)?.to_luma8(); + if b.dimensions() != (s.w, s.h) || l.dimensions() != (s.w, s.h) { + return Ok(None); + } + let mut changed = 0usize; + let img = RgbaImage::from_fn(rect[2], rect[3], |x, y| { + let (x, y) = (x + rect[0], y + rect[1]); + let i = (y * s.w + x) as usize; + let a = s.a[i]; + let (bv, lv) = (b.get_pixel(x, y)[0] as f32, l.get_pixel(x, y)[0] as f32); + if (lv - bv).abs() > 3.0 { + changed += 1; + } + if a < opt.min_alpha { + return image::Rgba([0, 0, 0, 0]); + } + let r = ((lv - bv * (1.0 - a)) / a).clamp(0.0, 255.0).round() as u8; + image::Rgba([r, r, r, (a * 255.0).round() as u8]) + }); + if changed < 64 { + return Ok(None); + } + let file = format!("layers/{id}.{m}.rough.png"); + img.save(opt.out.join(&file))?; + Ok(Some(file)) +} + +/// The layer's normals: CC5's exported normal map of the layer on the character +/// (`/tex/_Normal.png`, with `"export": true`) where it differs +/// from the character's own, opacity ramping in over 2..10 levels of change. +fn layer_normal(opt: &BuildOptions, id: &str, m: &str) -> Result> { + let path = |v: &str| opt.capture.join(v).join("tex").join(format!("{m}_Normal.png")); + let (pb, pl) = (path(&opt.character), path(&format!("{id}{}", opt.base_suffix))); + if !pb.is_file() || !pl.is_file() { + return Ok(None); + } + let b = image::open(&pb)?.to_rgb8(); + let l = image::open(&pl)?.to_rgb8(); + if b.dimensions() != l.dimensions() { + return Ok(None); + } + let (w, h) = b.dimensions(); + let alpha = |x: u32, y: u32| { + let (p, q) = (b.get_pixel(x, y), l.get_pixel(x, y)); + let d = (0..3).map(|c| (p[c] as f32 - q[c] as f32).abs()).fold(0.0, f32::max); + ((d - 2.0) / 8.0).clamp(0.0, 1.0) + }; + let (mut x0, mut y0, mut x1, mut y1, mut n) = (u32::MAX, u32::MAX, 0, 0, 0usize); + for y in 0..h { + for x in 0..w { + if alpha(x, y) > 0.0 { + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x + 1); + y1 = y1.max(y + 1); + n += 1; + } + } + } + if n < 64 { + return Ok(None); + } + let rect = [x0, y0, x1 - x0, y1 - y0]; + let img = RgbaImage::from_fn(rect[2], rect[3], |x, y| { + let (x, y) = (x + x0, y + y0); + let q = l.get_pixel(x, y); + image::Rgba([q[0], q[1], q[2], (alpha(x, y) * 255.0).round() as u8]) + }); + let file = format!("layers/{id}.{m}.normal.png"); + img.save(opt.out.join(&file))?; + println!("{id:24} {m:14} normals: {n} texels, rect {rect:?}"); + Ok(Some((file, rect))) +} + +/// The exported channels a skin swaps (others stay the character's). +const SKIN_CHANNELS: [&str; 8] = ["Diffuse", "Normal", "roughness", "ao", "SSSMap", "TransMap", "MicroNMask", "SpecMask"]; + +/// Builds the pack: `layers` (variant names; `X@male` = X on the alt base) and +/// `skins` (variant names with whole skins), with display names and groups. +pub fn build(opt: &BuildOptions, layers: &[(String, String, String)], skins: &[(String, String)]) -> Result { + std::fs::create_dir_all(opt.out.join("layers"))?; + let mut pack = SkinPack { format: "cc5-skin-pack".into(), version: 1, size: 0, layers: Vec::new(), skins: Vec::new(), dir: opt.out.clone() }; + for (id, name, group) in layers { + let alt_for = opt.alt_for.get(id); + let (alt, alt_suffix) = alt_for.map_or((&opt.alt, &opt.alt_suffix), |(a, s)| (a, s)); + let mut maps = BTreeMap::new(); + let mut color = [0.0f32; 3]; + let mut best = 0.0f32; + for m in &opt.materials { + let paths = [ + diffuse_path(&opt.capture, &opt.base, m), + diffuse_path(&opt.capture, &format!("{id}{}", opt.base_suffix), m), + opt.alt_image.clone().filter(|_| alt_for.is_none()).or_else(|| diffuse_path(&opt.capture, alt, m)), + diffuse_path(&opt.capture, &format!("{id}{alt_suffix}"), m), + ]; + let [Some(pb1), Some(pl1), Some(pb2), Some(pl2)] = paths else { + eprintln!("{id} {m}: missing exports {:?}", paths.iter().map(Option::is_some).collect::>()); + continue; + }; + let (w, h, b1) = rgb(&pb1)?; + let (_, _, l1) = rgb(&pl1)?; + let (_, _, b2) = rgb(&pb2)?; + let (_, _, l2) = rgb(&pl2)?; + anyhow::ensure!([l1.len(), b2.len(), l2.len()].iter().all(|&n| n == b1.len()), "{id} {m}: texture sizes differ"); + // CC5's SkinGen bake composites onto its own base skin: a second base + // the bake ignored gives the same texture twice and no opacity. + let same = l1.iter().zip(&l2).all(|(a, b)| (0..3).all(|c| (a[c] - b[c]).abs() < 1.0 / 255.0)); + anyhow::ensure!(!same, "{id} {m}: the bake on the second base equals the one on the character (the base was ignored)"); + // a preset CC5 did not apply on the second base (e.g. over its layer limit) + let changed = |b: &[[f32; 3]], l: &[[f32; 3]]| b.iter().zip(l).filter(|(b, l)| (0..3).any(|c| (b[c] - l[c]).abs() > 3.0 / 255.0)).count(); + let (n1, n2) = (changed(&b1, &l1), changed(&b2, &l2)); + anyhow::ensure!(n1 < 100 || n2 * 20 > n1, "{id} {m}: changes {n1} texels on the character but {n2} on the second base (not applied there?)"); + pack.size = pack.size.max(w); + // the blend mode that explains both bases best + let s = merge([Blend::Normal, Blend::Multiply, Blend::Screen].map(|b| solve(b, &b1, &l1, &b2, &l2, w, h)).into()); + let Some((img, part_imgs, rect, mean)) = crop(&s, opt.min_alpha) else { continue }; + let file = format!("layers/{id}.{m}.png"); + img.save(opt.out.join(&file))?; + let mut parts = Vec::new(); + for (blend, pi) in &part_imgs { + let f = format!("layers/{id}.{m}.{}.png", format!("{blend:?}").to_lowercase()); + pi.save(opt.out.join(&f))?; + parts.push(LayerPart { diffuse: f, blend: *blend }); + } + let cover = s.a.iter().filter(|&&a| a >= opt.min_alpha).count() as f32; + let li = crate::LayerImage { + rect, + size: w, + w: img.width(), + h: img.height(), + rgba: img.as_raw().clone(), + blend: s.blend, + parts: part_imgs + .iter() + .map(|(b, pi)| crate::LayerImage { rect, size: w, w: pi.width(), h: pi.height(), rgba: pi.as_raw().clone(), blend: *b, parts: Vec::new() }) + .collect(), + }; + let check = validate(opt, id, m, &li, w, h)?; + let mix: String = part_imgs.iter().map(|(b, pi)| format!(" +{} {}", format!("{b:?}").to_lowercase(), pi.pixels().filter(|p| p[3] > 0).count())).collect(); + println!( + "{id:24} {m:14} {:8} rect {rect:?} size {:.1} err {:.2} ({cover} texels{mix}){}", + format!("{:?}", s.blend).to_lowercase(), + s.size, + s.err_alt, + check.map_or(String::new(), |(err, size)| format!("; held-out base: err {err:.2} vs change {size:.2}")) + ); + if cover > best { + best = cover; + color = mean; + } + let roughness = layer_roughness(opt, id, m, &s, rect)?; + let normal = layer_normal(opt, id, m)?; + maps.insert(m.clone(), LayerMap { diffuse: file, rect, blend: s.blend, parts, roughness, normal }); + } + if !maps.is_empty() { + pack.layers.push(Layer { id: id.clone(), name: name.clone(), group: group.clone(), color, maps, source: String::new() }); + } + } + for (id, name) in skins { + let d = opt.capture.join(id); + let dst = opt.out.join("skins").join(id); + std::fs::create_dir_all(&dst)?; + let mut maps: BTreeMap> = BTreeMap::new(); + // baked channels (GetImage names) first, then the export's; each against the character's + let ch_dir = opt.capture.join(&opt.character); + let mut files: Vec<(PathBuf, PathBuf)> = Vec::new(); + for (dir, orig_dir) in [(d.clone(), ch_dir.clone()), (d.join("tex"), ch_dir.join("tex"))] { + let Ok(rd) = std::fs::read_dir(&dir) else { continue }; + for e in rd { + let p = e?.path(); + if p.is_file() { + let f = p.file_name().unwrap().to_owned(); + files.push((p, orig_dir.join(f))); + } + } + } + for (p, orig) in files { + let f = p.file_name().unwrap().to_string_lossy().to_string(); + let Some(m) = opt.materials.iter().find(|m| f.starts_with(&format!("{m}_"))) else { continue }; + let stem = p.file_stem().unwrap().to_string_lossy().to_string(); + let channel = match &stem[m.len() + 1..] { + "Roughness" => "roughness".to_string(), + "AmbientOcclusion" => "ao".to_string(), + c => c.to_string(), + }; + if !SKIN_CHANNELS.contains(&channel.as_str()) || maps.get(m).is_some_and(|mm| mm.contains_key(&channel_key(&channel))) { + continue; + } + // the same as the character's: nothing to swap + if std::fs::read(&orig).ok() == Some(std::fs::read(&p)?) { + continue; + } + // at most the pack's texture size: CC5 exports HD skins' maps at 8192 even + // from their 2K presets (its viewport shows 2K); 8K maps only alias + let fit = |img: image::DynamicImage| -> image::DynamicImage { + let max = pack.size.max(1024); + if img.width() > max || img.height() > max { + img.resize(max, max, image::imageops::FilterType::Lanczos3) + } else { + img + } + }; + if channel == "roughness" { + // as a glTF metallic-roughness map (roughness in G, metal 0) for StandardMaterial + let r = fit(image::open(&p)?).to_luma8(); + let mr = image::RgbImage::from_fn(r.width(), r.height(), |x, y| image::Rgb([0, r.get_pixel(x, y)[0], 0])); + let out = format!("{m}_MR.png"); + mr.save(dst.join(&out))?; + maps.entry(m.clone()).or_default().insert("MetallicRoughness".into(), format!("skins/{id}/{out}")); + continue; + } + // colour and normal maps as JPEG (q 92), the rest as PNG + let big = ["Diffuse", "Normal"].contains(&channel.as_str()); + let out = if big { format!("{m}_{channel}.jpg") } else { format!("{m}_{channel}.png") }; + let img = fit(image::open(&p)?); + if big { + let mut w = std::io::BufWriter::new(std::fs::File::create(dst.join(&out))?); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut w, 92).encode_image(&img.to_rgb8())?; + } else { + img.save(dst.join(&out))?; + } + maps.entry(m.clone()).or_default().insert(channel, format!("skins/{id}/{out}")); + } + let vars = export_vars(&d.join("character.json"), &opt.materials); + println!("skin {id}: {} materials, vars for {}", maps.len(), vars.len()); + pack.skins.push(Skin { id: id.clone(), name: name.clone(), maps, vars, source: String::new() }); + } + pack.save()?; + Ok(pack) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn solve_recovers_colour_and_opacity() { + let (c, a) = ([0.8f32, 0.1, 0.2], 0.6f32); + let b1 = vec![[0.7f32, 0.5, 0.4], [0.3, 0.3, 0.3]]; + let b2 = vec![[0.5f32, 0.35, 0.3], [0.6, 0.5, 0.4]]; + let over = |b: &[f32; 3]| [0, 1, 2].map(|k| b[k] * (1.0 - a) + c[k] * a); + let l1: Vec<[f32; 3]> = b1.iter().map(over).collect(); + let l2: Vec<[f32; 3]> = b2.iter().map(over).collect(); + let s = solve(Blend::Normal, &b1, &l1, &b2, &l2, 2, 1); + for i in 0..2 { + assert!((s.a[i] - a).abs() < 1e-3, "alpha {}", s.a[i]); + for k in 0..3 { + assert!((s.c[i][k] - c[k]).abs() < 1e-3, "colour {:?}", s.c[i]); + } + } + assert!(s.err_alt < 0.5, "alt err {} (0..255)", s.err_alt); + } + + #[test] + fn solve_tells_blend_modes_apart() { + // light and dark skin bases, as the capture's character and dark project + let b1 = vec![[0.75f32, 0.55, 0.45], [0.65, 0.45, 0.35]]; + let b2 = vec![[0.40f32, 0.28, 0.20], [0.30, 0.20, 0.15]]; + for (blend, c, a) in [(Blend::Multiply, [0.2f32, 0.0, 0.1], 0.7f32), (Blend::Screen, [1.0f32, 0.9, 0.8], 0.4), (Blend::Normal, [0.8, 0.1, 0.2], 0.6)] { + let over = |b: &[f32; 3]| [0, 1, 2].map(|k| blend.apply(b[k], c[k], a)); + let l1: Vec<[f32; 3]> = b1.iter().map(over).collect(); + let l2: Vec<[f32; 3]> = b2.iter().map(over).collect(); + let fits = [Blend::Normal, Blend::Multiply, Blend::Screen].map(|b| solve(b, &b1, &l1, &b2, &l2, 2, 1)); + let best = fits.iter().min_by(|x, y| x.err_alt.total_cmp(&y.err_alt)).unwrap(); + assert_eq!(best.blend, blend, "errors {:?}", fits.iter().map(|f| f.err_alt).collect::>()); + assert!(best.err_alt < 0.1, "{blend:?} err {}", best.err_alt); + // the recovered layer reproduces both bases + for i in 0..2 { + for k in 0..3 { + let p = blend.apply(b2[i][k], best.c[i][k], best.a[i]); + assert!((p - l2[i][k]).abs() < 1e-3, "{blend:?} texel {i}: {p} vs {}", l2[i][k]); + } + } + } + } +} diff --git a/crates/cc5-skin/src/generate.rs b/crates/cc5-skin/src/generate.rs new file mode 100644 index 0000000..190c7ec --- /dev/null +++ b/crates/cc5-skin/src/generate.rs @@ -0,0 +1,563 @@ +//! Makeup and skin details generated from the head's region masks (CC5's RLHead +//! maps: `Cheek Fore UpperLip Chin Mask`, `Nose Mouth UpperInnerLid Mask`, +//! `Ear Neck Mask`), any colour, no capture: lipstick, blush, eyeshadow, +//! contour, highlight, freckles, stubble. Each makes a [`LayerImage`] over the +//! masks' UV square, laid over the head's diffuse by [`crate::compose`] like a +//! captured layer. +use crate::LayerImage; +use serde::{Deserialize, Serialize}; + +/// The head's region masks (RGBA8, same size): channel order as CC5's maps. +pub struct Masks { + pub w: u32, + pub h: u32, + /// R cheek, G forehead, B upper lip, A chin + pub cfulc: Vec, + /// R nose, G mouth (lips), B upper lid, A inner lid + pub nmuil: Vec, + /// R neck / jaw, G ear + pub en: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + Lipstick, + Blush, + Eyeshadow, + Contour, + Highlight, + Freckles, + Stubble, + /// one mole / freckle at `Pattern::at` + Spot, +} + +impl Kind { + pub const ALL: [Kind; 8] = [Kind::Lipstick, Kind::Blush, Kind::Eyeshadow, Kind::Contour, Kind::Highlight, Kind::Freckles, Kind::Stubble, Kind::Spot]; + + pub fn name(self) -> &'static str { + match self { + Kind::Lipstick => "Lipstick", + Kind::Blush => "Blush", + Kind::Eyeshadow => "Eyeshadow", + Kind::Contour => "Contour", + Kind::Highlight => "Highlight", + Kind::Freckles => "Freckles", + Kind::Stubble => "Stubble", + Kind::Spot => "Mole / spot", + } + } + + /// A starting colour (sRGB 0..255) and strength: fitted through the shape to + /// the CC5 preset it stands for (`gen_fit.py`; the fitted shapes carry its + /// opacity, so mostly strength 1; contour lower: its shape spreads wider than + /// CC5's, and 0.35 matches CC5's render best). + pub fn default_look(self) -> ([u8; 3], f32) { + match self { + Kind::Lipstick => ([164, 41, 40], 0.95), + Kind::Blush => ([189, 91, 82], 1.0), + Kind::Eyeshadow => ([96, 38, 25], 0.73), + Kind::Contour => ([135, 52, 20], 0.35), + Kind::Highlight => ([220, 162, 140], 1.0), + Kind::Freckles => ([117, 17, 5], 0.15), + Kind::Stubble => ([105, 92, 80], 0.7), + Kind::Spot => ([120, 76, 52], 0.9), + } + } + + /// Its starting pattern: stubble denser than freckles' default. + pub fn default_pattern(self) -> Pattern { + match self { + // CC5's beard_stubble: fine, dense, pale hairs (with the colour + // [105, 92, 80], change error 0.88 -> 0.67, gen_sweep.py) + Kind::Stubble => Pattern { density: 3.0, size: 0.7, ..Pattern::default() }, + _ => Pattern::default(), + } + } + + /// How it blends, as the CC5 preset it is fitted to does. + pub fn default_blend(self) -> crate::Blend { + match self { + Kind::Contour | Kind::Freckles => crate::Blend::Multiply, + Kind::Highlight => crate::Blend::Screen, + _ => crate::Blend::Normal, + } + } + + /// Roughness it lays down (0..1), if any: lipstick's satin finish. + pub fn default_roughness(self) -> Option { + (self == Kind::Lipstick).then_some(150.0 / 255.0) + } +} + +/// Freckle / stubble pattern controls: dot size and density (1 = default) and +/// how much dots vary in size and darkness (0 = all alike, 1 = very uneven); and +/// SkinGen's Range controls of the shape: expansion (grow / shrink), blur +/// (feather) and contrast, -1..1 (0 = as fitted; blur 0..1). +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +pub struct Pattern { + pub size: f32, + pub density: f32, + pub variation: f32, + /// [`Kind::Spot`]: where (UV, 0..1) + #[serde(default = "centre")] + pub at: [f32; 2], + #[serde(default)] + pub expansion: f32, + #[serde(default)] + pub blur: f32, + #[serde(default)] + pub contrast: f32, +} + +fn centre() -> [f32; 2] { + [0.5, 0.5] +} + +impl Default for Pattern { + fn default() -> Self { + Self { size: 1.0, density: 1.0, variation: 0.5, at: centre(), expansion: 0.0, blur: 0.0, contrast: 0.0 } + } +} + +/// What a generator makes: the colour layer and, for lipstick, a roughness layer +/// (R = roughness, A = opacity) for its finish. +pub struct Generated { + pub color: LayerImage, + pub roughness: Option, +} + +/// The head's named regions, in [`Masks`]' channels: (name, which map, channel). +pub const REGIONS: [(&str, usize, usize); 10] = [ + ("cheek", 0, 0), + ("forehead", 0, 1), + ("upper_lip", 0, 2), + ("chin", 0, 3), + ("nose", 1, 0), + ("mouth", 1, 1), + ("upper_lid", 1, 2), + ("inner_lid", 1, 3), + ("neck", 2, 0), + ("ear", 2, 1), +]; + +impl Masks { + /// A named region (see [`REGIONS`]), 0..1 per texel. + pub fn region(&self, name: &str) -> Option> { + let &(_, map, c) = REGIONS.iter().find(|r| r.0 == name)?; + Some(channel([&self.cfulc, &self.nmuil, &self.en][map], c, self.w, self.h)) + } +} + +fn channel(m: &[u8], c: usize, w: u32, h: u32) -> Vec { + (0..(w * h) as usize).map(|i| m[i * 4 + c] as f32 / 255.0).collect() +} + +/// Separable box blur of radius `r`, `passes` times (a soft Gaussian-ish +/// feather); edges clamp. +fn blur(a: &[f32], w: u32, h: u32, r: u32, passes: u32) -> Vec { + let (w, h, r) = (w as usize, h as usize, r as isize); + // one 1-D pass over `n` samples, read at `at(k)` + let pass = |n: usize, at: &dyn Fn(usize) -> f32| -> Vec { + let mut pre = vec![0.0f32; n + 1]; + for k in 0..n { + pre[k + 1] = pre[k] + at(k); + } + let first = at(0); + let last = at(n - 1); + (0..n as isize) + .map(|k| { + let (lo, hi) = (k - r, k + r); + let inside = pre[hi.min(n as isize - 1) as usize + 1] - pre[lo.max(0) as usize]; + let before = (-lo).max(0) as f32 * first; + let after = (hi - (n as isize - 1)).max(0) as f32 * last; + (inside + before + after) / (2 * r + 1) as f32 + }) + .collect() + }; + let mut cur = a.to_vec(); + for _ in 0..passes { + let mut tmp = vec![0.0; w * h]; + for y in 0..h { + let row = pass(w, &|x| cur[y * w + x]); + tmp[y * w..(y + 1) * w].copy_from_slice(&row); + } + for x in 0..w { + let col = pass(h, &|y| tmp[y * w + x]); + for y in 0..h { + cur[y * w + x] = col[y]; + } + } + } + cur +} + +fn hash(x: u32, y: u32, seed: u32) -> f32 { + let mut h = x.wrapping_mul(0x8da6_b343) ^ y.wrapping_mul(0xd816_3841) ^ seed.wrapping_mul(0xcb1a_b31f); + h ^= h >> 13; + h = h.wrapping_mul(0x5bd1_e995); + h ^= h >> 15; + (h & 0x00ff_ffff) as f32 / 16_777_215.0 +} + +/// Speckle: round dots of radius `r` (texels) at density `d` (dots per texel), soft-edged. +fn speckle(w: u32, h: u32, r: f32, d: f32, var: f32, seed: u32) -> Vec { + let cell = (1.0 / d.max(1e-6)).sqrt().max(2.0 * r + 1.0); + let cells = ((w as f32 / cell).ceil() as u32, (h as f32 / cell).ceil() as u32); + let mut out = vec![0.0f32; (w * h) as usize]; + for cy in 0..cells.1 { + for cx in 0..cells.0 { + let (px, py) = ((cx as f32 + hash(cx, cy, seed)) * cell, (cy as f32 + hash(cy, cx, seed ^ 0x9e37)) * cell); + let v = var.clamp(0.0, 1.0); + let rr = r * (1.0 - 0.8 * v + 1.6 * v * hash(cx ^ 0x55, cy, seed)); + let k = 1.0 - v + v * hash(cx, cy ^ 0x77, seed); + let (x0, x1) = ((px - rr - 1.0).max(0.0) as u32, ((px + rr + 1.0) as u32).min(w - 1)); + let (y0, y1) = ((py - rr - 1.0).max(0.0) as u32, ((py + rr + 1.0) as u32).min(h - 1)); + for y in y0..=y1 { + for x in x0..=x1 { + let dd = ((x as f32 - px).powi(2) + (y as f32 - py).powi(2)).sqrt(); + let v = ((rr - dd) / 0.8 + 0.5).clamp(0.0, 1.0) * k; + let o = &mut out[(y * w + x) as usize]; + *o = o.max(v); + } + } + } + } + out +} + +/// `a` moved down by `dy` texels (negative: up), edges zero. +fn shift_y(a: &[f32], w: u32, h: u32, dy: i32) -> Vec { + let (w, h) = (w as usize, h as usize); + let mut out = vec![0.0; w * h]; + for y in 0..h { + let src = y as i64 - dy as i64; + if src >= 0 && (src as usize) < h { + out[y * w..(y + 1) * w].copy_from_slice(&a[src as usize * w..(src as usize + 1) * w]); + } + } + out +} + +/// Each half of `a` moved away from the face's centre line (u = 0.5) by `dx` +/// texels (negative: towards it), edges zero. +fn shift_out(a: &[f32], w: u32, h: u32, dx: i32) -> Vec { + let (w, h) = (w as usize, h as usize); + let half = w / 2; + let mut out = vec![0.0; w * h]; + for y in 0..h { + for x in 0..w { + let (src, ok) = if x < half { + let s = x as i64 + dx as i64; + (s, s >= 0 && (s as usize) < half) + } else { + let s = x as i64 - dx as i64; + (s, (s as usize) >= half && (s as usize) < w && s >= 0) + }; + if ok { + out[y * w + x] = a[y * w + src as usize]; + } + } + } + out +} + +/// A feature of a region mask channel, `.` (radii and moves in +/// texels at 1024): `b3` / `b6` / `b20` blurred; `out` the band around the region; +/// `in` its rim inside; `dy` moved down n (up when negative), `dx` moved +/// away from the face's centre line n (towards it when negative), blurred 10, or +/// with an `s` suffix blurred 4. +fn feature(chs: &[(&str, &[f32])], w: u32, h: u32, name: &str) -> Vec { + // "g,,": a Gaussian blob at (u, v) / 64 of the texture, sigma in + // texels at 1024, mirrored across the face's centre line (u = 0.5) + if let Some(g) = name.strip_prefix('g') { + let p: Vec = g.split(',').map(|x| x.parse().unwrap_or_else(|_| panic!("feature: {name}"))).collect(); + let (u, v, st) = (p[0] / 64.0, p[1] / 64.0, p[2] / 1024.0); + let k = -0.5 / (st * st); + let mut out = vec![0.0f32; (w * h) as usize]; + for y in 0..h { + let dy = (y as f32 + 0.5) / h as f32 - v; + for x in 0..w { + let px = (x as f32 + 0.5) / w as f32; + let mut g = ((px - u).powi(2) + dy * dy) * k; + g = g.exp(); + if p[0] != 32.0 { + g += (((px - (1.0 - u)).powi(2) + dy * dy) * k).exp(); + } + out[(y * w + x) as usize] = g; + } + } + return out; + } + let (ch, op) = name.split_once('.').expect("feature: ."); + let get = |ch: &str| chs.iter().find(|c| c.0 == ch).unwrap_or_else(|| panic!("feature: no channel {ch}")).1; + let s = w as f32 / 1024.0; + let b = |a: &[f32], r: f32| blur(a, w, h, (r * s).round() as u32, 3); + // "a*b.p": the border of two regions, both blurred by r, 1 where each is 0.5 + if let Some((ca, cb)) = ch.split_once('*') { + let r: f32 = op.strip_prefix('p').and_then(|r| r.parse().ok()).unwrap_or_else(|| panic!("feature: {name}")); + return b(get(ca), r).iter().zip(b(get(cb), r)).map(|(x, y)| (4.0 * x * y).clamp(0.0, 1.0)).collect(); + } + let m = get(ch); + match op { + "b3" => b(m, 3.0), + "b6" => b(m, 6.0), + "b20" => b(m, 20.0), + "out" => b(m, 20.0).iter().zip(m).map(|(b, m)| (b - m).clamp(0.0, 1.0)).collect(), + "in" => m.iter().zip(&b(m, 20.0)).map(|(m, b)| (m - b).clamp(0.0, 1.0) * 2.0).collect(), + _ => { + let (op, r) = match op.strip_suffix('s') { + Some(o) => (o, 4.0), + None => (op, 10.0), + }; + let n = |t: &str| (t.parse::().unwrap_or_else(|_| panic!("feature: {name}")) * s).round() as i32; + if let Some(t) = op.strip_prefix("dy") { + b(&shift_y(m, w, h, n(t)), r) + } else if let Some(t) = op.strip_prefix("dx") { + b(&shift_out(m, w, h, n(t)), r) + } else { + panic!("feature: unknown op {name}") + } + } + } +} + +/// A fitted shape: the weighted sum of its features, 0..1. +fn fitted(chs: &[(&str, &[f32])], w: u32, h: u32, shape: &[(&str, f32)]) -> Vec { + let mut out = vec![0.0f32; (w * h) as usize]; + for (name, k) in shape { + for (o, v) in out.iter_mut().zip(feature(chs, w, h, name)) { + *o += k * v; + } + } + out.iter().map(|v| v.clamp(0.0, 1.0)).collect() +} + +/// Shapes fitted to CC5's SkinGen presets (`gen_shape_fit.py`: the CC5 layer's +/// opacity as a weighted sum of [`feature`]s of the region masks; the preset in +/// the comment, soft IoU against it). Per kind the fit that renders closest to +/// CC5 (gen_validate.py change error): contour, highlight and stubble use the +/// mirrored UV blobs ("g,,"), blush and freckles the region-border +/// features ("a*b.p"), eyeshadow the region features alone. +/// blush_round, IoU 0.601 +const SHAPE_BLUSH: &[(&str, f32)] = &[("inner_lid*ear.p24", 27547131013629.965), ("inner_lid*neck.p24", 1843519202873.1365), ("upper_lid*neck.p24", 13787982385.9988), ("cheek*ear.p24", 11.7386), ("cheek.dx24", 0.1947), ("cheek.dy-48", 0.0434), ("cheek.dy-24", 0.002)]; +/// eyeshadow_smoky, IoU 0.773 +const SHAPE_EYESHADOW: &[(&str, f32)] = &[("inner_lid.out", 6.0932), ("upper_lid.out", 5.4479), ("upper_lid.dx24", 5.2087), ("inner_lid.dx-24", 3.4349), ("upper_lid.dy-24", 2.0685), ("inner_lid.dy12s", 1.5726), ("upper_lid.dx-24", 1.4144), ("inner_lid.dy-24", 1.0936), ("upper_lid.dx-12s", 0.9085), ("inner_lid.dx-12s", 0.7209), ("inner_lid.in", 0.0753), ("forehead.out", 0.0692), ("forehead.dy48", 0.0027)]; +/// contour_basic, IoU 0.876 +const SHAPE_CONTOUR: &[(&str, f32)] = &[("inner_lid*ear.p24", 657890830992.3206), ("forehead*ear.p10", 43266030140.677), ("inner_lid*neck.p24", 43237318235.3565), ("upper_lid*neck.p24", 1364292677.0043), ("upper_lid*ear.p24", 21256457.6875), ("nose*neck.p24", 540604.4567), ("forehead*inner_lid.p10", 387.6922), ("forehead*upper_lip.p24", 376.1623), ("forehead*upper_lid.p10", 94.7765), ("nose*upper_lid.p10", 35.1827), ("forehead*ear.p24", 22.521), ("chin*mouth.p10", 13.8989), ("inner_lid.dy-24", 1.1349), ("nose*upper_lid.p24", 1.0591), ("cheek*forehead.p24", 0.8236), ("chin*neck.p10", 0.5563), ("nose*inner_lid.p24", 0.3129), ("upper_lip*nose.p10", 0.22), ("upper_lid.dx24", 0.1763), ("upper_lid.dx-24", 0.13), ("g42,30,24", 0.1259), ("g42,38,24", 0.115), ("g40,32,24", 0.1042), ("g46,26,24", 0.1038), ("g44,36,24", 0.1002), ("g44,22,24", 0.0993), ("g46,24,24", 0.0952), ("g40,40,24", 0.0947), ("g46,30,24", 0.094), ("g46,18,24", 0.0927), ("g44,28,24", 0.0889), ("cheek*nose.p10", 0.0876), ("chin.dy48", 0.0871), ("g44,40,24", 0.0856), ("g34,42,24", 0.082), ("nose.dx12s", 0.0819), ("g46,16,24", 0.0781), ("g42,42,24", 0.0764), ("mouth.dy-48", 0.0755), ("g32,44,24", 0.0746), ("g46,34,24", 0.0726), ("g44,20,24", 0.0716), ("g42,14,24", 0.0705), ("g44,14,24", 0.0693), ("g50,34,24", 0.0655), ("g42,16,24", 0.0623), ("g46,28,24", 0.0612), ("g36,42,24", 0.061), ("g40,14,24", 0.0592), ("g44,32,24", 0.0572), ("g48,32,24", 0.0562), ("g46,38,24", 0.0562), ("g38,40,24", 0.0552), ("g40,12,24", 0.0545), ("g48,36,24", 0.053), ("g38,12,24", 0.0515), ("g40,34,24", 0.0512), ("chin*mouth.p24", 0.0485), ("g50,32,24", 0.0484), ("g48,20,24", 0.0478), ("g42,18,24", 0.0474), ("g38,44,24", 0.0459), ("g38,42,24", 0.0441), ("g44,18,24", 0.0431), ("g42,32,24", 0.042), ("g36,40,24", 0.0403), ("g42,22,24", 0.04), ("g40,42,24", 0.0363), ("g48,38,24", 0.0355), ("g46,32,24", 0.0347), ("g34,40,24", 0.0328), ("g50,36,24", 0.031), ("g34,44,24", 0.031), ("g36,44,24", 0.0301), ("g46,40,24", 0.0295), ("chin.dy-48", 0.029), ("g48,34,24", 0.0262), ("nose.dx24", 0.0256), ("g44,16,24", 0.0254), ("g34,22,24", 0.0234), ("g46,36,24", 0.0224), ("g34,46,24", 0.0211), ("g42,28,24", 0.0195), ("g48,22,24", 0.0194), ("g44,34,24", 0.018), ("g52,34,24", 0.0172), ("ear.out", 0.0171), ("g40,44,24", 0.0163), ("g48,24,24", 0.0137), ("g52,32,24", 0.0133), ("g46,20,24", 0.0132), ("chin.dy-24", 0.013), ("g40,22,24", 0.0129), ("g38,34,24", 0.0106), ("g32,42,24", 0.0084), ("g42,12,24", 0.0044), ("g36,46,24", 0.004)]; +/// highlight_basic, IoU 0.559 +const SHAPE_HIGHLIGHT: &[(&str, f32)] = &[("forehead*mouth.p24", 655421.9837), ("forehead*upper_lid.p10", 0.5964), ("forehead*nose.p10", 0.2796), ("inner_lid.dy-48", 0.1145), ("g34,18,24", 0.0887), ("chin.dy-12s", 0.0813), ("nose.dx-24", 0.0649), ("g32,18,24", 0.0581), ("g38,26,24", 0.0459), ("g38,20,24", 0.0458), ("g32,24,24", 0.0362), ("g36,18,24", 0.032), ("g40,26,24", 0.025), ("upper_lid.dy-24", 0.018), ("g32,16,24", 0.0087), ("chin.dy-24", 0.0082), ("g38,28,24", 0.0079), ("g32,26,24", 0.0051), ("g40,20,24", 0.0045), ("chin.dx-24", 0.0027), ("forehead.out", 0.0023)]; +/// freckle_01, IoU 0.721 +const SHAPE_FRECKLES: &[(&str, f32)] = &[("forehead*mouth.p24", 3090685.8117), ("upper_lid*ear.p24", 1992205.6961), ("cheek*chin.p10", 678118.019), ("upper_lip*chin.p10", 207565.1497), ("forehead*upper_lip.p24", 14794.6705), ("mouth*ear.p24", 3485.3992), ("forehead*neck.p24", 78.7555), ("cheek*inner_lid.p10", 23.8259), ("nose*upper_lid.p10", 22.5836), ("inner_lid.dy48", 8.5088), ("cheek*chin.p24", 3.0278), ("cheek*nose.p24", 1.1413), ("chin.dx-24", 0.33), ("cheek*upper_lip.p24", 0.283), ("chin.dy-24", 0.1828), ("nose.dx-12s", 0.1827), ("mouth.dy48", 0.1606), ("nose.dy-48", 0.1565), ("chin.dx24", 0.1564), ("cheek.dx-24", 0.1147), ("nose.dy-24", 0.0583), ("cheek.dy-48", 0.0532), ("chin.dy12s", 0.0402), ("chin.b3", 0.0361), ("upper_lid.dy-48", 0.0332), ("upper_lip.dx-12s", 0.0318), ("nose.dy12s", 0.0275), ("upper_lip.dy-12s", 0.0213), ("chin.dx12s", 0.0123), ("mouth.dy-24", 0.006), ("chin.in", 0.0021)]; +/// beard_stubble, IoU 0.925 +const SHAPE_STUBBLE: &[(&str, f32)] = &[("inner_lid*ear.p24", 5479545913392.647), ("cheek*chin.p10", 3732604.0488), ("upper_lip*chin.p10", 1373823.9532), ("upper_lip*neck.p24", 78.9638), ("chin*ear.p10", 27.582), ("cheek*neck.p24", 8.4685), ("chin.b20", 1.0691), ("chin*mouth.p24", 0.6941), ("g36,38,24", 0.3557), ("g36,36,24", 0.35), ("g34,40,24", 0.3301), ("g40,38,24", 0.3281), ("g42,36,24", 0.2756), ("g40,34,24", 0.2745), ("g42,32,24", 0.2682), ("g40,32,24", 0.2561), ("g38,36,24", 0.2445), ("g36,42,24", 0.243), ("g44,34,24", 0.2408), ("g38,32,24", 0.2391), ("g38,34,24", 0.2383), ("g38,40,24", 0.2298), ("g38,38,24", 0.2277), ("g44,28,24", 0.2194), ("g44,36,24", 0.2128), ("g42,38,24", 0.2046), ("g34,42,24", 0.2033), ("g32,42,24", 0.1999), ("g44,38,24", 0.1926), ("g36,40,24", 0.1891), ("g44,30,24", 0.1879), ("g46,32,24", 0.1738), ("g34,44,24", 0.1707), ("g42,30,24", 0.1702), ("g42,40,24", 0.1669), ("g34,38,24", 0.1622), ("g46,30,24", 0.1619), ("nose*mouth.p24", 0.161), ("upper_lip.dx24", 0.1544), ("g40,42,24", 0.1497), ("g36,44,24", 0.1467), ("g38,42,24", 0.1437), ("g46,28,24", 0.143), ("g44,32,24", 0.137), ("g36,32,24", 0.1362), ("g32,44,24", 0.136), ("chin.dx24", 0.1311), ("g40,36,24", 0.1279), ("g40,40,24", 0.1227), ("g32,30,24", 0.1219), ("g38,44,24", 0.1196), ("g46,34,24", 0.1132), ("mouth.dy48", 0.1122), ("g32,46,24", 0.1085), ("upper_lip.dy12", 0.106), ("chin.dy48", 0.0996), ("mouth.dy-24", 0.0853), ("g46,36,24", 0.085), ("g34,46,24", 0.0838), ("g32,40,24", 0.0639), ("upper_lip.dx-24", 0.0603), ("upper_lip.dy-12s", 0.0582), ("g46,26,24", 0.0574), ("g44,40,24", 0.0562), ("upper_lip.b6", 0.0457), ("g36,46,24", 0.0408), ("chin.dy-24", 0.0391), ("cheek.dy48", 0.0378), ("g42,42,24", 0.0302), ("g42,34,24", 0.029), ("chin.b3", 0.0287), ("g44,26,24", 0.0262), ("chin.dy-48", 0.0252), ("chin.dy12s", 0.0247), ("chin.dy24", 0.0242), ("mouth.dy-48", 0.0222), ("g38,30,24", 0.0215), ("g46,24,24", 0.021), ("chin.dy12", 0.02), ("chin.dy-12s", 0.0199), ("g38,46,24", 0.0192), ("g40,44,24", 0.0186), ("g34,36,24", 0.015), ("g34,48,24", 0.0143), ("g32,50,24", 0.0117), ("chin.dx-24", 0.01), ("g40,30,24", 0.0087), ("cheek.out", 0.0087), ("g46,38,24", 0.0085), ("upper_lip*nose.p10", 0.007), ("upper_lip.dy-12", 0.0069), ("g34,50,24", 0.0069), ("mouth.out", 0.0062), ("g32,48,24", 0.0061), ("chin.in", 0.0042), ("g44,24,24", 0.0039), ("upper_lip.dx-12s", 0.0037), ("g40,46,24", 0.0037), ("g48,30,24", 0.0036), ("g36,50,24", 0.0036), ("g42,28,24", 0.0035), ("neck.out", 0.0021), ("g44,42,24", 0.0016)]; + +/// Hairs: short soft strokes of half-width `r` and length `len` (texels) at +/// density `d` (per texel), pointing down the texture (hair grows down the face) +/// within ±35°, varying in length and darkness by `var`. +fn strokes(w: u32, h: u32, r: f32, len: f32, d: f32, var: f32, seed: u32) -> Vec { + let cell = (1.0 / d.max(1e-6)).sqrt().max(2.0 * r + 1.0); + let cells = ((w as f32 / cell).ceil() as u32, (h as f32 / cell).ceil() as u32); + let mut out = vec![0.0f32; (w * h) as usize]; + let v = var.clamp(0.0, 1.0); + for cy in 0..cells.1 { + for cx in 0..cells.0 { + let (px, py) = ((cx as f32 + hash(cx, cy, seed)) * cell, (cy as f32 + hash(cy, cx, seed ^ 0x9e37)) * cell); + let l = len * (1.0 - 0.6 * v + 1.2 * v * hash(cx ^ 0x55, cy, seed)); + let k = 1.0 - v + v * hash(cx, cy ^ 0x77, seed); + let a = (hash(cx ^ 0x33, cy ^ 0x11, seed) - 0.5) * 1.22; // ±35° + let (dx, dy) = (a.sin(), a.cos()); + let (qx, qy) = (px + dx * l, py + dy * l); + let (x0, x1) = ((px.min(qx) - r - 1.0).max(0.0) as u32, ((px.max(qx) + r + 1.0) as u32).min(w - 1)); + let (y0, y1) = ((py.min(qy) - r - 1.0).max(0.0) as u32, ((py.max(qy) + r + 1.0) as u32).min(h - 1)); + for y in y0..=y1 { + for x in x0..=x1 { + // distance to the segment + let (ex, ey) = (x as f32 - px, y as f32 - py); + let t = ((ex * dx + ey * dy) / l.max(1e-3)).clamp(0.0, 1.0) * l; + let dd = ((ex - dx * t).powi(2) + (ey - dy * t).powi(2)).sqrt(); + let val = ((r - dd) / 0.8 + 0.5).clamp(0.0, 1.0) * k; + let o = &mut out[(y * w + x) as usize]; + *o = o.max(val); + } + } + } + } + out +} + +/// SkinGen's Range controls on a shape (0..1): blur feathers it (radius up to 24 +/// texels at 1024), expansion grows (> 0) or shrinks (< 0) it (a power curve), +/// contrast hardens (> 0) or softens (< 0) its edge. +fn range(a: Vec, w: u32, h: u32, p: &Pattern) -> Vec { + let s = w as f32 / 1024.0; + let a = if p.blur > 0.0 { blur(&a, w, h, (p.blur.min(1.0) * 24.0 * s).round().max(1.0) as u32, 2) } else { a }; + if p.expansion == 0.0 && p.contrast == 0.0 { + return a; + } + let g = 2f32.powf(-2.0 * p.expansion.clamp(-1.0, 1.0)); + let k = if p.contrast >= 0.0 { 1.0 + 4.0 * p.contrast } else { 1.0 + p.contrast.max(-0.9) }; + a.into_iter() + .map(|v| { + let v = v.clamp(0.0, 1.0).powf(g); + if p.contrast == 0.0 { + v + } else { + ((v - 0.5) * k + 0.5).clamp(0.0, 1.0) + } + }) + .collect() +} + +/// A layer of `rgb` with `layer`'s opacity (and place): e.g. a layer's glow. +pub fn colour_layer(layer: &LayerImage, rgb: [u8; 3]) -> LayerImage { + let rgba = layer.rgba.chunks(4).flat_map(|p| [rgb[0], rgb[1], rgb[2], p[3]]).collect(); + LayerImage { rgba, blend: crate::Blend::Normal, parts: Vec::new(), ..layer.clone() } +} + +/// A roughness layer for `color`'s opacity at `value` (0..1): R = roughness, A = opacity. +/// (Any value: metallic, occlusion as well.) +pub fn roughness_layer(color: &LayerImage, value: f32) -> LayerImage { + let r = (value.clamp(0.0, 1.0) * 255.0).round() as u8; + let rgba = color.rgba.chunks(4).flat_map(|p| [r, r, r, p[3]]).collect(); + LayerImage { rgba, blend: crate::Blend::Normal, parts: Vec::new(), ..color.clone() } +} + +/// Stubble's haze between the hairs (opacity at strength 1, CC5's ~0.1-0.2). +const STUBBLE_HAZE: f32 = 0.3; + +/// The generator `kind` at colour `rgb`; `seed` varies freckles and stubble. +pub fn generate(m: &Masks, kind: Kind, rgb: [u8; 3], seed: u32, pattern: Pattern) -> Generated { + let (psize, pdens, pvar) = (pattern.size.max(0.05), pattern.density.max(0.01), pattern.variation); + let (w, h) = (m.w, m.h); + let s = w as f32 / 1024.0; + let cheek = channel(&m.cfulc, 0, w, h); + let fore = channel(&m.cfulc, 1, w, h); + let upper_lip = channel(&m.cfulc, 2, w, h); + let chin = channel(&m.cfulc, 3, w, h); + let nose = channel(&m.nmuil, 0, w, h); + let mouth = channel(&m.nmuil, 1, w, h); + let lid = channel(&m.nmuil, 2, w, h); + let neck = channel(&m.en, 0, w, h); + let inner_lid = channel(&m.nmuil, 3, w, h); + let ear = channel(&m.en, 1, w, h); + let chs: [(&str, &[f32]); 10] = [ + ("cheek", &cheek), + ("forehead", &fore), + ("upper_lip", &upper_lip), + ("chin", &chin), + ("nose", &nose), + ("mouth", &mouth), + ("upper_lid", &lid), + ("inner_lid", &inner_lid), + ("neck", &neck), + ("ear", &ear), + ]; + // stubble's pale haze under its hairs + let mut haze: Option> = None; + let alpha: Vec = match kind { + Kind::Lipstick => blur(&mouth, w, h, (1.5 * s).max(1.0) as u32, 1), + Kind::Blush => fitted(&chs, w, h, SHAPE_BLUSH), + Kind::Eyeshadow => fitted(&chs, w, h, SHAPE_EYESHADOW), + Kind::Contour => fitted(&chs, w, h, SHAPE_CONTOUR), + Kind::Highlight => fitted(&chs, w, h, SHAPE_HIGHLIGHT), + Kind::Freckles => { + let region = range(fitted(&chs, w, h, SHAPE_FRECKLES), w, h, &pattern); + let dots = speckle(w, h, 1.1 * s * psize, 0.006 * pdens / (s * s), pvar, seed); + region.iter().zip(&dots).map(|(r, d)| r * d).collect() + } + Kind::Spot => { + // a soft round spot, radius 1.2 texels at 1024 (a small mole) x size + let (cx, cy) = (pattern.at[0] * w as f32, pattern.at[1] * h as f32); + let r = 1.2 * s * psize; + (0..(w * h) as usize) + .map(|i| { + let (x, y) = ((i % w as usize) as f32 + 0.5, (i / w as usize) as f32 + 0.5); + let d = ((x - cx).powi(2) + (y - cy).powi(2)).sqrt(); + ((r - d) / (0.6 * s.max(1.0)) + 0.5).clamp(0.0, 1.0) + }) + .collect() + } + Kind::Stubble => { + // as CC5's stubble: a pale haze over the beard area (shaved hair under + // the skin) with the darker hairs on it (its layer: opacity ~0.1 between + // hairs, ~0.45 on them) + let region = range(fitted(&chs, w, h, SHAPE_STUBBLE), w, h, &pattern); + let hairs = strokes(w, h, 0.45 * s * psize, 3.0 * s * psize, 0.03 * pdens / (s * s), pvar, seed); + haze = Some(region.iter().map(|r| r * STUBBLE_HAZE).collect()); + region.iter().zip(&hairs).map(|(r, d)| r * d).collect() + } + }; + + let alpha = if matches!(kind, Kind::Freckles | Kind::Stubble) { alpha } else { range(alpha, w, h, &pattern) }; + let layer = |a: &[f32], c: [u8; 3]| { + let rgba: Vec = a.iter().flat_map(|a| [c[0], c[1], c[2], (a.clamp(0.0, 1.0) * 255.0).round() as u8]).collect(); + LayerImage { rect: [0, 0, w, h], size: w, w, h, rgba, blend: kind.default_blend(), parts: Vec::new() } + }; + let color = match &haze { + // the haze (the hair colour washed towards pale grey-beige), the hairs over it + Some(hz) => { + let pale = [0, 1, 2].map(|c| (rgb[c] as f32 * 0.35 + [176.0, 164.0, 150.0][c] * 0.65).round() as u8); + LayerImage { parts: vec![layer(&alpha, rgb)], ..layer(hz, pale) } + } + None => layer(&alpha, rgb), + }; + // lipstick: a satin finish (rougher than bare lips' wet sheen is CC5's matte look) + let roughness = kind.default_roughness().map(|r| roughness_layer(&color, r)); + Generated { color, roughness } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shifts_move_away_from_the_centre_line_and_down() { + // one texel on each side of an 8 x 2 map's centre line + let mut a = vec![0.0f32; 16]; + (a[3], a[4]) = (1.0, 1.0); + let o = shift_out(&a, 8, 2, 2); + assert_eq!((o[1], o[6], o[3], o[4]), (1.0, 1.0, 0.0, 0.0)); + let d = shift_y(&a, 8, 2, 1); + assert_eq!((d[8 + 3], d[3]), (1.0, 0.0)); + } + + #[test] + fn range_grows_shrinks_and_hardens() { + let a: Vec = (0..16).map(|i| i as f32 / 15.0).collect(); + let p = |e: f32, c: f32| Pattern { expansion: e, contrast: c, ..Pattern::default() }; + let sum = |v: &[f32]| v.iter().sum::(); + assert_eq!(range(a.clone(), 16, 1, &Pattern::default()), a); + assert!(sum(&range(a.clone(), 16, 1, &p(0.5, 0.0))) > sum(&a), "expansion grows"); + assert!(sum(&range(a.clone(), 16, 1, &p(-0.5, 0.0))) < sum(&a), "and shrinks"); + let hard = range(a.clone(), 16, 1, &p(0.0, 1.0)); + assert!(hard[2] < a[2] && hard[13] > a[13], "contrast hardens the edge"); + } + + #[test] + fn lipstick_follows_the_mouth_mask() { + let (w, h) = (64u32, 64u32); + let mut nmuil = vec![0u8; (w * h * 4) as usize]; + for y in 30..34 { + for x in 20..44 { + nmuil[((y * w + x) * 4 + 1) as usize] = 255; + } + } + let m = Masks { w, h, cfulc: vec![0; (w * h * 4) as usize], nmuil, en: vec![0; (w * h * 4) as usize] }; + let g = generate(&m, Kind::Lipstick, [200, 0, 0], 1, Pattern::default()); + let a = |x: u32, y: u32| g.color.rgba[((y * w + x) * 4 + 3) as usize]; + assert!(a(32, 32) > 200 && a(5, 5) == 0); + assert!(g.roughness.is_some()); + let f = generate(&m, Kind::Freckles, [100, 50, 30], 1, Pattern::default()); + let fa = |x: u32, y: u32| f.color.rgba[((y * w + x) * 4 + 3) as usize]; + assert!((0..8).all(|y| (0..8).all(|x| fa(x, y) == 0)), "no freckles far from the regions"); + } + + #[test] + fn blur_keeps_flat_fields_flat() { + let a = vec![0.5f32; 32 * 32]; + let b = blur(&a, 32, 32, 3, 2); + assert!(b.iter().all(|v| (v - 0.5).abs() < 1e-4)); + } +} diff --git a/crates/cc5-skin/src/lib.rs b/crates/cc5-skin/src/lib.rs new file mode 100644 index 0000000..f868d30 --- /dev/null +++ b/crates/cc5-skin/src/lib.rs @@ -0,0 +1,545 @@ +//! Character Creator 5 skin customization for cc-render. +//! +//! CC5 customizes skin with SkinGen: layers (makeup, freckles, moles, stubble, +//! scars, tattoos, ...) composited over a skin base into the materials' textures, +//! and whole skins. Their presets are encrypted, so a [`SkinPack`] holds what CC5 +//! makes of them on one character, captured from CC5 itself +//! (`remotemedia-sdk/scripts/avatars/cc5_editor/skin/cc5_skin_capture.py`, then +//! `cc5-skin build`): +//! +//! - a **layer** per preset and material: colour `C` and opacity `α` per texel +//! (sRGB RGBA PNG, cropped to where it is visible) and a [`Blend`] mode, solved +//! from CC5's textures of two skin bases with and without the layer (the mode +//! that explains both best: normal base·(1−α) + C·α for makeup colour, +//! multiply for tattoos / contour / tans / beards / freckles, screen for +//! highlights). [`compose`] lays layers over any base at a strength (α·s), +//! optionally recoloured (C scaled towards a tint, keeping its pattern); +//! - a **skin**: CC5's whole texture set for a skin preset, and its shader settings. +//! +//! ```json +//! {"format": "cc5-skin-pack", "version": 1, "size": 2048, +//! "layers": [{"id": "lip_rouge", "name": "Rouge", "group": "Lip Makeup", "color": [190, 40, 40], +//! "maps": {"Std_Skin_Head": {"diffuse": "layers/lip_rouge.Std_Skin_Head.png", "rect": [x, y, w, h]}}}], +//! "skins": [{"id": "full_default_male", "name": "Default Male", +//! "maps": {"Std_Skin_Head": {"Diffuse": "skins/full_default_male/Std_Skin_Head_Diffuse.jpg", ...}}, +//! "vars": {"Std_Skin_Head": {...}}}]} +//! ``` +use anyhow::{Context, Result}; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +pub mod build; +pub mod generate; + +/// How a layer's colour `C` at opacity `α` combines with the texel under it `B` +/// (sRGB 0..1), as SkinGen's layer blend modes do. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Blend { + /// B·(1−α) + C·α (makeup colour) + #[default] + Normal, + /// B·(1 − α + α·C) (tattoos, contour, tans, beards, freckles) + Multiply, + /// 1 − (1−B)·(1 − α·C) (highlights) + Screen, + /// overlay (multiply under mid grey, screen over it), at opacity α + Overlay, + /// soft light (the W3C / Photoshop curve), at opacity α + #[serde(rename = "soft_light")] + SoftLight, +} + +impl Blend { + pub fn is_normal(&self) -> bool { + *self == Blend::Normal + } + + /// Every mode, for pickers. + pub const ALL: [Blend; 5] = [Blend::Normal, Blend::Multiply, Blend::Screen, Blend::Overlay, Blend::SoftLight]; + + /// The texel under the layer `b`, with colour `c` at opacity `a` laid over it. + pub fn apply(self, b: f32, c: f32, a: f32) -> f32 { + match self { + Blend::Normal => b * (1.0 - a) + c * a, + Blend::Multiply => b * (1.0 - a + a * c), + Blend::Screen => 1.0 - (1.0 - b) * (1.0 - a * c), + Blend::Overlay => { + let f = if b < 0.5 { 2.0 * b * c } else { 1.0 - 2.0 * (1.0 - b) * (1.0 - c) }; + b + (f - b) * a + } + Blend::SoftLight => { + let f = if c <= 0.5 { + b - (1.0 - 2.0 * c) * b * (1.0 - b) + } else { + let d = if b <= 0.25 { ((16.0 * b - 12.0) * b + 4.0) * b } else { b.sqrt() }; + b + (2.0 * c - 1.0) * (d - b) + }; + b + (f - b) * a + } + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct LayerMap { + /// RGBA PNG (sRGB colour, opacity), relative to the pack + pub diffuse: String, + /// where it sits in the material's texture: x, y, w, h (texels at `size`) + pub rect: [u32; 4], + /// how its colour combines with the skin under it + #[serde(default, skip_serializing_if = "Blend::is_normal")] + pub blend: Blend, + /// parts in other blend modes, laid after it (same rect) + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub parts: Vec, + /// roughness the layer lays down (R) with the same opacity (A), when it + /// changes roughness (lipstick, sweat, ...); same rect + #[serde(default, skip_serializing_if = "Option::is_none")] + pub roughness: Option, + /// normal map texels the layer lays down (RGB, tangent space as CC5's maps) + /// with their opacity (A), and where they sit, when it changes normals + /// (scars, wrinkles, acne) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub normal: Option<(String, [u32; 4])>, +} + +/// Part of a layer map in another blend mode than the map's (RGBA PNG like it). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct LayerPart { + pub diffuse: String, + pub blend: Blend, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Layer { + pub id: String, + pub name: String, + pub group: String, + /// opacity-weighted mean colour (sRGB 0..255): what a tint replaces + pub color: [f32; 3], + pub maps: BTreeMap, + /// the CC5 preset it was captured from + #[serde(default)] + pub source: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Skin { + pub id: String, + pub name: String, + /// material -> CC5 channel / slot -> image relative to the pack + pub maps: BTreeMap>, + /// material -> CC5 shader variables (`extras.cc5.vars` form) + #[serde(default)] + pub vars: BTreeMap, + #[serde(default)] + pub source: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct SkinPack { + pub format: String, + pub version: u32, + /// texture size the layer rects are in + pub size: u32, + pub layers: Vec, + pub skins: Vec, + #[serde(skip)] + pub dir: PathBuf, +} + +impl SkinPack { + pub fn load(dir: &Path) -> Result { + let p = dir.join("skin.json"); + let mut pack: SkinPack = + serde_json::from_slice(&std::fs::read(&p).with_context(|| format!("reading {}", p.display()))?).with_context(|| format!("parsing {}", p.display()))?; + anyhow::ensure!(pack.format == "cc5-skin-pack", "{}: not a cc5-skin-pack", p.display()); + pack.dir = dir.to_path_buf(); + Ok(pack) + } + + pub fn save(&self) -> Result<()> { + std::fs::write(self.dir.join("skin.json"), serde_json::to_vec_pretty(self)?)?; + Ok(()) + } + + pub fn layer(&self, id: &str) -> Option<&Layer> { + self.layers.iter().find(|l| l.id == id) + } + + pub fn skin(&self, id: &str) -> Option<&Skin> { + self.skins.iter().find(|s| s.id == id) + } + + /// A layer's map for `material`, decoded. + pub fn layer_image(&self, layer: &Layer, material: &str) -> Result> { + let Some(m) = layer.maps.get(material) else { return Ok(None) }; + let mut li = self.decode(&m.diffuse, m.rect, m.blend)?; + for p in &m.parts { + li.parts.push(self.decode(&p.diffuse, m.rect, p.blend)?); + } + Ok(Some(li)) + } + + /// A layer's roughness map for `material`, decoded (R = roughness, A = opacity). + pub fn layer_roughness(&self, layer: &Layer, material: &str) -> Result> { + let Some(m) = layer.maps.get(material) else { return Ok(None) }; + let Some(r) = &m.roughness else { return Ok(None) }; + self.decode(r, m.rect, Blend::Normal).map(Some) + } + + /// A layer's normal map for `material`, decoded (RGB = normal, A = opacity). + pub fn layer_normal(&self, layer: &Layer, material: &str) -> Result> { + let Some(m) = layer.maps.get(material) else { return Ok(None) }; + let Some((f, rect)) = &m.normal else { return Ok(None) }; + self.decode(f, *rect, Blend::Normal).map(Some) + } + + fn decode(&self, file: &str, rect: [u32; 4], blend: Blend) -> Result { + let img = image::open(self.dir.join(file)).with_context(|| file.to_string())?.to_rgba8(); + Ok(LayerImage { rect, size: self.size, w: img.width(), h: img.height(), rgba: img.into_raw(), blend, parts: Vec::new() }) + } +} + +/// A decoded layer map. +#[derive(Clone)] +pub struct LayerImage { + pub rect: [u32; 4], + /// texture size `rect` is in + pub size: u32, + pub w: u32, + pub h: u32, + pub rgba: Vec, + /// how the colour combines with the texel under it (colour maps) + pub blend: Blend, + /// parts in other blend modes (same rect), laid after this one + pub parts: Vec, +} + +/// Where a layer goes, relative to where it was captured or generated: moved +/// (`offset`, as a fraction of the texture), scaled and rotated (degrees) about +/// its rect's centre, as SkinGen places a layer. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct Xf { + pub offset: [f32; 2], + pub scale: f32, + pub rotation: f32, +} + +impl Default for Xf { + fn default() -> Self { + Self { offset: [0.0, 0.0], scale: 1.0, rotation: 0.0 } + } +} + +impl Xf { + pub fn is_identity(&self) -> bool { + *self == Self::default() + } +} + +/// SkinGen's "Adjust Color" of a layer's colour: hue turn (degrees), and +/// saturation, brightness and contrast (-1..1, 0 = as it is). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Adjust { + #[serde(default)] + pub hue: f32, + #[serde(default)] + pub saturation: f32, + #[serde(default)] + pub brightness: f32, + #[serde(default)] + pub contrast: f32, +} + +impl Adjust { + pub fn is_identity(&self) -> bool { + *self == Self::default() + } + + /// The adjusted colour (sRGB 0..255). + pub fn apply(&self, c: [f32; 3]) -> [f32; 3] { + if self.is_identity() { + return c; + } + let mut v = c.map(|x| x / 255.0); + if self.hue != 0.0 { + // rotation about the grey axis + let (sn, cs) = self.hue.to_radians().sin_cos(); + let k = (1.0 - cs) / 3.0; + let q = sn / 3f32.sqrt(); + let m = [[cs + k, k - q, k + q], [k + q, cs + k, k - q], [k - q, k + q, cs + k]]; + v = [0, 1, 2].map(|r| m[r][0] * v[0] + m[r][1] * v[1] + m[r][2] * v[2]); + } + let l = 0.2126 * v[0] + 0.7152 * v[1] + 0.0722 * v[2]; + let sat = 1.0 + self.saturation; + let bright = 2f32.powf(self.brightness * 1.5); + let con = 1.0 + self.contrast; + [0, 1, 2].map(|i| (((l + (v[i] - l) * sat) * bright - 0.5) * con + 0.5).clamp(0.0, 1.0) * 255.0) + } +} + +/// A layer mask over a material's whole texture (0..1 per texel, any size): +/// multiplies the layer's opacity where the layer lands (SkinGen's "Layer Mask"). +#[derive(Clone, Debug)] +pub struct Mask { + pub w: u32, + pub h: u32, + pub v: Vec, +} + +impl Mask { + /// At texture coordinates `u`, `v` (0..1), bilinear. + pub fn at(&self, u: f32, v: f32) -> f32 { + let x = (u * self.w as f32 - 0.5).clamp(0.0, self.w as f32 - 1.0); + let y = (v * self.h as f32 - 0.5).clamp(0.0, self.h as f32 - 1.0); + let (x0, y0) = (x.floor() as u32, y.floor() as u32); + let (x1, y1) = ((x0 + 1).min(self.w - 1), (y0 + 1).min(self.h - 1)); + let (fx, fy) = (x - x0 as f32, y - y0 as f32); + let g = |x: u32, y: u32| self.v[(y * self.w + x) as usize]; + (g(x0, y0) * (1.0 - fx) + g(x1, y0) * fx) * (1.0 - fy) + (g(x0, y1) * (1.0 - fx) + g(x1, y1) * fx) * fy + } + + /// 1 − the mask. + pub fn inverted(mut self) -> Self { + self.v.iter_mut().for_each(|x| *x = 1.0 - *x); + self + } +} + +/// One layer to lay over a texture. +pub struct Apply<'a> { + pub image: &'a LayerImage, + /// opacity scale (CC5's layer strength), 0..1 (above 1: stronger, clamped per texel) + pub strength: f32, + /// (new colour, the layer's [`Layer::color`]), sRGB 0..255: recolour the + /// layer, keeping its pattern (each texel's brightness relative to the mean) + pub tint: Option<([f32; 3], [f32; 3])>, + /// 0..1: how much the skin's own shading (each texel's brightness) shows + /// through the layer's colour (generated lipstick: the lips' texture) + pub detail: f32, + pub xf: Xf, + /// colour adjustment (after the tint) + pub adjust: Adjust, + /// opacity mask over the texture + pub mask: Option<&'a Mask>, +} + +impl<'a> Apply<'a> { + /// `image` as it is: full strength, no tint, adjustment or mask. + pub fn new(image: &'a LayerImage) -> Self { + Self { image, strength: 1.0, tint: None, detail: 0.0, xf: Xf::default(), adjust: Adjust::default(), mask: None } + } +} + +/// Runs `f(texel, sample, mask)` for every texel of a `w` x `h` RGBA8 texture the +/// placed layer covers, in parallel; `sample` is the layer's bilinear RGBA there +/// and `mask` the mask's value at the texel (1 without one). +fn each_texel(buf: &mut [u8], w: u32, h: u32, li: &LayerImage, xf: Xf, mask: Option<&Mask>, f: impl Fn(&mut [u8], [f32; 4], f32) + Sync) { + let k = w as f32 / li.size as f32; + let (rx, ry) = (li.rect[0] as f32 * k, li.rect[1] as f32 * k); + let (rw, rh) = (li.rect[2] as f32 * k, li.rect[3] as f32 * k); + let (cx, cy) = (rx + rw * 0.5, ry + rh * 0.5); + let (ox, oy) = (xf.offset[0] * w as f32, xf.offset[1] * h as f32); + let sc = xf.scale.max(1e-3); + let (sn, cs) = xf.rotation.to_radians().sin_cos(); + // destination bounds: the rect's corners, placed + let corners = [(rx, ry), (rx + rw, ry), (rx, ry + rh), (rx + rw, ry + rh)].map(|(x, y)| { + let (dx, dy) = ((x - cx) * sc, (y - cy) * sc); + (cx + ox + dx * cs - dy * sn, cy + oy + dx * sn + dy * cs) + }); + let x0 = corners.iter().map(|c| c.0).fold(f32::MAX, f32::min).floor().max(0.0) as u32; + let x1 = (corners.iter().map(|c| c.0).fold(f32::MIN, f32::max).ceil().max(0.0) as u32).min(w); + let y0 = corners.iter().map(|c| c.1).fold(f32::MAX, f32::min).floor().max(0.0) as u32; + let y1 = (corners.iter().map(|c| c.1).fold(f32::MIN, f32::max).ceil().max(0.0) as u32).min(h); + if x1 <= x0 || y1 <= y0 { + return; + } + buf.par_chunks_mut(w as usize * 4).enumerate().filter(|(y, _)| (*y as u32) >= y0 && (*y as u32) < y1).for_each(|(y, row)| { + for x in x0..x1 { + // texel centre -> the layer's own frame (inverse placement) -> its image + let (px, py) = (x as f32 + 0.5 - cx - ox, y as f32 + 0.5 - cy - oy); + let (ux, uy) = ((px * cs + py * sn) / sc + cx, (-px * sn + py * cs) / sc + cy); + if ux < rx || uy < ry || ux >= rx + rw || uy >= ry + rh { + continue; + } + let lx = ((ux - rx) / rw * li.w as f32 - 0.5).clamp(0.0, li.w as f32 - 1.0); + let ly = ((uy - ry) / rh * li.h as f32 - 0.5).clamp(0.0, li.h as f32 - 1.0); + let m = mask.map_or(1.0, |m| m.at((x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32)); + if m > 0.0 { + f(&mut row[x as usize * 4..x as usize * 4 + 4], bilinear(li, lx, ly), m); + } + } + }); +} + +/// Lays layers' roughness (their maps' R, at their opacity x strength) over the +/// G channel of a glTF metallic-roughness texture (RGBA8, `w` x `h`), in place. +pub fn compose_roughness(mr: &mut [u8], w: u32, h: u32, layers: &[(&LayerImage, f32, Xf, Option<&Mask>)]) { + compose_channel(mr, w, h, layers, 1); +} + +/// Lays layers' values (their maps' R, at their opacity x strength) over one +/// channel of an RGBA8 texture, in place: metallic (B of a glTF metallic-roughness +/// map), occlusion (R of an occlusion map), roughness (G). +pub fn compose_channel(buf: &mut [u8], w: u32, h: u32, layers: &[(&LayerImage, f32, Xf, Option<&Mask>)], channel: usize) { + for (li, strength, xf, mask) in layers { + if *strength <= 0.0 { + continue; + } + each_texel(buf, w, h, li, *xf, *mask, |p, s, m| { + let a = (s[3] / 255.0 * strength * m).clamp(0.0, 1.0); + p[channel] = (p[channel] as f32 * (1.0 - a) + s[0] * a).round().clamp(0.0, 255.0) as u8; + }); + } +} + +/// Lays layers' normals over a normal map (RGBA8, `w` x `h`), in place: the +/// unpacked normals (turned with the layer's rotation) blend by the layers' +/// opacity x strength and renormalize. +pub fn compose_normal(nm: &mut [u8], w: u32, h: u32, layers: &[(&LayerImage, f32, Xf, Option<&Mask>)]) { + for (li, strength, xf, mask) in layers { + if *strength <= 0.0 { + continue; + } + let (sn, cs) = xf.rotation.to_radians().sin_cos(); + each_texel(nm, w, h, li, *xf, *mask, |p, s, m| { + let a = (s[3] / 255.0 * strength * m).clamp(0.0, 1.0); + if a <= 0.0 { + return; + } + let l = [0, 1, 2].map(|c| s[c] / 255.0 * 2.0 - 1.0); + let l = [l[0] * cs - l[1] * sn, l[0] * sn + l[1] * cs, l[2]]; + let n: [f32; 3] = [0, 1, 2].map(|c| (p[c] as f32 / 255.0 * 2.0 - 1.0) * (1.0 - a) + l[c] * a); + let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt().max(1e-6); + for c in 0..3 { + p[c] = ((n[c] / len * 0.5 + 0.5) * 255.0).round().clamp(0.0, 255.0) as u8; + } + }); + } +} + +/// Lays `layers` over `base` (RGBA8 sRGB, `w` x `h`, any size: rects scale) in +/// order, in place, each by its [`Blend`] (colour tinted, then adjusted; opacity +/// masked). CC5 blends layers in sRGB. +pub fn compose(base: &mut [u8], w: u32, h: u32, layers: &[Apply]) { + let lum = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for a in layers { + if a.strength <= 0.0 { + continue; + } + let tint = a.tint.map(|(to, from)| (to, 1.0 / lum(from).max(1.0))); + for li in std::iter::once(a.image).chain(&a.image.parts) { + each_texel(base, w, h, li, a.xf, a.mask, |p, s, m| { + let alpha = (s[3] / 255.0 * a.strength * m).clamp(0.0, 1.0); + if alpha <= 0.0 { + return; + } + let rel = tint.map(|(_, k)| lum([s[0], s[1], s[2]]) * k); + let col = [0, 1, 2].map(|c| match (tint, rel) { + (Some((to, _)), Some(r)) => (to[c] * r).min(255.0), + _ => s[c], + }); + let col = a.adjust.apply(col); + // the skin's shading through the colour (relative to a mid skin tone) + let shade = 1.0 + a.detail * (lum([p[0] as f32, p[1] as f32, p[2] as f32]) / 115.0 - 1.0); + for c in 0..3 { + let v = li.blend.apply(p[c] as f32 / 255.0, (col[c] * shade).min(255.0) / 255.0, alpha); + p[c] = (v * 255.0).round().clamp(0.0, 255.0) as u8; + } + }); + } + } +} + +fn bilinear(li: &LayerImage, x: f32, y: f32) -> [f32; 4] { + let (x0, y0) = (x.floor() as u32, y.floor() as u32); + let (x1, y1) = ((x0 + 1).min(li.w - 1), (y0 + 1).min(li.h - 1)); + let (fx, fy) = (x - x0 as f32, y - y0 as f32); + let px = |x: u32, y: u32| { + let i = (y * li.w + x) as usize * 4; + [li.rgba[i] as f32, li.rgba[i + 1] as f32, li.rgba[i + 2] as f32, li.rgba[i + 3] as f32] + }; + let (a, b, c, d) = (px(x0, y0), px(x1, y0), px(x0, y1), px(x1, y1)); + // colour weighted by opacity, so transparent texels' colour does not bleed in + let mut out = [0.0; 4]; + let wts = [(1.0 - fx) * (1.0 - fy), fx * (1.0 - fy), (1.0 - fx) * fy, fx * fy]; + let mut aw = 0.0; + for (p, wt) in [a, b, c, d].iter().zip(wts) { + out[3] += p[3] * wt; + aw += p[3] * wt; + for k in 0..3 { + out[k] += p[k] * p[3] * wt; + } + } + for k in 0..3 { + out[k] = if aw > 0.0 { out[k] / aw } else { 0.0 }; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compose_over_with_strength_and_tint() { + let li = LayerImage { rect: [0, 0, 2, 2], size: 2, w: 2, h: 2, rgba: [200, 0, 0, 255].repeat(4), blend: Blend::Normal, parts: Vec::new() }; + let mut base = [100u8, 100, 100, 255].repeat(4); + compose(&mut base, 2, 2, &[Apply { strength: 0.5, ..Apply::new(&li) }]); + assert_eq!(&base[0..3], &[150, 50, 50]); + let mut base = [100u8, 100, 100, 255].repeat(4); + compose(&mut base, 2, 2, &[Apply { tint: Some(([0.0, 0.0, 200.0], [200.0, 0.0, 0.0])), ..Apply::new(&li) }]); + assert_eq!(&base[0..3], &[0, 0, 200]); + } + + #[test] + fn placement_moves_the_layer() { + // a 2x2 red patch at the top-left of a 8x8 texture, moved by half the texture + let li = LayerImage { rect: [0, 0, 2, 2], size: 8, w: 2, h: 2, rgba: [255, 0, 0, 255].repeat(4), blend: Blend::Normal, parts: Vec::new() }; + let mut base = [0u8, 0, 0, 255].repeat(64); + let xf = Xf { offset: [0.5, 0.5], scale: 1.0, rotation: 0.0 }; + compose(&mut base, 8, 8, &[Apply { xf, ..Apply::new(&li) }]); + let px = |x: usize, y: usize| base[(y * 8 + x) * 4]; + assert_eq!((px(0, 0), px(4, 4), px(5, 5)), (0, 255, 255)); + } + + #[test] + fn overlay_and_soft_light() { + // mid grey leaves both as they are; they darken / lighten towards the colour + for m in [Blend::Overlay, Blend::SoftLight] { + assert!((m.apply(0.3, 0.5, 1.0) - 0.3).abs() < 1e-6, "{m:?} with grey"); + assert!(m.apply(0.3, 0.1, 1.0) < 0.3 && m.apply(0.3, 0.9, 1.0) > 0.3, "{m:?}"); + assert_eq!(m.apply(0.3, 0.9, 0.0), 0.3, "{m:?} at opacity 0"); + } + assert!((Blend::Overlay.apply(0.25, 0.8, 1.0) - 0.4).abs() < 1e-6); + } + + #[test] + fn mask_and_adjust() { + // a red layer over grey, masked to the left half; the adjustment turns it blue-ish + let li = LayerImage { rect: [0, 0, 2, 1], size: 2, w: 2, h: 1, rgba: [200, 0, 0, 255].repeat(2), blend: Blend::Normal, parts: Vec::new() }; + let mask = Mask { w: 2, h: 1, v: vec![1.0, 0.0] }; + let mut base = [100u8, 100, 100, 255].repeat(2); + compose(&mut base, 2, 1, &[Apply { mask: Some(&mask), ..Apply::new(&li) }]); + assert_eq!((base[0], base[4]), (200, 100), "masked half untouched"); + let turned = Adjust { hue: 240.0, ..Default::default() }.apply([200.0, 0.0, 0.0]); + assert!(turned[2] > 150.0 && turned[0] < 50.0, "{turned:?}"); + assert_eq!(Adjust::default().apply([10.0, 20.0, 30.0]), [10.0, 20.0, 30.0]); + } + + #[test] + fn compose_normal_blends_and_renormalizes() { + // flat normal (0, 0, 1) blended halfway towards (1, 0, 0) + let li = LayerImage { rect: [0, 0, 1, 1], size: 1, w: 1, h: 1, rgba: vec![255, 128, 128, 255], blend: Blend::Normal, parts: Vec::new() }; + let mut nm = vec![128u8, 128, 255, 255]; + compose_normal(&mut nm, 1, 1, &[(&li, 0.5, Xf::default(), None)]); + let n: Vec = nm[..3].iter().map(|&v| v as f32 / 255.0 * 2.0 - 1.0).collect(); + let len = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt(); + assert!((len - 1.0).abs() < 0.02 && (n[0] - n[2]).abs() < 0.03, "{n:?}"); + } +} diff --git a/crates/cc5-skin/src/main.rs b/crates/cc5-skin/src/main.rs new file mode 100644 index 0000000..7835b0a --- /dev/null +++ b/crates/cc5-skin/src/main.rs @@ -0,0 +1,74 @@ +//! cc5-skin build +//! +//! spec.json: {"base": "base", "base_suffix": "", "alt_image": "blue2048.png", "alt_suffix": "@blue", +//! "character": "base", (or "alt": "" instead of alt_image) +//! "alt_for": {"": ["", ""]}, (per-layer second base) +//! "materials": ["Std_Skin_Head", ...], +//! "layers": [["lip_rouge", "Rouge", "Lip Makeup"], ...], +//! "skins": [["full_default_male", "Default Male"], ...]} +//! cc5-skin generate (each generator at its default look) +//! See the crate docs (`cc5_skin`). +use anyhow::{Context, Result}; +use cc5_skin::build::{build, BuildOptions}; +use cc5_skin::generate::{generate, Kind, Masks}; +use serde_json::Value; +use std::path::PathBuf; + +fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + match args.get(1).map(String::as_str) { + Some("build") if args.len() == 5 => { + let spec: Value = serde_json::from_slice(&std::fs::read(&args[4]).context("spec")?)?; + let s = |k: &str, d: &str| spec[k].as_str().unwrap_or(d).to_string(); + let list = |k: &str| -> Vec> { + spec[k].as_array().into_iter().flatten().map(|r| r.as_array().into_iter().flatten().filter_map(|x| x.as_str().map(str::to_owned)).collect()).collect() + }; + let opt = BuildOptions { + capture: PathBuf::from(&args[2]), + out: PathBuf::from(&args[3]), + base: s("base", "skinbase_female_clean"), + alt: s("alt", "skinbase_male_clean"), + base_suffix: s("base_suffix", "@female"), + alt_suffix: s("alt_suffix", "@male"), + character: s("character", "base"), + alt_image: spec["alt_image"].as_str().map(PathBuf::from), + check: spec["check_image"].as_str().map(|p| (PathBuf::from(p), s("check_suffix", "@gray"))), + materials: spec["materials"].as_array().into_iter().flatten().filter_map(|x| x.as_str().map(str::to_owned)).collect(), + min_alpha: spec["min_alpha"].as_f64().unwrap_or(0.02) as f32, + alt_for: spec["alt_for"] + .as_object() + .into_iter() + .flatten() + .filter_map(|(k, v)| Some((k.clone(), (v[0].as_str()?.to_owned(), v[1].as_str()?.to_owned())))) + .collect(), + }; + let layers: Vec<(String, String, String)> = + list("layers").into_iter().map(|r| (r[0].clone(), r.get(1).cloned().unwrap_or_else(|| r[0].clone()), r.get(2).cloned().unwrap_or_default())).collect(); + let skins: Vec<(String, String)> = list("skins").into_iter().map(|r| (r[0].clone(), r.get(1).cloned().unwrap_or_else(|| r[0].clone()))).collect(); + let t = std::time::Instant::now(); + let pack = build(&opt, &layers, &skins)?; + println!("{} layers, {} skins in {:.1} s -> {}", pack.layers.len(), pack.skins.len(), t.elapsed().as_secs_f32(), opt.out.display()); + Ok(()) + } + Some("generate") if args.len() == 4 => { + // every generator at its default look, from a head's region masks + // (/Std_Skin_Head_{CFULC,NMUIL,EN}Mask.png): /.png + // (RGBA: colour, opacity at strength 1) and kinds.json (blend, colour, strength) + let (dir, out) = (PathBuf::from(&args[2]), PathBuf::from(&args[3])); + let mask = |n: &str| -> Result { Ok(image::open(dir.join(format!("Std_Skin_Head_{n}Mask.png"))).context(n.to_string())?.to_rgba8()) }; + let (a, b, c) = (mask("CFULC")?, mask("NMUIL")?, mask("EN")?); + let m = Masks { w: a.width(), h: a.height(), cfulc: a.into_raw(), nmuil: b.into_raw(), en: c.into_raw() }; + std::fs::create_dir_all(&out)?; + let mut kinds = serde_json::Map::new(); + for k in Kind::ALL { + let (rgb, strength) = k.default_look(); + let g = generate(&m, k, rgb, 1, k.default_pattern()); + image::RgbaImage::from_raw(m.w, m.h, g.color.rgba).context("size")?.save(out.join(format!("{k:?}.png")))?; + kinds.insert(format!("{k:?}"), serde_json::json!({"blend": g.color.blend, "color": rgb, "strength": strength})); + } + std::fs::write(out.join("kinds.json"), serde_json::to_vec_pretty(&kinds)?)?; + Ok(()) + } + _ => anyhow::bail!("usage: cc5-skin build | generate "), + } +} diff --git a/docs/design/avatar-editor.md b/docs/design/avatar-editor.md new file mode 100644 index 0000000..a99d073 --- /dev/null +++ b/docs/design/avatar-editor.md @@ -0,0 +1,266 @@ +# Avatar shape editor (CC5 sliders outside CC5) + +`examples/avatar_editor.rs` reshapes a Character Creator 5 character with CC5's own +Morphs-panel sliders (body type, height, face features — about 3,400 for a CC3+/HD +character) without CC5. `examples/shape_avatar.rs` does the same headless from a +weights JSON. Both write a shaped GLB for the normal avatar pipeline. + +## How it works + +CC5 is used once, unattended, to record what every slider does to *this* +character (`remotemedia-sdk/scripts/avatars/cc5_capture_sliders.py`): + +- **bind mode**: per slider state, the displayed bind-pose mesh from CC5's + subdivision-0 OBJ export. This already contains the skeletal stretch — many + sliders act through the skeleton (Character Height moves CC5's internal mesh + <1 cm but the bones ~12 cm). +- **capture mode**: per slider state, skin-bone positions (CC5 live pose). The + pack builder maps them into the bind pose (`bind_joints.py`: chain rebuilt + with per-bone pose rotations, grounding anchored on rigidly skinned joints). + +`build_editor_pack.py` turns that into `pack.json` + `pack.bin` +([`cc5_shape::ShapePack`](../../crates/cc5-shape/src/pack.rs); the +[`cc5-shape`](../../crates/cc5-shape/README.md) crate is also a Python module, +`import cc5_shape`): sparse vertex and joint +deltas in the glTF frame. The reference GLB (CC5's FBX export converted by +`cc5-fbx`, ufbx) carries a `_CCIDX` vertex attribute, the CC5 vertex +index of every glTF vertex, so seam-split vertices map back exactly. + +A slider mix is `bind + Σ tᵢ·Δᵢ` for vertices and joints, `tᵢ` linear on each +side of the slider's captured value. On CC5's internal (pre-skinning) mesh, +sliders are exactly linear and additive (0 error for a half weight and for two +sliders combined); for sliders that move bones, the bind-pose shape includes +skinning, so mixes are a close linear approximation there. +[`shape_glb`](../../crates/cc5-shape/src/glb.rs) writes it into a GLB: positions, +joint translations (rotations unchanged), and inverse bind matrices +`IBM' = W'⁻¹·W·IBM` so the new bind mesh renders exactly at bind. + +Normals follow the shape ([`cc5_shape::normals`](../../crates/cc5-shape/src/normals.rs), +in `shape_glb` and in the editor, which also covers expressions): per vertex, the +rotation from the bind mesh's smooth normal to the shaped mesh's turns CC5's +authored normal (and tangent). Smooth normals are area-weighted and welded by +CC5 vertex, so seams stay closed; where the surface did not move the normal is +kept bit for bit. CC5's own normals are these smooth normals (Beautiful's body: +median 0.00°, p99 0.05° apart), so the turned normals are what the shaped +surface has. Without this, a large reshape (e.g. the neutral male base on a +female capture) kept the capture's shading: on the fitted male chest the old +normals were 24° off the surface (median; up to 88°) and rendered breasts on a +flat chest. + +## Building the inputs + +On the Windows machine with CC5 (5.13), Python and cc-render's `cc5-fbx` built +(`cargo build -p cc5-fbx --release --target-dir target-cc5`; no Blender): + +```powershell +remotemedia-sdk\scripts\avatars\cc5_editor\build_cc5_editor_pack.ps1 ` + -Project C:\Users\me\Documents\avatars\Beautiful.ccProject -OutDir D:\cc5_editor\beautiful +``` + +It drives CC5 with no clicks (loads CC5's slider library via "Load CC5 HD +Neutral Base" — without that step CC5 exposes no head/body sliders to Python — +opens the project, captures, exports the reference FBX), converts it with `cc5-fbx`, +bakes hair/brow alpha (`bake_alpha.py`) and builds the pack. The first run takes +hours (bind capture ≈ 3.5 s per slider state); reruns resume. Any running CC5 +is closed first; the project is never saved. + +## Running + +``` +cargo run --example avatar_editor -- D:\cc5_editor\beautiful\character.glb D:\cc5_editor\beautiful\pack [shaped.glb] +cargo run --example shape_avatar -- character.glb pack weights.json shaped.glb +``` + +With no `character.glb pack_dir`, the editor opens `$CC5_EDITOR_DIR` (default +`D:\cc5_editor\beautiful_e2e`): its `character.glb` and `pack_xl` (else `pack`). + +Editor: category tree with search, per-slider reset, "Reset all", "Save GLB"; +with `--expressions `, an "Expressions" section (see below). +Right-drag orbits, middle-drag pans, wheel zooms. `weights.json` maps slider +names or ids to values (`{"Character Height": 0.5}`). + +The editor starts from, and "Reset all" returns to, its base body: by default +CC5's neutral female when the pack has it (`pack_xl`, where `avatar_slider.py` +adds the neutral bases as `avatar_neutral_f` / `_m` sliders), else the captured +character. `--base none | neutral_f | neutral_m | | ` +chooses another. Changed sliders are counted against the base, and saved recipes +keep it apart from the changes: `{"base": {...}, "sliders": {...}}` (loading +applies both, so a recipe means the same body in any session). + +Timelapse (the shape changing over time, e.g. weight gain): the "Timelapse" +section moves the sliders from a start (the sliders when the editor opened, or +"Current sliders") to a target, linearly per slider weight, with a time slider, +Play / Pause, a duration and back-and-forth playback. Targets: "Heavy (female)" = +the start plus `Female Heavy Full-body` 1 and `Face Heavy` 0.5 ("Heavy (male)" +uses `Male Heavy Full-body`), the current sliders, or a recipe file. +`--timelapse heavy-female | heavy-male | ` opens with that target +and plays it; in `--snapshot-batch` a job's `"time": 0..1` shows that point of +it, for rendering frames. Every frame is a plain slider mix, so "Save GLB" and +recipes save the shape on screen. On Beautiful, the CC5 "HA" heavy sliders +(`Body/Head HA Female Heavy`) swap in another head and height and `Body Fat A/B` +is extreme, so the preset uses the HD Ultimate heavy sliders. + +``` +cargo run --example avatar_editor -- --base none --timelapse heavy-female +``` + +## Accuracy (Beautiful / HD Ariana, Character Height = 1 vs CC5's own FBX export) + +| | error | +|---|---| +| bind mesh, 67,419 vertices | 0.314 cm, uniform (CC5's OBJ and FBX exports ground the character 3 mm apart); shape itself exact | +| joints | median 0.35 cm, max 0.79 cm (mostly the same 3 mm shift; fingers worst) | +| skin consistency `W·IBM` | 1e-6 | + +For scale, the slider moves vertices up to 12.3 cm and joints up to 11.8 cm. + +## Facial expressions (CPU morphing) + +A CC5 HD character carries ~400 facial morph targets per face mesh (body 349, +brows/lashes 395, tear line 437...). Bevy 0.15 refuses meshes with more than 64 +(`MAX_MORPH_WEIGHTS`), so the reference GLB has none (`cc5-fbx` drops them) +and the expressions live in a sidecar *expression pack*, applied on the CPU like +the shaping sliders: + +```powershell +cc5-fbx expressions D:\cc5_editor\beautiful\ref\character.fbx D:\cc5_editor\beautiful\pack D:\cc5_editor\beautiful\expressions +cargo run --example avatar_editor -- character.glb pack --expressions expressions +``` + +`expressions.json` + `expressions.bin` ([`cc5_shape::ExpressionPack`](../../crates/cc5-shape/src/expr.rs)): +per shape-key name, the sparse delta `key − Basis` at weight 1, merged over every +mesh that has that key, in the slider pack's global vertex space and glTF frame +(the FBX's vertex order is CC5's). Beautiful: 480 expressions (56 empty +correctives, e.g. `Mouth_Lips_Together_*`, which CC5 exports as zero), 785k +vertex deltas, 12.6 MB. A mix is `bind + shape Δ + Σ wᵢ·exprᵢ`; morphing is +linear, so this is exact — against Blender evaluating the FBX's shape keys +(one key at 1, one at 0.5, two 3-key mixes across body, lashes, brows, tear +line, eye occlusion, eyes and tongue; up to 29 mm of motion) the max error is +1.2e-4 mm. + +The editor's "Expressions" section (0..1 sliders, shares the search box) layers +expressions on the shaped positions in `apply_shape`; an expression-only change +rewrites positions without touching joints. Expressions are a preview there: +"Save GLB" and recipes hold the shape only. Only morphs are covered: CC5's jaw +and eye *bones* (Jaw_Open also rotates CC_Base_JawRoot in CC5) are not part of the +pack, exactly as with glTF morph targets. `cc5_shape::ArkitMap` resolves ARKit-52 +weights (Audio2Face) through a runtime `*.arkit_map*.json` (`{"mapping": +{arkitName: [{morph, weight, meshes}]}}`; `meshes` is ignored since an expression +spans all meshes). + +### Runtime: meshes with more than 64 morphs (design, not implemented) + +cc-render's runtime (`src/cc_render/bevy_app`) drives morph weights per frame +from Audio2Face ARKit weights. For a character whose GLB exceeds 64 morphs per +mesh it would switch that mesh to the CPU path: + +1. Load the GLB without morphs (as now) and the character's expression pack + + ARKit map. At spawn, record per primitive what the editor records: base + positions, `_CCIDX` globals, and the bind-space rotation (`MeshRec`). +2. Per frame: ARKit weights → `ArkitMap::weights` → `ExpressionPack::add_to` + into a reused per-character buffer (zero only the vertices touched last frame), + then rewrite `ATTRIBUTE_POSITION` of the affected primitives only (face, brows, + lashes, tear line, occlusion, eyes, tongue — not body-wide). Cost is linear in + the active expressions' vertex counts (typically 20–50 active ARKit targets, + a few ×10k vertex adds) plus the position upload (~15k body vertices ≈ 180 KB), + well within a frame. Skip the write when weights are unchanged. +3. Normals: morph targets in the GLB carry normal deltas; the CPU path would + turn the normals of the touched region with `cc5_shape::normals`, as the + editor does. Bone-driven parts (jaw, eyes) stay on the skeleton as today. +4. Keep the GPU morph path for characters with ≤ 64 targets; pick per mesh at + load (count targets in the glTF, or the presence of an expression pack). + +Alternatives considered: splitting a mesh into ≤64-target primitives breaks +along seams and still caps per primitive; a custom storage-buffer morph shader +avoids the CPU upload but needs a forked skinning/morph pipeline in Bevy 0.15. + +## Skin (CC5's skin settings and SkinGen, outside CC5) + +The editor's **Skin** section (`examples/avatar_editor/skin.rs`) customizes the +character's skin the ways CC5 does: + +- **Material settings**: CC5's own RLHead / RLSkin variables (names, ranges and + groups of `Program/Assets/Share/Shaders/Human/RLHead.shader` / `RLSkin.shader`): + colour (brightness, saturation, human colour: red / yellow / black / range, blend + map), roughness per region, specular, subsurface, micro normal. Edits rebuild + the skin uniform live (`Cc5Skin::set_vars`); `skin.wgsl` implements them. +- **Skins**: whole skin presets (CC5's Skin Base / Full Skin / HD skins): their + textures and settings swap in. +- **SkinGen layers**: makeup (lips, eyes, blush, contour, foundation), blemishes, + freckles, moles, stubble, scars, tattoos, ... laid over the diffuse in a + background task (`cc5_skin::compose`), each at a strength and optionally recoloured. + +Skins and layers come from a **skin pack** (`/skin`, `--skin-pack`), +captured from CC5 itself — its presets are encrypted: + +1. `remotemedia-sdk/scripts/avatars/cc5_editor/skin/cc5_skin_capture.py` (inside + CC5, `run_capture.ps1` restarts CC5 per variant) applies each preset to the + character, bakes the SkinGen layers (CC5's "Bake All Skin Textures"; they + otherwise exist only on the GPU) and saves the textures, renders a face view + and records the camera, lights and IBL (`scene.json`). +2. `cc5-skin build ` solves every layer's colour and + opacity per texel from the layer on two bases: the character, and the + character saved with a contrasting skin (CC5's bake composites onto SkinGen's + own base skin, so a flat diffuse in the material never reaches it). Each + SkinGen blend mode is fitted — normal B·(1−α) + C·α (makeup colour), multiply + (tattoos, contour, tans, beards, freckles), screen (highlights) — and the best + kept per layer, with parts in other modes where a preset mixes them (full + makeup's lipstick multiplies). Layers reproduce CC5's bakes to ~0.15 levels + (0..255); free CC5 caps SkinGen at 15 layers, so a preset over the cap on the + second skin is skipped. + +Recipes (`--skin recipe.json`, Save / Load, snapshot jobs' `skin`) keep the skin, +the layers and the settings that differ from the character. + +**Generated layers** (`cc5_skin::generate`, no capture needed): lipstick, blush, +eyeshadow, contour, highlight, freckles, stubble, spots, from the head's region +masks. Their shapes are weighted sums of mask features fitted to CC5's presets +(`gen_shape_fit.py`), their default colour / strength / blend fitted through +those shapes (`gen_fit.py`); rendered against CC5's presets the change from the +bare skin is within 0.4–0.7 dE for lipstick, blush, highlight, freckles, +contour (CC5's change 0.7–2.2), stubble 1.4 of 1.6. + +**Controls against SkinGen's** (Appearance Editor, Layer Advanced Settings): + +| SkinGen | Here | +|---|---| +| layer opacity, show / delete | strength, on / off (captured), remove (generated, decals) | +| blend mode | captured: CC5's own (solved: normal / multiply / screen, mixed parts); generated, decals: normal / multiply / screen / overlay / soft light | +| colour, Adjust Color (hue, saturation, brightness, contrast) | tint; hue, saturation, brightness, contrast on every layer | +| Layer Mask (greyscale image) | head region and / or greyscale image, invert | +| transform (offset, scale, rotation) | move, scale, rotate on every layer; decals: place, size, rotation | +| Range (expansion, blur, contrast) | generated layers | +| roughness, normal strength | captured: roughness and normal scales; generated: roughness finish | +| metallic, AO, glow | every layer: a metallic value, an occlusion (darkening) and a glow colour / strength, through its opacity | +| displacement | not offered (no tessellation in the renderer) | +| pattern (tile, scale / rotation / opacity variation, disorder, seed) | freckles / stubble: dot size, density, variation, seed | +| duplicate | generated: any number of each kind; captured: one each | +| layer stack order | captured in the order switched on, then generated, then decals; generated and decals reorder among themselves | +| Skin Decal | decal layers: any image on any skin material | +| skins (Skin Base, Full Skin, HD) | the pack's skins | +| material: Head / Body / Arm / Leg / Nails | captured layers on every material CC5 put them; decals on any; generated on the head | + +**Matching CC5's look.** `--cc5-scene scene.json --cc5-lights` renders with CC5's +camera (its 36x24 mm film gate: render at width x width/1.5), its lights (rect +lights without inverse-square falloff, soft tube lights) and its IBL prefiltered +in memory, so renders compare pixel for pixel with CC5's (`--hide-hair` as CC5 +hides hair while editing SkinGen). Calibrated on a flat grey skin, then on the +character's own skin; the skin shader's constants are `SkinTune`. Findings: +CC5's skin normal maps are DirectX-style (flipped for glTF's tangents), and its +response to colour changes is ACES-like: with ACES tonemapping (+0.9 EV, +contrast-adaptive sharpening 0.15) our renders of CC5's own bakes match its +renders of 8 skins and 25 SkinGen layers at mean dE 4.4, the layers' change from +the base within 0.6 dE (AgX with a saturation boost: 5.1 and 0.9). + +## Limits + +- The reference GLB drops CC5's ~400 facial morph targets (Bevy loads at most + 64); they are applied on the CPU from the expression pack instead (above). +- Joint *rotations* are not captured; sliders are applied as joint translations. +- Normals are turned with the surface, not re-authored: a twist of the surface + about its own normal does not turn tangents, and hard edges in CC5's normals + (rare on the body) are carried as they are. Skin textures are the captured + character's (e.g. a female capture's areolae on a neutral male body). +- The pack is specific to the character it was captured from. +- The deltas are derived from Reallusion's slider content packs; check the + Reallusion content licence before distributing packs or shaped characters. diff --git a/examples/avatar_editor.rs b/examples/avatar_editor.rs new file mode 100644 index 0000000..2adf7fd --- /dev/null +++ b/examples/avatar_editor.rs @@ -0,0 +1,1665 @@ +//! CC5-style avatar shape editor: the sliders captured from Character Creator 5, +//! applied live to the character outside CC5. +//! +//! Usage: avatar_editor [character.glb pack_dir [shaped_out.glb]] [--recipe recipe.json] +//! [--base none||] +//! [--style other_character.glb] [--expressions expressions_dir] +//! +//! Without `character.glb pack_dir`, the editor opens `$CC5_EDITOR_DIR` (default +//! `D:\cc5_editor\beautiful_e2e`): its `character.glb` and `pack_xl` (else `pack`). +//! +//! `character.glb` is the CC5 reference export converted with `cc5-fbx` +//! (it carries the `_CCIDX` vertex attribute); `pack_dir` holds `pack.json` and +//! `pack.bin` from `build_editor_pack.py`. Every slider change re-evaluates the +//! mix and rewrites the bind mesh, joint translations and inverse bind matrices +//! the same way `shape_glb` does; "Save GLB" writes that as a GLB. A recipe (the +//! changed sliders as JSON, e.g. from `cc5_shape.fit`) can be opened with +//! `--recipe` or loaded/saved from the panel. +//! +//! `--base` sets the body the editor starts from and resets to: a slider at 1 +//! (id or name; `neutral_f` / `neutral_m` = CC5's neutral bases, which +//! `avatar_slider.py` adds to a pack as `avatar_neutral_f` / `_m`), a recipe file, +//! or `none` (the character as captured). Default: the neutral female when the pack +//! has it (`pack_xl`), else the captured character. Changes are counted against +//! the base; saved recipes keep it apart (`"base"`) from the changes (`"sliders"`). +//! +//! `--style` loads another CC character (a GLB, e.g. the one a recipe was fitted +//! to) for two optional view layers: "Surface" puts its skin / eye / teeth +//! materials on our CC_Base_* meshes (same CC UV layout, matched by material +//! name); "Assets" shows its hair, clothes and accessories, bound to *our* +//! skeleton by joint name so they follow our shape, in place of ours. Its body is +//! never shown. +//! +//! `--expressions` loads the character's facial expression pack +//! (`cc5-fbx expressions`: CC5's ~400 morph targets, more than Bevy's 64 per +//! mesh) for an "Expressions" section: 0..1 sliders applied on the CPU on top of +//! the shape (positions = bind + shape delta + sum of w_i * expression_i). +//! Expressions are a preview layer: "Save GLB" and recipes hold the shape only. +//! +//! `--hd map_dir` edits CC5's HD (subdivided) character instead: `character.glb` +//! is then the HD export (`build_cc5_editor_pack.ps1 -HdLevel 1`: `hd/character_hd.glb`) +//! and `map_dir` its `hd_map.json` + `.bin`, which spreads the pack's (level 0) +//! slider and expression moves over the HD mesh. "Save GLB" writes the HD GLB. +//! +//! `--hair-lib dir [--hair-style name]` adds a Hair section: card-hair parts from +//! a hair library (`remotemedia-sdk/scripts/avatars/cc5_editor/hair/`), one per +//! slot (Base, Top, Rear, Bangs, Beard, Brows, ...) or a named style, conformed +//! to the shaped body (see `avatar_editor/hair.rs`). +//! +//! `--timelapse heavy-female|heavy-male|target.recipe.json` opens the Timelapse +//! section with that target and plays it: the sliders move from the starting +//! shape to the target over time (e.g. the character gaining weight; see +//! `avatar_editor/timelapse.rs`). The section can also pick or change the start, +//! the target and the duration, scrub the time and play back and forth. +//! +//! Controls: right-drag orbits, middle-drag pans, wheel zooms towards the cursor +//! (down to 2 cm); Head / Body / Feet buttons frame those parts. +//! +//! `--snapshot out.png [--view head|body|feet] [--yaw deg] [--pitch deg] [--zoom k] +//! [--pan right,up (m)] [--size WxH] [--no-cc5]` renders one frame without the panel once the character +//! and its CC5 materials are in, writes it to `out.png` and exits (for comparing +//! the look against CC5's viewport). `--ev100` sets the exposure as usual. +use bevy::gltf::{Gltf, GltfPlugin}; +use bevy::input::mouse::{AccumulatedMouseMotion, AccumulatedMouseScroll}; +use bevy::core_pipeline::{bloom::Bloom, tonemapping::Tonemapping, Skybox}; +use bevy::core_pipeline::experimental::taa::{TemporalAntiAliasPlugin, TemporalAntiAliasing}; +use bevy::pbr::{ScreenSpaceAmbientOcclusion, ShadowFilteringMethod}; +use bevy::math::{Affine3A, Mat3}; +use bevy::pbr::{light_consts::lux, CascadeShadowConfigBuilder, DirectionalLightShadowMap}; +use bevy::prelude::*; +use bevy::render::camera::Exposure; +use bevy::render::view::{ColorGrading, ColorGradingGlobal}; +use bevy::render::mesh::skinning::{SkinnedMesh, SkinnedMeshInverseBindposes}; +use bevy::render::mesh::{MeshVertexAttribute, VertexAttributeValues}; +use bevy::render::render_resource::VertexFormat; +use bevy::picking::mesh_picking::ray_cast::{MeshRayCast, RayCastSettings}; +use bevy::render::view::NoFrustumCulling; +use bevy_egui::{egui, EguiContexts, EguiPlugin}; +use cc_render_plugin::cc_render::shape::{ShapeDelta, ShapePack}; +use cc5_shape::{shape_glb_hd, ExpressionPack, SubdivMap}; +use cc_render_plugin::cc_render::shape_glb::shape_glb; +use cc_render_plugin::cc_render::cc5_materials::{Cc5Material, Cc5Materials, Cc5MaterialsPlugin, Cc5MaterialsSet, SkinTune}; +use std::collections::HashMap; +use std::path::PathBuf; + +#[path = "avatar_editor/hair.rs"] +mod hair; +#[path = "avatar_editor/hair_edit.rs"] +mod hair_edit; +use hair::{hair_bind, hair_shape, hair_sync, hair_ui, HairLib}; +use hair_edit::{hair_edit_run, hair_edit_ui, HairEdit}; +#[path = "avatar_editor/skin.rs"] +mod skin; +#[path = "avatar_editor/timelapse.rs"] +mod timelapse; +#[path = "avatar_editor/cc5_scene.rs"] +mod cc5_scene; +#[path = "avatar_editor/ibl.rs"] +mod ibl; +use cc5_scene::{cc5_backdrop_follow, cc5_lights, cc5_live, Cc5Live, Cc5Scene}; +use skin::{skin_layers, skin_sync, skin_ui, SkinEdit}; +use timelapse::{timelapse, timelapse_ui, Timelapse}; + +const ATTRIBUTE_CCIDX: MeshVertexAttribute = MeshVertexAttribute::new("CcIdx", 0x0CC5_1D_E7, VertexFormat::Float32); + +#[derive(Resource)] +struct Editor { + /// Camera exposure (EV100); the portrait rig is lit for studio levels, not daylight. + ev100: f32, + pack: ShapePack, + categories: Vec<(String, Vec)>, + weights: Vec, + /// `--base`: the weights "Reset all" returns to, and what it is. + base: Vec, + base_label: String, + delta: ShapeDelta, + dirty: bool, + /// `--expressions`: the pack and one weight per expression (0 = neutral). + expr: Option, + expr_weights: Vec, + /// Only expression weights changed: positions are rewritten, joints are not. + expr_dirty: bool, + /// The sliders moving from one shape to another over time. + lapse: Timelapse, + /// `--hd`: the character is CC5's HD mesh; vertex deltas go through this map. + hd: Option, + search: String, + glb_path: PathBuf, + out_path: PathBuf, + recipe_path: String, + status: String, + /// `--hide-hair` / a job's `hide_hair`: the character's own hair and + /// earrings hidden (as in CC5 while SkinGen edits: its skin renders lack them). + hide_hair: bool, + /// With `hide_hair`: brow cards and eyelashes too (CC5 hides every hair object + /// once a SkinGen preset is applied). + hide_brows: bool, + /// Framing requested by the UI: (joints whose midpoint to aim at, camera distance); + /// no joints = whole body. + focus_request: Option<(&'static [&'static str], f32)>, +} + +/// Base state of the spawned character, captured once it has settled. +#[derive(Resource, Default)] +struct Rig { + ready: bool, + settle_frames: u32, + meshes: Vec, + joints: Vec, + skins: Vec, + /// Smooth normals of the base meshes per global vertex (`cc5_shape::normals`). + base_smooth: Vec, +} + +struct MeshRec { + /// glTF mesh name + name: String, + handle: Handle, + base: Vec<[f32; 3]>, + global: Vec, + /// Triangle list (empty: not triangles, normals left alone). + tris: Vec, + base_normals: Option>, + base_tangents: Option>, + /// mesh (bind) space -> world at bind + bind_space: Mat4, + to_mesh: Mat3, +} + +struct JointRec { + entity: Entity, + name: String, + pack: Option, + parent: Option, + base_local: Vec3, + parent_lin_inv: Mat3, + base_world: Affine3A, +} + +struct SkinRec { + handle: Handle, + joints: Vec, + base: Vec, +} + +/// Root of the character being edited (the pack's character). +#[derive(Component)] +struct OurCharacter; + +/// Root of the `--style` character. +#[derive(Component)] +struct StyleCharacter; + +/// `--style` view layers. +#[derive(Resource, Default)] +struct Style { + path: Option, + our_gltf: Handle, + style_gltf: Handle, + ready: bool, + surface: bool, + assets: bool, + applied: Option<(bool, bool)>, + /// Our CC_Base_* primitives: entity, original material, its glTF material name. + our_base: Vec<(Entity, Handle, Option)>, + /// Our hair / clothes / accessories. + our_assets: Vec, + style_assets: Vec, + style_materials: HashMap>, + status: String, +} + +/// One `--snapshot-batch` job: what to show and where to save it. +#[derive(serde::Deserialize, Clone)] +struct SnapJob { + out: PathBuf, + /// Hair style to apply ("" or absent: none / own hair). + #[serde(default)] + style: Option, + /// Shape recipe (absent: the character as captured). + #[serde(default)] + recipe: Option, + /// Live hair edit: {"group": ..., settings...} (needs --hair-src). + #[serde(default)] + edit: Option, + /// Save the edited part into the library (as the panel's Save) before the shot. + #[serde(default)] + save: bool, + /// Skin recipe (absent: the character's skin). + #[serde(default)] + skin: Option, + /// Hide the character's own hair and earrings. + #[serde(default)] + hide_hair: bool, + /// Also hide its brow cards and eyelashes (as CC5 does after a SkinGen preset). + #[serde(default)] + hide_brows: bool, + /// CC5 mode settings for this shot (`cc5_scene::Cc5Settings::with`): a + /// calibration sweep in one launch. Absent: the defaults. + #[serde(default)] + cc5: Option, + #[serde(default = "head_view")] + view: String, + #[serde(default = "one")] + zoom: f32, + #[serde(default)] + yaw: f32, + #[serde(default)] + pitch: f32, + /// Show the `--timelapse` at this time (0 = start, 1 = target) instead of the recipe. + #[serde(default)] + time: Option, +} + +fn head_view() -> String { + "head".into() +} +fn one() -> f32 { + 1.0 +} + +/// `--snapshot`: render a camera preset to a file, then exit (`--snapshot-batch`: +/// several, in one process). +#[derive(Resource)] +struct Snapshot { + path: PathBuf, + view: String, + yaw: f32, + pitch: f32, + zoom: f32, + /// Focus offset along the camera's right / up (m). + pan: Vec2, + /// Seconds since the rig was ready and the CC5 material count last changed. + stable: f32, + cc5_count: usize, + /// Frames since the materials settled. + step: u32, + /// Seconds since the camera preset was applied. + framed: f32, + /// `--snapshot-batch`: the jobs after this one (one process renders them all). + queue: std::collections::VecDeque, + /// The camera's yaw before any job's (batch jobs set theirs from it). + base_yaw: Option, + /// Shots taken (later ones skip the pipeline warm-up). + taken: u32, + /// Save the live hair edit before this shot (`SnapJob::save`). + save: bool, +} + +#[derive(Component)] +struct Orbit { + focus: Vec3, + yaw: f32, + pitch: f32, + dist: f32, +} + +/// Default camera exposure: Bevy's default (EV100 9.7, daylight) leaves the +/// portrait rig (a few hundred lux) far too dark. +const DEFAULT_EV100: f32 = 7.25; + +/// `--snapshot`: seconds to render the framed view before the shot (shader and +/// pipeline compilation for new views and materials is asynchronous). +const SNAPSHOT_SETTLE_SECS: f32 = 6.0; + +/// Size of the key light for its soft (PCSS) shadows. +const KEY_SOFT_SHADOW: f32 = 6.0; + +/// Vertical field of view: CC5's Preview Camera, an 80 mm lens on its 36 x 24 mm +/// film gate (2·atan(12 / 80); 25.4 degrees across the gate's width), so faces +/// have CC5's perspective. Bevy's default 45 degrees distorts close-ups of the +/// face like a wide angle. +pub(crate) const FOV_DEG: f32 = 17.06; + +/// Scalp hair's diffuse / specular gains: `CC5_HAIR_GAIN` ("d,s"), else with +/// CC5's lights [`cc5_scene::HAIR_GAIN`], else 1 (the studio calibration). +fn hair_gain(cc5_lights: bool) -> [f32; 2] { + let d = if cc5_lights { cc5_scene::HAIR_GAIN } else { [1.0, 1.0] }; + let v: Vec = std::env::var("CC5_HAIR_GAIN").map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect()).unwrap_or_default(); + [v.first().copied().unwrap_or(d[0]), v.get(1).copied().unwrap_or(d[1])] +} + +fn knob_f32(name: &str, default: f32) -> f32 { + std::env::var(name).ok().and_then(|v| v.parse().ok()).unwrap_or(default) +} + +/// Scalp hair's colour multiplier: `CC5_HAIR_TINT` ("r,g,b"), else with CC5's +/// lights [`cc5_scene::HAIR_TINT`], else 1. +fn hair_tint(cc5_lights: bool) -> [f32; 3] { + let v: Vec = std::env::var("CC5_HAIR_TINT").map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect()).unwrap_or_default(); + if v.len() == 3 { + [v[0], v[1], v[2]] + } else if cc5_lights { + cc5_scene::HAIR_TINT + } else { + [1.0; 3] + } +} + +/// Camera distances below are framed for 45 degrees; scale them to FOV_DEG. +fn fov_scale() -> f32 { + (22.5f32.to_radians()).tan() / (FOV_DEG.to_radians() * 0.5).tan() +} + +fn main() -> anyhow::Result<()> { + let mut args: Vec = std::env::args().collect(); + let mut opt = |flag: &str| -> anyhow::Result> { + match args.iter().position(|a| a == flag) { + Some(i) if i + 1 < args.len() => Ok(Some(args.drain(i..i + 2).nth(1).unwrap())), + Some(_) => anyhow::bail!("{flag} needs a file"), + None => Ok(None), + } + }; + let recipe_arg = opt("--recipe")?; + let base_arg = opt("--base")?; + let timelapse_arg = opt("--timelapse")?; + // `--snapshot-batch jobs.json`: [{out, style?, recipe?, edit?, view?, zoom?, yaw?, pitch?}] + // rendered one after another in this process (no restart, pipelines compiled once) + let batch_jobs: Option> = opt("--snapshot-batch")?.map(|f| -> anyhow::Result<_> { Ok(serde_json::from_str(&std::fs::read_to_string(&f)?)?) }).transpose()?; + let first_job = batch_jobs.as_ref().and_then(|j| j.first().cloned()); + let recipe_arg = match first_job.as_ref().and_then(|j| j.recipe.clone()) { + Some(r) => Some(r.to_string_lossy().into_owned()), + None => recipe_arg, + }; + let ev100: f32 = opt("--ev100")?.map(|v| v.parse()).transpose()?.unwrap_or(DEFAULT_EV100); + let style_arg = opt("--style")?.map(std::fs::canonicalize).transpose()?; + let expr = opt("--expressions")?.map(|d| ExpressionPack::load(std::path::Path::new(&d))).transpose()?; + let hd = opt("--hd")?.map(|d| SubdivMap::load(std::path::Path::new(&d))).transpose()?; + let cc5_scene_arg = opt("--cc5-scene")?; + let cc5_ibl_arg = opt("--cc5-ibl")?.map(PathBuf::from); + let skin_recipe = opt("--skin")?.map(PathBuf::from).or_else(|| first_job.as_ref().and_then(|j| j.skin.clone())); + let skin_pack_arg = opt("--skin-pack")?.map(PathBuf::from); + let mut hair_lib = opt("--hair-lib")?.map(|d| HairLib::load(std::path::Path::new(&d))).transpose()?; + if let Some(name) = opt("--hair-style")?.or_else(|| first_job.as_ref().and_then(|j| j.style.clone()).filter(|s| !s.is_empty())) { + hair_lib.as_mut().ok_or_else(|| anyhow::anyhow!("--hair-style needs --hair-lib"))?.apply_style(&name)?; + } + // live hair editing: CC5 Smart Hair exports rebuilt with cc5-hair in the background + let hair_edit = match opt("--hair-src")? { + Some(src) => { + anyhow::ensure!(hair_lib.is_some(), "--hair-src needs --hair-lib"); + let head = opt("--hair-head")?.ok_or_else(|| anyhow::anyhow!("--hair-src needs --hair-head "))?; + let mut e = HairEdit::new(PathBuf::from(src), PathBuf::from(head), opt("--hair-config")?.map(PathBuf::from))?; + let edit_arg = match opt("--hair-edit")? { + Some(f) => Some(serde_json::from_str::(&std::fs::read_to_string(&f)?)?), + None => first_job.as_ref().and_then(|j| j.edit.clone()), + }; + if let Some(v) = edit_arg { + // {"group": "...", ...settings}: start editing it (scripted checks) + let g = v["group"].as_str().ok_or_else(|| anyhow::anyhow!("--hair-edit: no \"group\""))?.to_string(); + e.select(&g, Some(&v)); + } + Some(e) + } + None => None, + }; + let snapshot_arg = opt("--snapshot")?.or_else(|| first_job.as_ref().map(|j| j.out.to_string_lossy().into_owned())); + let view = opt("--view")?.unwrap_or_else(|| "head".into()); + let deg = |v: Option| -> anyhow::Result { Ok(v.map(|v| v.parse::()).transpose()?.unwrap_or(0.0).to_radians()) }; + let (yaw, pitch) = (deg(opt("--yaw")?)?, deg(opt("--pitch")?)?); + let zoom: f32 = opt("--zoom")?.map(|v| v.parse()).transpose()?.unwrap_or(1.0); + let pan = match opt("--pan")? { + Some(p) => { + let (x, y) = p.split_once(',').ok_or_else(|| anyhow::anyhow!("--pan right,up"))?; + Vec2::new(x.parse()?, y.parse()?) + } + None => Vec2::ZERO, + }; + let size = opt("--size")?.unwrap_or_else(|| "1139x978".into()); + let (w, h) = size.split_once('x').ok_or_else(|| anyhow::anyhow!("--size WxH"))?; + let size = (w.parse::()?, h.parse::()?); + let no_cc5 = match args.iter().position(|a| a == "--no-cc5") { + Some(i) => { + args.remove(i); + true + } + None => false, + }; + let hide_hair = args.iter().any(|a| a == "--hide-hair") || first_job.as_ref().is_some_and(|j| j.hide_hair); + args.retain(|a| a != "--hide-hair"); + let cc5_lights_flag = args.iter().any(|a| a == "--cc5-lights"); + args.retain(|a| a != "--cc5-lights"); + let cc5_scene = cc5_scene_arg.map(|p| Cc5Scene::load(std::path::Path::new(&p), cc5_lights_flag, cc5_ibl_arg.clone())).transpose()?; + if args.len() == 1 { + // no character given: the default editor folder's character and pack + let (glb, pack) = default_inputs(); + args.push(glb.to_string_lossy().into_owned()); + args.push(pack.to_string_lossy().into_owned()); + } + anyhow::ensure!(args.len() == 3 || args.len() == 4, "[character.glb pack_dir [shaped_out.glb]] [--recipe recipe.json] ..."); + let glb_path = std::fs::canonicalize(&args[1]).map_err(|e| anyhow::anyhow!("{}: {e}", args[1]))?; + let skin_edit = SkinEdit::new(skin_recipe, Some(skin_pack_arg.unwrap_or_else(|| glb_path.parent().unwrap().join("skin"))))?; + let pack = ShapePack::load(std::path::Path::new(&args[2]))?; + let out_path = args.get(3).map(PathBuf::from).unwrap_or_else(|| glb_path.with_extension("shaped.glb")); + let (base, base_label) = resolve_base(&pack, base_arg.as_deref())?; + let weights = match &recipe_arg { + Some(r) => pack.weights_from_recipe(&serde_json::from_slice(&std::fs::read(r)?)?)?, + None => base.clone(), + }; + let delta = pack.evaluate(&weights); + let mut lapse = Timelapse::new(weights.clone()); + if let Some(arg) = &timelapse_arg { + let (w, label) = timelapse::resolve_target(&pack, &weights, arg)?; + lapse.set_target(w, label); + match first_job.as_ref().and_then(|j| j.time) { + Some(t) => lapse.seek(t), + None if snapshot_arg.is_none() => lapse.play(), + None => {} + } + } + if let Some(x) = &expr { + anyhow::ensure!( + x.vertex_count == pack.vertex_count, + "expression pack has {} vertices, slider pack {}: built for another character", + x.vertex_count, + pack.vertex_count + ); + } + if let Some(h) = &hd { + h.check(&pack).map_err(|e| anyhow::anyhow!("--hd map does not belong to this pack: {e:#}"))?; + } + let editor = Editor { + categories: pack.categories(), + status: format!("{} sliders in {} categories; base: {base_label}", pack.sliders.len(), pack.categories().len()), + pack, + weights, + base, + base_label, + delta, + dirty: false, + expr_weights: vec![0.0; expr.as_ref().map_or(0, |x| x.len())], + expr, + expr_dirty: false, + lapse, + hd, + search: String::new(), + glb_path: glb_path.clone(), + focus_request: None, + hide_hair, + hide_brows: first_job.as_ref().is_some_and(|j| j.hide_brows), + ev100, + recipe_path: recipe_arg.unwrap_or_else(|| out_path.with_extension("recipe.json").to_string_lossy().into_owned()), + out_path, + }; + let asset_dir = glb_path.parent().unwrap().to_string_lossy().to_string(); + let mut window = Window { title: "CC5 avatar editor".into(), ..default() }; + if snapshot_arg.is_some() { + window.resolution = bevy::window::WindowResolution::new(size.0, size.1).with_scale_factor_override(1.0); + } + let mut snap = snapshot_arg.map(|p| Snapshot { + path: PathBuf::from(p), + view, + yaw, + pitch, + zoom, + pan, + stable: 0.0, + cc5_count: 0, + step: 0, + framed: 0.0, + queue: Default::default(), + base_yaw: None, + taken: 0, + save: false, + }); + // batch: the first job is set up like --snapshot (style / recipe / edit applied below); + // the rest run in the same process, one after another + if let Some(jobs) = &batch_jobs { + let mut q: std::collections::VecDeque = jobs.iter().cloned().collect(); + let first = q.pop_front().ok_or_else(|| anyhow::anyhow!("--snapshot-batch: no jobs"))?; + snap = Some(Snapshot { + path: first.out.clone(), + view: first.view.clone(), + yaw: first.yaw.to_radians(), + pitch: first.pitch.to_radians(), + zoom: first.zoom, + pan: Vec2::ZERO, + stable: 0.0, + cc5_count: 0, + step: 0, + framed: 0.0, + queue: q, + base_yaw: None, + taken: 0, + save: first.save, + }); + } + let mut app = App::new(); + app.add_plugins( + DefaultPlugins + .set(WindowPlugin { primary_window: Some(window), ..default() }) + .set(AssetPlugin { file_path: asset_dir, ..default() }) + .set( + GltfPlugin::default() + .add_custom_vertex_attribute("CCIDX", ATTRIBUTE_CCIDX) + .add_custom_vertex_attribute("_CCIDX", ATTRIBUTE_CCIDX), + ), + ) + .add_plugins(EguiPlugin) + .add_plugins(TemporalAntiAliasPlugin) + .add_plugins(Cc5MaterialsPlugin) + .insert_resource(DirectionalLightShadowMap { size: 4096 }) + .insert_resource(editor) + .init_resource::() + .insert_resource(Style { path: style_arg, ..default() }) + // CC5_SKIN_TUNE: the skin shader's calibration (SkinTune fields, comma-separated) + .insert_resource(Cc5Materials { + enabled: !no_cc5, + skin_tune: std::env::var("CC5_SKIN_TUNE").map(|v| SkinTune::parse(&v)).unwrap_or_else(|_| { + if cc5_lights_flag { + SkinTune { extra_wrap: cc5_scene::SKIN_WRAP, sss_desaturate: cc5_scene::SKIN_SSS_DESAT, ..default() } + } else { + SkinTune::default() + } + }), + // CC5_HAIR_GAIN: "diffuse,specular" (with --cc5-lights: calibrated against CC5) + hair_gain: hair_gain(cc5_lights_flag), + hair_tint: hair_tint(cc5_lights_flag), + hair_root_occlusion: knob_f32("CC5_HAIR_ROOT_OCC", if cc5_lights_flag { cc5_scene::HAIR_ROOT_OCCLUSION } else { 0.55 }), + ..default() + }) + .insert_resource(skin_edit) + .add_systems(Startup, (setup, cc5_lights.run_if(resource_exists::)).chain()) + .add_systems( + Update, + ( + no_culling, + capture_rig, + style_setup, + ui.run_if(not(resource_exists::)), + style_apply, + (hair_sync, hair_bind).chain().run_if(resource_exists::), + hair_edit_run.run_if(resource_exists::), + (skin_sync, skin_layers).chain(), + hide_own_hair, + timelapse, + apply_shape, + snapshot.run_if(resource_exists::), + cc5_live, + orbit_camera, + cc5_backdrop_follow, + ) + .chain() + .after(Cc5MaterialsSet), + ); + if let Some(s) = snap { + app.insert_resource(s); + } + if let Some(s) = cc5_scene { + app.insert_resource(s); + } + if let Some(h) = hair_lib { + app.insert_resource(h); + } + if let Some(e) = hair_edit { + app.insert_resource(e); + } + if std::env::var_os("CC_DEBUG_INPUT").is_some() { + app.add_systems(Update, debug_input); + } + app.run(); + Ok(()) +} + +/// Studio backdrop like CC5's viewport: soft grey-blue, darkening above eye level +/// (at the editor's 25 degree lens the top of a level close-up is only ~12 degrees +/// up). A small sRGB cubemap for `Skybox`: it stays out of the lighting and of +/// mesh picking. +fn studio_backdrop(images: &mut Assets) -> Handle { + use bevy::render::render_asset::RenderAssetUsages; + use bevy::render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureViewDescriptor, TextureViewDimension}; + const N: usize = 64; + let base = Vec3::new(84.0, 90.0, 104.0); + let mut data = Vec::with_capacity(N * N * 6 * 4); + for face in 0..6 { + for y in 0..N { + for x in 0..N { + let s = (x as f32 + 0.5) / N as f32 * 2.0 - 1.0; + let t = (y as f32 + 0.5) / N as f32 * 2.0 - 1.0; + // cube face order +X -X +Y -Y +Z -Z, t pointing down the face + let dir = match face { + 0 => Vec3::new(1.0, -t, -s), + 1 => Vec3::new(-1.0, -t, s), + 2 => Vec3::new(s, 1.0, t), + 3 => Vec3::new(s, -1.0, -t), + 4 => Vec3::new(s, -t, 1.0), + _ => Vec3::new(-s, -t, -1.0), + } + .normalize(); + let top = ((dir.y + 0.02) / 0.26).clamp(0.0, 1.0); + let c = base * (1.0 - 0.5 * top * top * (3.0 - 2.0 * top)); + data.extend_from_slice(&[c.x as u8, c.y as u8, c.z as u8, 255]); + } + } + } + let mut image = Image::new( + Extent3d { width: N as u32, height: N as u32, depth_or_array_layers: 6 }, + TextureDimension::D2, + data, + TextureFormat::Rgba8UnormSrgb, + RenderAssetUsages::RENDER_WORLD, + ); + image.texture_view_descriptor = Some(TextureViewDescriptor { dimension: Some(TextureViewDimension::Cube), ..default() }); + images.add(image) +} + +/// Backdrop luminance (cd/m²) for a texel of 1.0, such that the backdrop shows +/// the same at any exposure (it is a backdrop, not part of the lit scene). +fn backdrop_brightness(ev100: f32) -> f32 { + // Bevy's exposure: 1 / (1.2 * 2^ev100); 0.55 = calibrated against CC5's viewport. + 0.4 * 1.2 * 2f32.powf(ev100) +} + +fn setup( + mut commands: Commands, + assets: Res, + editor: Res, + mut style: ResMut