diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 6c2c8a768..13ce87614 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -87,6 +87,56 @@ pub fn segmentation_runtime_available() -> bool { openscreen_compositor::segmentation::runtime_available() } +pub struct SegmentFrameTask { + model_path: String, + rgba: Vec, +} + +impl Task for SegmentFrameTask { + type Output = Vec; + type JsValue = Buffer; + + fn compute(&mut self) -> Result { + let rgb: Vec = self + .rgba + .chunks_exact(4) + .flat_map(|p| [p[0], p[1], p[2]]) + .collect(); + // ponytail: une session ONNX chargée par appel (quelques dizaines de ms), parce qu'il + // n'y a qu'un appel par caméra et par ouverture du panneau. À garder en cache si un + // appelant se met à segmenter en continu. + let mut segmenter = openscreen_compositor::segmentation::Segmenter::load( + std::path::Path::new(&self.model_path), + ) + .map_err(|e| Error::from_reason(format!("{e:#}")))?; + segmenter + .run(&rgb) + .map(<[u8]>::to_vec) + .map_err(|e| Error::from_reason(format!("{e:#}"))) + } + + fn resolve(&mut self, _env: Env, mask: Self::Output) -> Result { + Ok(mask.into()) + } +} + +/// Masque du sujet pour UNE image, sans vue : la vignette de recadrage du panneau caméra s'en +/// sert pour montrer le fond choisi sur sa propre frame. +/// +/// `rgba` est une frame RGBA8 déjà réduite à la taille du modèle (`segmentation::MODEL_WIDTH` x +/// `MODEL_HEIGHT`), soit le `getImageData` d'un canvas tel quel. Le retour a le format de +/// `Segmenter::run` : un octet par pixel, 0 = fond, 255 = sujet. +/// +/// `AsyncTask`, comme `remux_seekable` : charger la session prend plus longtemps que l'inférence, +/// et ce temps n'a rien à faire sur le thread principal de Node. +#[napi] +pub fn segment_frame(model_path: String, rgba: Buffer) -> AsyncTask { + AsyncTask::new(SegmentFrameTask { + model_path, + rgba: rgba.to_vec(), + }) +} + #[napi] pub fn create_view( rect: CompositorViewRect, diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 2bf0e04c4..e25ea048b 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -372,6 +372,10 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { return createSuccessResponse(requestId, { support: compositorViewService.probeSegmentation(), }); + case "segmentFrame": + return createSuccessResponse(requestId, { + mask: await compositorViewService.segmentFrame(request.payload.rgba), + }); case "setRect": compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 918b6ea43..aa9774e07 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -556,6 +556,20 @@ export class CompositorViewService { return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model"; } + /** Subject mask for one camera frame, no view needed — see `segmentFrame` in the addon. + * `null` when the addon (or this version of it) or the model is missing. */ + async segmentFrame(rgba: Uint8Array): Promise { + const addon = this.ensureAddon(); + const modelPath = resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET); + if (!addon?.segmentFrame || !modelPath) { + return null; + } + return addon.segmentFrame( + modelPath, + Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), + ); + } + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`. * `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer * keeps them on the wire so the existing `CompositorViewRect` shape stays diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 2ac90e08e..1365846c7 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -123,6 +123,11 @@ export interface CompositorViewAddon { * `--dir` build has none either. */ segmentationRuntimeAvailable(): boolean; + /** Subject mask for ONE frame, outside any view — the crop thumbnail shows the camera + * background with it. `rgba` is already at the model's size (256x144 RGBA8); resolves one + * byte per pixel, 0 = background, 255 = subject. Optional: an older `.node` predates it. */ + segmentFrame?(modelPath: string, rgba: Buffer): Promise; + /** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the * target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side). * No HWND/native-window-handle is passed: there's no OS window to parent to. The diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index 7a8572f5e..95e823fec 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -350,7 +350,7 @@ var(--scene-foliage-d) 100%); /* ponytail: the wallpaper card is a floating, rounded surface (per the v4 design's preview frame) — rounded corners + a layered drop shadow lift it - off the scene gradient. The custom background (buildFrameStyle) fills this + off the scene gradient. The custom background (wallpaperStyle) fills this card; the video sits on top with its own padding so the wallpaper shows around it. */ border-radius: 16px; diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index fe2a1eef4..f9ca4e1e8 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -57,7 +57,7 @@ import { type WebcamCompositeLayout, } from "@/lib/compositeLayout"; import { webcamAnchorAt } from "@/lib/projectDefaults"; -import { classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; +import { wallpaperStyle } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils"; import { AnnotationLayer } from "./AnnotationLayer"; @@ -321,7 +321,7 @@ export function PreviewCanvas(props: PreviewCanvasProps) { // The stage hosting the interactive overlays is the CONTENT rect, and a frame never changes // it: the compositor keeps the footage the same size with or without a frame and grows the // frame outward, so every handle (annotations, privacy blur, zoom focus) sits on `layout`. - const frameStyle = useMemo(() => buildFrameStyle(settings), [settings]); + const frameStyle = useMemo(() => wallpaperStyle(settings.wallpaper), [settings.wallpaper]); const screenStyle = useMemo( () => buildScreenStyle(layout, settings, frameSize), [layout, settings, frameSize], @@ -509,38 +509,6 @@ export function PreviewCanvas(props: PreviewCanvasProps) { ); } -// ponytail: resolve `settings.wallpaper` to an actual CSS background. Image -// wallpapers must go through resolveImageWallpaperUrl (→ getAssetPath): in the -// packaged Electron app the renderer loads over file://, where a bare -// `/wallpapers/foo.jpg` points at the filesystem root and 404s, so the custom -// background silently failed to paint (worked in the http dev server only). -// classifyWallpaper also handles color-function (rgb/hsl/…) and every gradient -// variant, which the old ad-hoc startsWith checks missed. -function resolveWallpaperImageUrl(imagePath: string): string | null { - try { - return resolveImageWallpaperUrl(imagePath); - } catch { - return null; - } -} - -// Canvas: wallpaper only — no padding, no shadow. -function buildFrameStyle( - settings: ReturnType["settings"], -): React.CSSProperties { - const w = classifyWallpaper(settings.wallpaper); - if (w.kind === "color") return { backgroundColor: w.value }; - if (w.kind === "gradient") return { backgroundImage: w.value, backgroundSize: "cover" }; - const url = resolveWallpaperImageUrl(w.path); - if (!url) return {}; - return { - backgroundImage: `url(${url})`, - backgroundSize: "cover", - backgroundPosition: "center", - backgroundRepeat: "no-repeat", - }; -} - // Screen stage: rectangle from the composite layout, converted to percentages // of the canvas. Positions/sizes the interactive overlays (ZoomFocusOverlay, // AnnotationLayer) hosted inside it and the CSS-hidden