feat(camera): show the camera background on the crop thumbnail

crates/compositor-view-napi/src/lib.rs

@@ -87,6 +87,56 @@ pub fn segmentation_runtime_available() -> bool {
     openscreen_compositor::segmentation::runtime_available()
 }
 
+pub struct SegmentFrameTask {
+    model_path: String,
+    rgba: Vec<u8>,
+}
+
+impl Task for SegmentFrameTask {
+    type Output = Vec<u8>;
+    type JsValue = Buffer;
+
+    fn compute(&mut self) -> Result<Self::Output> {
+        let rgb: Vec<u8> = self
+            .rgba
+            .chunks_exact(4)
+            .flat_map(|p| [p[0], p[1], p[2]])
+            .collect();
+        // ponytail: une session ONNX chargée par appel (quelques dizaines de ms), parce qu'il
+        // n'y a qu'un appel par caméra et par ouverture du panneau. À garder en cache si un
+        // appelant se met à segmenter en continu.
+        let mut segmenter = openscreen_compositor::segmentation::Segmenter::load(
+            std::path::Path::new(&self.model_path),
+        )
+        .map_err(|e| Error::from_reason(format!("{e:#}")))?;
+        segmenter
+            .run(&rgb)
+            .map(<[u8]>::to_vec)
+            .map_err(|e| Error::from_reason(format!("{e:#}")))
+    }
+
+    fn resolve(&mut self, _env: Env, mask: Self::Output) -> Result<Self::JsValue> {
+        Ok(mask.into())
+    }
+}
+
+/// Masque du sujet pour UNE image, sans vue : la vignette de recadrage du panneau caméra s'en
+/// sert pour montrer le fond choisi sur sa propre frame.
+///
+/// `rgba` est une frame RGBA8 déjà réduite à la taille du modèle (`segmentation::MODEL_WIDTH` x
+/// `MODEL_HEIGHT`), soit le `getImageData` d'un canvas tel quel. Le retour a le format de
+/// `Segmenter::run` : un octet par pixel, 0 = fond, 255 = sujet.
+///
+/// `AsyncTask`, comme `remux_seekable` : charger la session prend plus longtemps que l'inférence,
+/// et ce temps n'a rien à faire sur le thread principal de Node.
+#[napi]
+pub fn segment_frame(model_path: String, rgba: Buffer) -> AsyncTask<SegmentFrameTask> {
+    AsyncTask::new(SegmentFrameTask {
+        model_path,
+        rgba: rgba.to_vec(),
+    })
+}
+
 #[napi]
 pub fn create_view(
     rect: CompositorViewRect,

electron/ipc/nativeBridge.ts

@@ -372,6 +372,10 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) {
 							return createSuccessResponse(requestId, {
 								support: compositorViewService.probeSegmentation(),
 							});
+						case "segmentFrame":
+							return createSuccessResponse(requestId, {
+								mask: await compositorViewService.segmentFrame(request.payload.rgba),
+							});
 						case "setRect":
 							compositorViewService.setRect(request.payload.id, request.payload.rect);
 							return createSuccessResponse(requestId, { ok: true });

electron/native-bridge/services/compositorViewService.ts

@@ -556,6 +556,20 @@ export class CompositorViewService {
 		return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model";
 	}
 
+	/** Subject mask for one camera frame, no view needed — see `segmentFrame` in the addon.
+	 *  `null` when the addon (or this version of it) or the model is missing. */
+	async segmentFrame(rgba: Uint8Array): Promise<Uint8Array | null> {
+		const addon = this.ensureAddon();
+		const modelPath = resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET);
+		if (!addon?.segmentFrame || !modelPath) {
+			return null;
+		}
+		return addon.segmentFrame(
+			modelPath,
+			Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength),
+		);
+	}
+
 	/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`.
 	 *  `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer
 	 *  keeps them on the wire so the existing `CompositorViewRect` shape stays

electron/native/compositor-view/addon.d.ts

@@ -123,6 +123,11 @@ export interface CompositorViewAddon {
 	 *  `--dir` build has none either. */
 	segmentationRuntimeAvailable(): boolean;
 
+	/** Subject mask for ONE frame, outside any view — the crop thumbnail shows the camera
+	 *  background with it. `rgba` is already at the model's size (256x144 RGBA8); resolves one
+	 *  byte per pixel, 0 = background, 255 = subject. Optional: an older `.node` predates it. */
+	segmentFrame?(modelPath: string, rgba: Buffer): Promise<Buffer>;
+
 	/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the
 	 *  target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side).
 	 *  No HWND/native-window-handle is passed: there's no OS window to parent to. The

src/components/ai-edition/NewEditorShell.module.css

@@ -350,7 +350,7 @@
 			var(--scene-foliage-d) 100%);
 	/* ponytail: the wallpaper card is a floating, rounded surface (per the v4
 	   design's preview frame) — rounded corners + a layered drop shadow lift it
-	   off the scene gradient. The custom background (buildFrameStyle) fills this
+	   off the scene gradient. The custom background (wallpaperStyle) fills this
 	   card; the video sits on top with its own padding so the wallpaper shows
 	   around it. */
 	border-radius: 16px;

src/components/ai-edition/PreviewCanvas.tsx

@@ -57,7 +57,7 @@ import {
 	type WebcamCompositeLayout,
 } from "@/lib/compositeLayout";
 import { webcamAnchorAt } from "@/lib/projectDefaults";
-import { classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
+import { wallpaperStyle } from "@/lib/wallpaper";
 import { getCssClipPath } from "@/lib/webcamMaskShapes";
 import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils";
 import { AnnotationLayer } from "./AnnotationLayer";
@@ -321,7 +321,7 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
 	// The stage hosting the interactive overlays is the CONTENT rect, and a frame never changes
 	// it: the compositor keeps the footage the same size with or without a frame and grows the
 	// frame outward, so every handle (annotations, privacy blur, zoom focus) sits on `layout`.
-	const frameStyle = useMemo(() => buildFrameStyle(settings), [settings]);
+	const frameStyle = useMemo(() => wallpaperStyle(settings.wallpaper), [settings.wallpaper]);
 	const screenStyle = useMemo(
 		() => buildScreenStyle(layout, settings, frameSize),
 		[layout, settings, frameSize],
@@ -509,38 +509,6 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
 	);
 }
 
-// ponytail: resolve `settings.wallpaper` to an actual CSS background. Image
-// wallpapers must go through resolveImageWallpaperUrl (→ getAssetPath): in the
-// packaged Electron app the renderer loads over file://, where a bare
-// `/wallpapers/foo.jpg` points at the filesystem root and 404s, so the custom
-// background silently failed to paint (worked in the http dev server only).
-// classifyWallpaper also handles color-function (rgb/hsl/…) and every gradient
-// variant, which the old ad-hoc startsWith checks missed.
-function resolveWallpaperImageUrl(imagePath: string): string | null {
-	try {
-		return resolveImageWallpaperUrl(imagePath);
-	} catch {
-		return null;
-	}
-}
-
-// Canvas: wallpaper only — no padding, no shadow.
-function buildFrameStyle(
-	settings: ReturnType<typeof useEditorSettings>["settings"],
-): React.CSSProperties {
-	const w = classifyWallpaper(settings.wallpaper);
-	if (w.kind === "color") return { backgroundColor: w.value };
-	if (w.kind === "gradient") return { backgroundImage: w.value, backgroundSize: "cover" };
-	const url = resolveWallpaperImageUrl(w.path);
-	if (!url) return {};
-	return {
-		backgroundImage: `url(${url})`,
-		backgroundSize: "cover",
-		backgroundPosition: "center",
-		backgroundRepeat: "no-repeat",
-	};
-}
-
 // Screen stage: rectangle from the composite layout, converted to percentages
 // of the canvas. Positions/sizes the interactive overlays (ZoomFocusOverlay,
 // AnnotationLayer) hosted inside it and the CSS-hidden <video> (decode/clock

src/components/ai-edition/RightPanes.layout.test.tsx

@@ -5,7 +5,7 @@
 // the saved preference is left untouched on disk and the help popover says so.
 
 import "@testing-library/jest-dom";
-import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";
+import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { I18nProvider } from "@/contexts/I18nContext";
@@ -14,6 +14,15 @@ import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema
 import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
 import { LayoutPane } from "./RightPanes";
 
+const segmentation = vi.hoisted(() => ({ supported: false, segmentCameraFrame: vi.fn() }));
+vi.mock("@/native/hooks/useSegmentationSupport", () => ({
+	useCanSegmentCamera: () => segmentation.supported,
+}));
+vi.mock("@/native/compositorViewClient", async (importOriginal) => ({
+	...(await importOriginal<typeof import("@/native/compositorViewClient")>()),
+	segmentCameraFrame: segmentation.segmentCameraFrame,
+}));
+
 function seedProject(hasCamera: boolean): AxcutDocument {
 	const base = createEmptyDocument({ projectId: "project_layout", title: "Layout" });
 	return {
@@ -154,6 +163,72 @@ describe("LayoutPane webcam framing picture", () => {
 	});
 });
 
+// The thumbnail shows the camera background the preview draws. The promise that makes it
+// cheap: nothing runs while the background is the camera's own, and the frame is segmented
+// once, every later change of mode, blur or wallpaper being CSS over the same mask.
+describe("LayoutPane webcam framing background", () => {
+	beforeEach(() => {
+		segmentation.supported = true;
+		segmentation.segmentCameraFrame.mockResolvedValue(new Uint8Array(256 * 144).fill(255));
+		// jsdom has no 2D canvas; the thumbnail only draws into one and reads it back.
+		vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+			drawImage: vi.fn(),
+			getImageData: (_x: number, _y: number, w: number, h: number) => ({
+				data: new Uint8ClampedArray(w * h * 4),
+			}),
+			createImageData: (w: number, h: number) => ({ data: new Uint8ClampedArray(w * h * 4) }),
+			putImageData: vi.fn(),
+		} as unknown as CanvasRenderingContext2D);
+		vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue("data:image/png;base64,AA");
+	});
+	afterEach(() => {
+		segmentation.supported = false;
+		segmentation.segmentCameraFrame.mockReset();
+		vi.restoreAllMocks();
+	});
+
+	const withBackground = (webcamBackgroundMode: string) => {
+		const doc = seedProject(true);
+		return { ...doc, legacyEditor: { ...doc.legacyEditor, webcamBackgroundMode } };
+	};
+	/** Loads the thumbnail's frame the way the browser would: metadata, then the seek. */
+	const showFrame = () => {
+		const video = document.querySelector('[class*="framingVideo"]') as HTMLVideoElement;
+		Object.defineProperty(video, "videoWidth", { configurable: true, value: 640 });
+		Object.defineProperty(video, "videoHeight", { configurable: true, value: 480 });
+		fireEvent.loadedMetadata(video);
+		fireEvent.seeked(video);
+		return video;
+	};
+	const backdrop = () =>
+		document.querySelector('[class*="framingBox"] > div[aria-hidden="true"]') as HTMLElement | null;
+
+	it("segments nothing while the camera keeps its own background", () => {
+		renderLayout(withBackground("none"));
+		const video = showFrame();
+
+		expect(segmentation.segmentCameraFrame).not.toHaveBeenCalled();
+		expect(video.style.maskImage).toBe("");
+		expect(backdrop()).toBeNull();
+	});
+
+	it("cuts the subject out once, then restyles the background around it", async () => {
+		renderLayout(withBackground("blur"));
+		const video = showFrame();
+
+		await waitFor(() => expect(video.style.maskImage).toContain("data:image/png"));
+		expect(backdrop()?.style.filter).toMatch(/^blur\(/);
+
+		fireEvent.click(screen.getByRole("button", { name: "Custom" }));
+		expect(backdrop()?.style.filter).toBe("");
+		fireEvent.click(screen.getByRole("button", { name: "Cutout" }));
+		expect(backdrop()).toBeNull();
+		expect(video.style.maskImage).toContain("data:image/png");
+
+		expect(segmentation.segmentCameraFrame).toHaveBeenCalledTimes(1);
+	});
+});
+
 describe("LayoutPane picture-in-picture camera", () => {
 	const stored = () => useProjectStore.getState().document?.legacyEditor as Record<string, unknown>;