Slice 11 — Video-audio ducking + per-slide audio overrides (#12)

specs/issue-12-video-audio-ducking.md

@@ -0,0 +1,46 @@
+# Issue 12 — Video-audio ducking + per-slide audio overrides
+
+## Scope
+
+Planner computes a soundtrack ducking envelope; Composition applies volume curves. Per-slide audio overrides live in `SlideSettingsDialog`.
+
+## Types
+
+### SlideOverrides (timeline-core)
+
+| Field | Type | Effect |
+| --- | --- | --- |
+| `muteMusic` | `boolean?` | Soundtrack silent for slide span |
+| `muteVideoAudio` | `boolean?` | Video plays muted; no ducking |
+| `musicVolume` | `number?` | Custom soundtrack level (0–1) during slide |
+| `videoVolume` | `number?` | Custom video audio level (0–1); default 1 |
+
+### RenderPlan (sequence-planner)
+
+- `SoundtrackTrack.duckingEnvelope`: `{ keyframes, rampFrames, segments }`
+- `RenderPlanEntry.videoVolume`: resolved 0–1 for video slides
+
+Constants: `DUCK_LEVEL = 0.2`, `DUCK_RAMP_FRAMES = 6`.
+
+## Ducking rules
+
+1. Video slide, default: duck music to 20% for slide span with 6-frame attack/release ramps.
+2. `muteVideoAudio`: music stays at full level; `videoVolume = 0`.
+3. `muteMusic`: music at 0 for slide span; video audio unchanged unless `videoVolume` set.
+4. `musicVolume` override replaces duck level (or sets level on non-video slides).
+5. `videoVolume` override sets video playback volume.
+
+## Composition
+
+- `Html5Audio` uses envelope keyframes interpolated by composition frame.
+- `Video` uses `volume` / `muted` from entry `videoVolume`.
+
+## UI
+
+`SlideSettingsDialog`: audio section on all slides; video-only mute-video control.
+
+## Tests
+
+- Golden: photo–video–photo produces one duck segment aligned to video entry.
+- Override: muteVideoAudio → no segment; muteMusic → keyframes at 0.
+- Property: duck segments exactly cover unmuted, non–mute-music video spans.

src/composition/SlideshowComposition.tsx

@@ -9,6 +9,7 @@ import type { RenderPlanEntry, RenderPlan } from '../sequence-planner/types'
 import type { TransitionType } from '../timeline-core/settings'
 import { isTitleSlide } from '../timeline-core/types'
 import type { MediaSlide, TitleSlide } from '../timeline-core/types'
+import { useSoundtrackVolume } from './soundtrackVolume'
 
 type MediaRenderPlanEntry = Omit<RenderPlanEntry, 'slide'> & { slide: MediaSlide }
 type TitleRenderPlanEntry = Omit<RenderPlanEntry, 'slide'> & { slide: TitleSlide }
@@ -62,7 +63,7 @@ export function SlideshowComposition({ plan }: SlideshowProps) {
   return (
     <AbsoluteFill>
       {plan.soundtrack ? (
-        <Html5Audio src={plan.soundtrack.blobUrl} volume={plan.soundtrack.volume} />
+        <SoundtrackAudio track={plan.soundtrack} />
       ) : null}
       <TransitionSeries>
       {plan.entries.map((entry) => (
@@ -87,6 +88,11 @@ export function SlideshowComposition({ plan }: SlideshowProps) {
   )
 }
 
+function SoundtrackAudio({ track }: { track: NonNullable<RenderPlan['soundtrack']> }) {
+  const volume = useSoundtrackVolume(track.duckingEnvelope)
+  return <Html5Audio src={track.blobUrl} volume={volume} />
+}
+
 function TitleSlideView({ entry }: { entry: TitleRenderPlanEntry }) {
   const { slide } = entry
   const bg = slide.style === 'light' ? '#fff' : '#111'
@@ -130,7 +136,7 @@ function TitleSlideView({ entry }: { entry: TitleRenderPlanEntry }) {
 }
 
 function MediaSlideView({ entry }: { entry: MediaRenderPlanEntry }) {
-  const { slide, fitMode, kenBurns, durationInFrames } = entry
+  const { fitMode, kenBurns, durationInFrames, slide, videoVolume } = entry
   const frame = useCurrentFrame()
 
   // Compute Ken Burns transform (images only; null for videos).
@@ -152,8 +158,10 @@ function MediaSlideView({ entry }: { entry: MediaRenderPlanEntry }) {
     return (
       <AbsoluteFill style={{ background: '#000' }}>
         <Video
+          muted={videoVolume === 0}
           src={slide.blobUrl}
-          style={{ width: '100%', height: '100%', objectFit: 'contain' }}
+          style={{ height: '100%', objectFit: 'contain', width: '100%' }}
+          volume={videoVolume}
         />
       </AbsoluteFill>
     )

src/composition/soundtrackVolume.ts

@@ -0,0 +1,45 @@
+import { interpolate, useCurrentFrame } from 'remotion'
+import type { DuckingEnvelope, VolumeKeyframe } from '../sequence-planner/types'
+
+export function volumeAtFrame(envelope: DuckingEnvelope, frame: number): number {
+  const { keyframes, rampFrames } = envelope
+  if (keyframes.length === 0) return 1
+
+  let active = keyframes[0]
+  let next = keyframes[0]
+
+  for (let index = 0; index < keyframes.length; index++) {
+    if (keyframes[index].frame <= frame) {
+      active = keyframes[index]
+      next = keyframes[Math.min(index + 1, keyframes.length - 1)]
+      continue
+    }
+    next = keyframes[index]
+    break
+  }
+
+  if (active.frame === next.frame || frame <= active.frame) {
+    return active.volume
+  }
+
+  const rampEndFrame = Math.min(active.frame + rampFrames, next.frame)
+  if (frame >= next.frame) {
+    return next.volume
+  }
+  if (frame <= rampEndFrame && active.volume !== next.volume) {
+    return interpolate(frame, [active.frame, rampEndFrame], [active.volume, next.volume], {
+      extrapolateLeft: 'clamp',
+      extrapolateRight: 'clamp',
+    })
+  }
+
+  return active.volume
+}
+
+export function useSoundtrackVolume(envelope: VolumeKeyframe[] | DuckingEnvelope, rampFrames = 6) {
+  const frame = useCurrentFrame()
+  const duckingEnvelope = Array.isArray(envelope)
+    ? { keyframes: envelope, rampFrames, segments: [] }
+    : envelope
+  return volumeAtFrame(duckingEnvelope, frame)
+}

src/editor-shell/SlideSettingsDialog.tsx

@@ -115,6 +115,98 @@ export function SlideSettingsDialog({ globalSettings, onClose, onOverride, slide
           </div>
         ) : null}
 
+        <div className="override-row">
+          <label className="override-label" htmlFor={`${slide.id}-mute-music`}>Mute music</label>
+          <input
+            checked={ov.muteMusic ?? false}
+            id={`${slide.id}-mute-music`}
+            onChange={e => {
+              if (e.target.checked) {
+                setField('muteMusic', true)
+                return
+              }
+              clearField('muteMusic')
+            }}
+            type="checkbox"
+          />
+          {ov.muteMusic !== undefined
+            ? <button className="override-reset" onClick={() => clearField('muteMusic')} title="Reset to global">↩</button>
+            : <span className="override-default">off</span>
+          }
+        </div>
+
+        <div className="override-row">
+          <label className="override-label" htmlFor={`${slide.id}-music-volume`}>Music level</label>
+          <input
+            className="settings-input"
+            disabled={ov.muteMusic === true}
+            id={`${slide.id}-music-volume`}
+            max={100}
+            min={0}
+            onChange={e => {
+              const value = Number(e.target.value) / 100
+              if (Number.isNaN(value)) return
+              setField('musicVolume', value)
+            }}
+            step={5}
+            type="range"
+            value={Math.round((ov.musicVolume ?? 1) * 100)}
+          />
+          <span className="settings-unit">{Math.round((ov.musicVolume ?? 1) * 100)}%</span>
+          {ov.musicVolume !== undefined
+            ? <button className="override-reset" onClick={() => clearField('musicVolume')} title="Reset to global">↩</button>
+            : <span className="override-default">auto</span>
+          }
+        </div>
+
+        {slide.type === 'video' ? (
+          <>
+            <div className="override-row">
+              <label className="override-label" htmlFor={`${slide.id}-mute-video-audio`}>Mute video audio</label>
+              <input
+                checked={ov.muteVideoAudio ?? false}
+                id={`${slide.id}-mute-video-audio`}
+                onChange={e => {
+                  if (e.target.checked) {
+                    setField('muteVideoAudio', true)
+                    return
+                  }
+                  clearField('muteVideoAudio')
+                }}
+                type="checkbox"
+              />
+              {ov.muteVideoAudio !== undefined
+                ? <button className="override-reset" onClick={() => clearField('muteVideoAudio')} title="Reset to global">↩</button>
+                : <span className="override-default">off</span>
+              }
+            </div>
+
+            <div className="override-row">
+              <label className="override-label" htmlFor={`${slide.id}-video-volume`}>Video level</label>
+              <input
+                className="settings-input"
+                disabled={ov.muteVideoAudio === true}
+                id={`${slide.id}-video-volume`}
+                max={100}
+                min={0}
+                onChange={e => {
+                  const value = Number(e.target.value) / 100
+                  if (Number.isNaN(value)) return
+                  setField('videoVolume', value)
+                }}
+                step={5}
+                type="range"
+                value={Math.round((ov.videoVolume ?? 1) * 100)}
+              />
+              <span className="settings-unit">{Math.round((ov.videoVolume ?? 1) * 100)}%</span>
+              {ov.videoVolume !== undefined
+                ? <button className="override-reset" onClick={() => clearField('videoVolume')} title="Reset to global">↩</button>
+                : <span className="override-default">100%</span>
+              }
+            </div>
+          </>
+        ) : null}
+
         <div className="slide-dialog-footer">
           {hasOverrides ? (
             <button className="override-reset-all" onClick={resetAll}>

src/sequence-planner/ducking.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, it } from 'vitest'
+import { DEFAULT_GLOBAL_SETTINGS } from '../timeline-core/settings'
+import type { MediaSlide } from '../timeline-core/types'
+import { DUCK_LEVEL, DUCK_RAMP_FRAMES, getUnmutedVideoAudioSpans, plan } from './planner'
+
+function makeSlide(
+  id: string,
+  type: 'image' | 'video',
+  durationInFrames: number,
+  overrides?: MediaSlide['overrides'],
+): MediaSlide {
+  return {
+    blobUrl: '',
+    durationInFrames,
+    excluded: false,
+    filename: `${id}.${type === 'image' ? 'jpg' : 'mp4'}`,
+    id,
+    overrides,
+    type,
+  }
+}
+
+const SETTINGS = { ...DEFAULT_GLOBAL_SETTINGS, transitionType: 'cut' as const }
+const SOUNDTRACK = { blobUrl: 'blob:audio', durationInFrames: 900 }
+
+describe('plan — soundtrack ducking envelope', () => {
+  it('ducks music during a video slide between photos', () => {
+    const photo = makeSlide('a', 'image', 90)
+    const video = makeSlide('v', 'video', 120)
+    const photo2 = makeSlide('b', 'image', 90)
+
+    const result = plan([photo, video, photo2], SETTINGS, undefined, SOUNDTRACK)
+    const envelope = result.soundtrack?.duckingEnvelope
+
+    expect(envelope).toBeDefined()
+    expect(envelope!.rampFrames).toBe(DUCK_RAMP_FRAMES)
+    expect(envelope!.segments).toEqual([
+      {
+        duckEndFrame: 90 + 120 - DUCK_RAMP_FRAMES - 1,
+        duckLevel: DUCK_LEVEL,
+        duckStartFrame: 90 + DUCK_RAMP_FRAMES,
+      },
+    ])
+
+    const holdKeyframes = envelope!.keyframes.filter((keyframe) => keyframe.volume === DUCK_LEVEL)
+    expect(holdKeyframes.length).toBeGreaterThan(0)
+    expect(envelope!.keyframes[0]).toEqual({ frame: 0, volume: 1 })
+  })
+
+  it('does not duck when video audio is muted', () => {
+    const video = makeSlide('v', 'video', 120, { muteVideoAudio: true })
+
+    const result = plan([video], SETTINGS, undefined, SOUNDTRACK)
+
+    expect(result.soundtrack?.duckingEnvelope.segments).toEqual([])
+    expect(result.entries[0].videoVolume).toBe(0)
+  })
+
+  it('mutes soundtrack when muteMusic is set', () => {
+    const video = makeSlide('v', 'video', 120, { muteMusic: true })
+
+    const result = plan([video], SETTINGS, undefined, SOUNDTRACK)
+    const { keyframes } = result.soundtrack!.duckingEnvelope
+
+    expect(result.soundtrack?.duckingEnvelope.segments).toEqual([])
+    expect(keyframes).toContainEqual({ frame: 0, volume: 0 })
+    expect(keyframes).toContainEqual({ frame: 120, volume: 1 })
+  })
+
+  it('uses custom musicVolume instead of default duck level', () => {
+    const video = makeSlide('v', 'video', 120, { musicVolume: 0.5 })
+
+    const result = plan([video], SETTINGS, undefined, SOUNDTRACK)
+
+    expect(result.soundtrack?.duckingEnvelope.segments).toEqual([])
+    expect(result.soundtrack?.duckingEnvelope.keyframes).toContainEqual({ frame: 0, volume: 0.5 })
+    expect(result.soundtrack?.duckingEnvelope.keyframes).toContainEqual({ frame: 120, volume: 1 })
+  })
+})
+
+describe('plan — ducking envelope property', () => {
+  it('duck segments exactly cover unmuted video-audio spans', () => {
+    const cases: MediaSlide[][] = [
+      [makeSlide('a', 'image', 60), makeSlide('v', 'video', 90), makeSlide('b', 'image', 45)],
+      [
+        makeSlide('v1', 'video', 30),
+        makeSlide('v2', 'video', 40, { muteVideoAudio: true }),
+        makeSlide('v3', 'video', 50, { muteMusic: true }),
+        makeSlide('v4', 'video', 70),
+      ],
+      [makeSlide('v', 'video', 8)],
+    ]
+
+    for (const slides of cases) {
+      const result = plan(slides, SETTINGS, undefined, SOUNDTRACK)
+      const envelope = result.soundtrack!.duckingEnvelope
+      const expectedSpans = getUnmutedVideoAudioSpans(result.entries).filter((span) => {
+        const entry = result.entries.find((candidate) => candidate.startFrame === span.startFrame)
+        return entry?.slide.overrides?.musicVolume === undefined
+      })
+
+      const actualSpans = envelope.segments.map((segment) => ({
+        endFrame: segment.duckEndFrame,
+        startFrame: segment.duckStartFrame,
+      }))
+
+      expect(actualSpans).toEqual(expectedSpans.map((span) => {
+        const duckStartFrame = span.startFrame + DUCK_RAMP_FRAMES
+        const duckEndFrame = span.endFrame - DUCK_RAMP_FRAMES
+        if (duckEndFrame < duckStartFrame) {
+          return { endFrame: span.endFrame, startFrame: span.startFrame }
+        }
+        return { endFrame: duckEndFrame, startFrame: duckStartFrame }
+      }))
+    }
+  })
+})