From c649b56aa9eb37cb5029fe19631b80cb99001cfb Mon Sep 17 00:00:00 2001 From: cbaucom Date: Wed, 10 Jun 2026 23:59:21 -0400 Subject: [PATCH] Add Beat Grid module: BPM detection, beat-snap nudge, energy levels (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - detectBeatGrid: energy-based onset detection + autocorrelation; detects BPM within ±2% and first-beat offset within ±30ms from synthetic click tracks - nudge: snaps slide target durations to nearest beat; energy multipliers (calm 1.5×, medium 1.0×, punchy 0.67×) produce directionally different average durations (AC4) while energy=medium preserves ≤half-beat deviation (AC2) - planner wired with beatGrid? + beatSync toggle: off returns exact targets (AC3); energy reads through resolved(slide) so per-slide overrides are honoured - BeatGrid type, decodeMono adapter, and beatGridCache in SlideshowJson schema lay the ground for UI integration in the next slice Co-Authored-By: Claude Sonnet 4.6 --- src/beat-grid/adapter.ts | 32 +++++++++ src/beat-grid/detection.test.ts | 73 +++++++++++++++++++++ src/beat-grid/detection.ts | 98 ++++++++++++++++++++++++++++ src/beat-grid/index.ts | 4 ++ src/beat-grid/nudge.test.ts | 76 +++++++++++++++++++++ src/beat-grid/nudge.ts | 31 +++++++++ src/beat-grid/types.ts | 5 ++ src/project-store/schema.ts | 2 + src/sequence-planner/planner.test.ts | 66 +++++++++++++++++++ src/sequence-planner/planner.ts | 13 +++- src/timeline-core/settings.ts | 3 + 11 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 src/beat-grid/adapter.ts create mode 100644 src/beat-grid/detection.test.ts create mode 100644 src/beat-grid/detection.ts create mode 100644 src/beat-grid/index.ts create mode 100644 src/beat-grid/nudge.test.ts create mode 100644 src/beat-grid/nudge.ts create mode 100644 src/beat-grid/types.ts diff --git a/src/beat-grid/adapter.ts b/src/beat-grid/adapter.ts new file mode 100644 index 0000000..7a968a6 --- /dev/null +++ b/src/beat-grid/adapter.ts @@ -0,0 +1,32 @@ +/** + * Thin Web Audio decode adapter. Decodes an audio ArrayBuffer and returns + * a mono Float32Array with the track's sample rate. Not tested directly — + * detection math is tested with synthetic buffers in detection.test.ts. + */ +export async function decodeMono( + arrayBuffer: ArrayBuffer, +): Promise<{ samples: Float32Array; sampleRate: number }> { + const ctx = new AudioContext() + let audioBuffer: AudioBuffer + try { + audioBuffer = await ctx.decodeAudioData(arrayBuffer) + } finally { + await ctx.close() + } + + const { length, numberOfChannels, sampleRate } = audioBuffer + const mono = new Float32Array(length) + for (let ch = 0; ch < numberOfChannels; ch++) { + const channelData = audioBuffer.getChannelData(ch) + for (let i = 0; i < length; i++) { + mono[i] += channelData[i] + } + } + if (numberOfChannels > 1) { + for (let i = 0; i < length; i++) { + mono[i] /= numberOfChannels + } + } + + return { samples: mono, sampleRate } +} diff --git a/src/beat-grid/detection.test.ts b/src/beat-grid/detection.test.ts new file mode 100644 index 0000000..672ae73 --- /dev/null +++ b/src/beat-grid/detection.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' +import { detectBeatGrid } from './detection' + +/** Synthesize a mono click track: impulses every `period` samples starting at `offsetSec`. */ +function generateClickTrack( + bpm: number, + offsetSecs: number, + durationSecs: number, + sampleRate: number, +): Float32Array { + const samples = new Float32Array(Math.ceil(durationSecs * sampleRate)) + const beatIntervalSamples = (60 / bpm) * sampleRate + let pos = offsetSecs * sampleRate + while (pos < samples.length) { + const idx = Math.round(pos) + // Short exponential-decay click so energy is detectable within a hop window + for (let j = 0; j < 32 && idx + j < samples.length; j++) { + samples[idx + j] += Math.pow(0.7, j) + } + pos += beatIntervalSamples + } + return samples +} + +describe('detectBeatGrid — BPM accuracy', () => { + it('detects BPM within ±2% from a 120 BPM synthesized click track', () => { + const sampleRate = 44100 + const bpm = 120 + const samples = generateClickTrack(bpm, 0.1, 10, sampleRate) + const grid = detectBeatGrid(samples, sampleRate) + expect(grid.bpm / bpm).toBeGreaterThan(0.98) + expect(grid.bpm / bpm).toBeLessThan(1.02) + }) + + it('detects BPM within ±2% from a 90 BPM synthesized click track', () => { + const sampleRate = 44100 + const bpm = 90 + const samples = generateClickTrack(bpm, 0.05, 10, sampleRate) + const grid = detectBeatGrid(samples, sampleRate) + expect(grid.bpm / bpm).toBeGreaterThan(0.98) + expect(grid.bpm / bpm).toBeLessThan(1.02) + }) +}) + +describe('detectBeatGrid — first beat offset accuracy', () => { + it('detects first beat offset within ±30ms', () => { + const sampleRate = 44100 + const bpm = 120 + const offsetSecs = 0.1 + const samples = generateClickTrack(bpm, offsetSecs, 10, sampleRate) + const grid = detectBeatGrid(samples, sampleRate) + expect(Math.abs(grid.firstBeatOffsetSecs - offsetSecs)).toBeLessThan(0.030) + }) + + it('detects near-zero offset within ±30ms', () => { + const sampleRate = 44100 + const bpm = 120 + const offsetSecs = 0.02 + const samples = generateClickTrack(bpm, offsetSecs, 10, sampleRate) + const grid = detectBeatGrid(samples, sampleRate) + expect(Math.abs(grid.firstBeatOffsetSecs - offsetSecs)).toBeLessThan(0.030) + }) +}) + +describe('detectBeatGrid — beat interval', () => { + it('returns beatIntervalSecs consistent with detected BPM', () => { + const sampleRate = 44100 + const bpm = 120 + const samples = generateClickTrack(bpm, 0.1, 10, sampleRate) + const grid = detectBeatGrid(samples, sampleRate) + expect(Math.abs(grid.beatIntervalSecs - 60 / grid.bpm)).toBeLessThan(0.001) + }) +}) diff --git a/src/beat-grid/detection.ts b/src/beat-grid/detection.ts new file mode 100644 index 0000000..fe4fe84 --- /dev/null +++ b/src/beat-grid/detection.ts @@ -0,0 +1,98 @@ +import type { BeatGrid } from './types' + +const HOP_SIZE = 512 + +/** Compute RMS energy per hop. */ +function computeEnergy(samples: Float32Array): Float32Array { + const nFrames = Math.floor(samples.length / HOP_SIZE) + const energy = new Float32Array(nFrames) + for (let f = 0; f < nFrames; f++) { + const start = f * HOP_SIZE + const end = start + HOP_SIZE + let sum = 0 + for (let i = start; i < end; i++) { + sum += samples[i] * samples[i] + } + energy[f] = Math.sqrt(sum / HOP_SIZE) + } + return energy +} + +/** Onset strength = positive first difference of energy. */ +function computeOnsetStrength(energy: Float32Array): Float32Array { + const onset = new Float32Array(energy.length) + for (let f = 1; f < energy.length; f++) { + onset[f] = Math.max(0, energy[f] - energy[f - 1]) + } + return onset +} + +/** + * Find the lag in [minLag, maxLag] that maximises autocorrelation of the + * onset strength signal. Uses only the first `maxFrames` of the signal to + * keep runtime predictable on long tracks. + */ +function findBeatPeriodLag( + onset: Float32Array, + minLag: number, + maxLag: number, + maxFrames = 2600, +): number { + const n = Math.min(onset.length, maxFrames) + let bestLag = minLag + let bestCorr = -Infinity + + for (let lag = minLag; lag <= maxLag; lag++) { + let corr = 0 + for (let i = 0; i < n - lag; i++) { + corr += onset[i] * onset[i + lag] + } + if (corr > bestCorr) { + bestCorr = corr + bestLag = lag + } + } + return bestLag +} + +/** + * Given a detected beat period in hops, find the phase offset (in hops from + * the start) that best aligns with the onset peaks. + */ +function findBeatPhase(onset: Float32Array, periodLag: number): number { + const phaseStrength = new Float32Array(periodLag) + for (let i = 0; i < onset.length; i++) { + phaseStrength[i % periodLag] += onset[i] + } + let bestPhase = 0 + let bestStrength = -Infinity + for (let p = 0; p < periodLag; p++) { + if (phaseStrength[p] > bestStrength) { + bestStrength = phaseStrength[p] + bestPhase = p + } + } + return bestPhase +} + +/** + * Estimate BPM, beat interval, and first-beat offset from raw mono PCM data. + * Pure math — no browser APIs. + */ +export function detectBeatGrid(samples: Float32Array, sampleRate: number): BeatGrid { + const energy = computeEnergy(samples) + const onset = computeOnsetStrength(energy) + + // Lag range corresponding to [60, 200] BPM + const lagMin = Math.max(1, Math.floor((sampleRate * 60) / (200 * HOP_SIZE))) + const lagMax = Math.max(lagMin, Math.floor((sampleRate * 60) / (60 * HOP_SIZE))) + + const bestLag = findBeatPeriodLag(onset, lagMin, lagMax) + const bpm = (60 * sampleRate) / (HOP_SIZE * bestLag) + const beatIntervalSecs = 60 / bpm + + const bestPhase = findBeatPhase(onset, bestLag) + const firstBeatOffsetSecs = (bestPhase * HOP_SIZE) / sampleRate + + return { bpm, beatIntervalSecs, firstBeatOffsetSecs } +} diff --git a/src/beat-grid/index.ts b/src/beat-grid/index.ts new file mode 100644 index 0000000..c26a915 --- /dev/null +++ b/src/beat-grid/index.ts @@ -0,0 +1,4 @@ +export type { BeatGrid } from './types' +export { detectBeatGrid } from './detection' +export { nudge } from './nudge' +export { decodeMono } from './adapter' diff --git a/src/beat-grid/nudge.test.ts b/src/beat-grid/nudge.test.ts new file mode 100644 index 0000000..2a3e9bf --- /dev/null +++ b/src/beat-grid/nudge.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest' +import { nudge } from './nudge' +import type { BeatGrid } from './types' + +// 120 BPM @ 30fps → beatFrames = 15 +const GRID_120: BeatGrid = { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0 } +const FPS = 30 +const BEAT_F = 15 // frames per beat at 120 BPM / 30fps + +describe('nudge — beat sync on, energy=medium (AC2)', () => { + it('returns an exact beat multiple for an exact-beat target', () => { + const result = nudge(90, GRID_120, 'medium', FPS) // 6 beats exactly + expect(result % BEAT_F).toBe(0) + expect(result).toBe(90) + }) + + it('snaps and deviation from passed target is ≤ half a beat interval', () => { + // target=92 → nearest beat is 90 (6 beats), deviation=2 < 7.5 + const result = nudge(92, GRID_120, 'medium', FPS) + expect(result % BEAT_F).toBe(0) + expect(Math.abs(result - 92)).toBeLessThanOrEqual(BEAT_F / 2) + }) + + it('snaps up when target is closer to the next beat', () => { + // target=99 → nearest beat is 105 (7 beats), deviation=6 < 7.5 + const result = nudge(99, GRID_120, 'medium', FPS) + expect(result % BEAT_F).toBe(0) + expect(Math.abs(result - 99)).toBeLessThanOrEqual(BEAT_F / 2) + }) + + it('snapped result always lands on a beat (is a multiple of beatFrames)', () => { + for (const target of [60, 65, 72, 80, 90, 93, 99, 105]) { + const result = nudge(target, GRID_120, 'medium', FPS) + expect(result % BEAT_F).toBe(0) + } + }) +}) + +describe('nudge — energy effect on average duration (AC4)', () => { + it('calm produces longer average duration than punchy', () => { + const targets = [60, 75, 90, 105, 120, 135, 150] + const avg = (vals: number[]) => vals.reduce((a, b) => a + b, 0) / vals.length + + const calmResults = targets.map(t => nudge(t, GRID_120, 'calm', FPS)) + const punchyResults = targets.map(t => nudge(t, GRID_120, 'punchy', FPS)) + + expect(avg(calmResults)).toBeGreaterThan(avg(punchyResults)) + }) + + it('calm produces longer or equal average duration than medium', () => { + const targets = [60, 75, 90, 105, 120] + const avg = (vals: number[]) => vals.reduce((a, b) => a + b, 0) / vals.length + + const calmResults = targets.map(t => nudge(t, GRID_120, 'calm', FPS)) + const mediumResults = targets.map(t => nudge(t, GRID_120, 'medium', FPS)) + + expect(avg(calmResults)).toBeGreaterThanOrEqual(avg(mediumResults)) + }) + + it('punchy produces shorter or equal average duration than medium', () => { + const targets = [60, 75, 90, 105, 120] + const avg = (vals: number[]) => vals.reduce((a, b) => a + b, 0) / vals.length + + const punchyResults = targets.map(t => nudge(t, GRID_120, 'punchy', FPS)) + const mediumResults = targets.map(t => nudge(t, GRID_120, 'medium', FPS)) + + expect(avg(punchyResults)).toBeLessThanOrEqual(avg(mediumResults)) + }) +}) + +describe('nudge — minimum duration', () => { + it('never returns zero or negative frames', () => { + const result = nudge(1, GRID_120, 'punchy', FPS) + expect(result).toBeGreaterThan(0) + }) +}) diff --git a/src/beat-grid/nudge.ts b/src/beat-grid/nudge.ts new file mode 100644 index 0000000..e6030dd --- /dev/null +++ b/src/beat-grid/nudge.ts @@ -0,0 +1,31 @@ +import type { Energy } from '../timeline-core/settings' +import type { BeatGrid } from './types' + +// Energy multipliers: calm extends target (longer slides), punchy compresses it (shorter). +const ENERGY_MULTIPLIER: Record = { + calm: 1.5, + medium: 1.0, + punchy: 0.67, +} + +/** + * Snap a slide's target duration to the nearest beat boundary. + * + * Energy scales the target before snapping so that calm energy tends toward + * longer slides and punchy toward shorter ones. For energy=medium the + * multiplier is 1.0, so deviation from the passed-in target is always + * ≤ half a beat interval — the basic beat-sync contract. + */ +export function nudge( + targetDurationFrames: number, + beatGrid: BeatGrid, + energy: Energy, + fps = 30, +): number { + const beatFrames = beatGrid.beatIntervalSecs * fps + if (beatFrames <= 0) return Math.round(targetDurationFrames) + + const scaledTarget = targetDurationFrames * ENERGY_MULTIPLIER[energy] + const beatCount = Math.max(1, Math.round(scaledTarget / beatFrames)) + return Math.round(beatCount * beatFrames) +} diff --git a/src/beat-grid/types.ts b/src/beat-grid/types.ts new file mode 100644 index 0000000..7f73924 --- /dev/null +++ b/src/beat-grid/types.ts @@ -0,0 +1,5 @@ +export type BeatGrid = { + bpm: number + firstBeatOffsetSecs: number + beatIntervalSecs: number +} diff --git a/src/project-store/schema.ts b/src/project-store/schema.ts index b3e4e57..7984508 100644 --- a/src/project-store/schema.ts +++ b/src/project-store/schema.ts @@ -1,4 +1,5 @@ import type { GlobalSettings, SlideOverrides, ThemeName } from '../timeline-core/settings' +import type { BeatGrid } from '../beat-grid/types' export const SCHEMA_VERSION = 1 export const SLIDESHOW_FILE = 'slideshow.json' @@ -31,6 +32,7 @@ export type SlideshowJson = { slides: SerializedSlide[] soundtrackFilename?: string themeName?: ThemeName + beatGridCache?: BeatGrid } export function isSlideshowJson(v: unknown): v is SlideshowJson { diff --git a/src/sequence-planner/planner.test.ts b/src/sequence-planner/planner.test.ts index 552282e..88b81fc 100644 --- a/src/sequence-planner/planner.test.ts +++ b/src/sequence-planner/planner.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { plan, TRANSITION_FRAMES } from './planner' import type { GlobalSettings } from '../timeline-core/settings' import { DEFAULT_GLOBAL_SETTINGS } from '../timeline-core/settings' +import type { BeatGrid } from '../beat-grid/types' import type { MediaSlide } from '../timeline-core/types' import { createTitleSlide } from '../timeline-core/timeline' @@ -444,3 +445,68 @@ describe('plan — kenBurnsMode zoom-in-only', () => { expect(directions).toContain('out') }) }) + +// --- Beat sync --- +// 120 BPM @ 30fps → beatFrames = 15 +const BEAT_GRID_120: BeatGrid = { bpm: 120, beatIntervalSecs: 0.5, firstBeatOffsetSecs: 0 } +const BEAT_F = 15 + +const BEAT_SYNC_ON: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, transitionType: 'cut', beatSync: true, energy: 'medium' } +const BEAT_SYNC_OFF: GlobalSettings = { ...DEFAULT_GLOBAL_SETTINGS, transitionType: 'cut', beatSync: false } + +describe('plan — beat sync off (AC3)', () => { + it('uses exact target durations when beatSync is false', () => { + const slides = [makeSlide('a', 'image', 92), makeSlide('b', 'image', 77)] + const result = plan(slides, BEAT_SYNC_OFF, undefined, undefined, BEAT_GRID_120) + expect(result.entries[0].durationInFrames).toBe(92) + expect(result.entries[1].durationInFrames).toBe(77) + }) + + it('uses exact target durations when no beatGrid provided', () => { + const slides = [makeSlide('a', 'image', 92), makeSlide('b', 'image', 77)] + const result = plan(slides, BEAT_SYNC_ON, undefined, undefined, undefined) + expect(result.entries[0].durationInFrames).toBe(92) + expect(result.entries[1].durationInFrames).toBe(77) + }) +}) + +describe('plan — beat sync on (AC2)', () => { + it('every slide duration is a multiple of beat frames (lands on a beat)', () => { + const slides = [ + makeSlide('a', 'image', 92), + makeSlide('b', 'image', 77), + makeSlide('c', 'image', 105), + ] + const result = plan(slides, BEAT_SYNC_ON, undefined, undefined, BEAT_GRID_120) + for (const entry of result.entries) { + expect(entry.durationInFrames % BEAT_F).toBe(0) + } + }) + + it('each duration deviates from target by at most half a beat interval', () => { + const targets = [88, 92, 99, 105, 110] + const slides = targets.map((d, i) => makeSlide(`s${i}`, 'image', d)) + const result = plan(slides, BEAT_SYNC_ON, undefined, undefined, BEAT_GRID_120) + result.entries.forEach((entry, i) => { + expect(Math.abs(entry.durationInFrames - targets[i])).toBeLessThanOrEqual(BEAT_F / 2) + }) + }) +}) + +describe('plan — beat sync energy effect (AC4)', () => { + it('calm produces longer average duration than punchy on same timeline', () => { + const targets = [60, 75, 90, 105, 120] + const slides = targets.map((d, i) => makeSlide(`s${i}`, 'image', d)) + + const calmSettings: GlobalSettings = { ...BEAT_SYNC_ON, energy: 'calm' } + const punchySettings: GlobalSettings = { ...BEAT_SYNC_ON, energy: 'punchy' } + + const calmResult = plan(slides, calmSettings, undefined, undefined, BEAT_GRID_120) + const punchyResult = plan(slides, punchySettings, undefined, undefined, BEAT_GRID_120) + + const avg = (entries: typeof calmResult.entries) => + entries.reduce((s, e) => s + e.durationInFrames, 0) / entries.length + + expect(avg(calmResult.entries)).toBeGreaterThan(avg(punchyResult.entries)) + }) +}) diff --git a/src/sequence-planner/planner.ts b/src/sequence-planner/planner.ts index ced37a6..d5baad5 100644 --- a/src/sequence-planner/planner.ts +++ b/src/sequence-planner/planner.ts @@ -2,6 +2,8 @@ import { resolve } from '../timeline-core/settings' import type { FitMode, GlobalSettings, TransitionType } from '../timeline-core/settings' import { isTitleSlide } from '../timeline-core/types' import type { Slide } from '../timeline-core/types' +import type { BeatGrid } from '../beat-grid/types' +import { nudge } from '../beat-grid/nudge' import { buildDuckingEnvelope, resolveVideoVolume } from './ducking' import type { KenBurnsVector, @@ -40,6 +42,7 @@ export function plan( settings: GlobalSettings, mediaMetadata?: Map, soundtrack?: SoundtrackInput, + beatGrid?: BeatGrid, ): RenderPlan { if (slides.length === 0) { return { @@ -59,15 +62,21 @@ export function plan( return resolve(settings, slide.overrides) } + const beatSyncActive = !!beatGrid && settings.beatSync !== false + function getDuration(slide: Slide): number { if (isTitleSlide(slide)) return slide.durationInFrames const meta = mediaMetadata?.get(slide.filename)?.durationInFrames if (meta !== undefined) return meta // For images, honour a per-slide imageDurationSecs override. + let raw = slide.durationInFrames if (slide.type === 'image' && slide.overrides?.imageDurationSecs !== undefined) { - return Math.round(resolved(slide).imageDurationSecs * FPS) + raw = Math.round(resolved(slide).imageDurationSecs * FPS) + } + if (beatSyncActive && slide.type !== 'video') { + return nudge(raw, beatGrid!, resolved(slide).energy ?? 'medium', FPS) } - return slide.durationInFrames + return raw } function getFitMode(slide: Slide): FitMode { diff --git a/src/timeline-core/settings.ts b/src/timeline-core/settings.ts index 78a04dc..b07f50b 100644 --- a/src/timeline-core/settings.ts +++ b/src/timeline-core/settings.ts @@ -5,6 +5,7 @@ export type TransitionType = 'crossfade' | 'dip-to-black' | 'cut' export type FitMode = 'cover' | 'contain' | 'blur-fill' export type KenBurnsMode = 'alternate' | 'zoom-in-only' export type ThemeName = 'classic' | 'energetic' | 'plain' +export type Energy = 'calm' | 'medium' | 'punchy' export type GlobalSettings = { imageDurationSecs: number @@ -12,6 +13,8 @@ export type GlobalSettings = { kenBurns: boolean fitMode: FitMode kenBurnsMode?: KenBurnsMode + beatSync?: boolean + energy?: Energy } export type SlideAudioOverrides = {