Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 48 additions & 1 deletion .github/workflows/ios-testflight.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,53 @@ on:
type: string

jobs:
audio-v2-release-gate:
if: github.event_name == 'push' || inputs.build_id == ''
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
working-directory: ./backend

steps:
- name: Setup repo
uses: actions/checkout@v4

- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Setup uv
uses: astral-sh/setup-uv@v4
with:
version: latest

- name: Install Opus runtime
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libopus0

- name: Install backend test dependencies
run: uv sync --locked --group test

- name: Verify Audio V2 phone path
run: >-
uv run --group test pytest
tests/test_audio_v2_websocket_entrypoint.py
tests/test_audio_protocol_v2.py
tests/test_audio_v2_ingress.py
tests/test_audio_v2_streams.py
tests/test_audio_persistence_lifecycle.py
tests/test_audio_durability.py
-q

build-and-submit:
needs: audio-v2-release-gate
if: >-
${{
always() &&
(needs.audio-v2-release-gate.result == 'success' ||
needs.audio-v2-release-gate.result == 'skipped')
}}
runs-on: macos-26
timeout-minutes: 120
defaults:
Expand Down Expand Up @@ -51,10 +97,11 @@ jobs:
run: |
npm run test:wearable-activation
npm run typecheck
npm run test:durable-audio-spool
npm run test:phone-audio-diagnostics
npm run test:push-notifications
npm run check:theme
npx --no-install expo-modules-autolinking verify --platform ios --verbose
swift test --package-path modules/chronicle-duplex-audio/ios

- name: Write and validate ASC API key
env:
Expand Down
2 changes: 1 addition & 1 deletion app/app.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"expo": {
"name": "chronicle",
"slug": "friend-lite-app",
"version": "1.14.0",
"version": "1.15.0",
"scheme": "chronicle",
"orientation": "portrait",
"icon": "./assets/icon.png",
Expand Down
33 changes: 7 additions & 26 deletions app/app/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,34 +38,24 @@ export default function App() {
// Bluetooth
const { bleManager, bluetoothState, permissionGranted, requestBluetoothPermission, isPermissionsLoading } = useBluetoothManager();

// Settings (must be before audioStreamer so the token refresh callback can reference it)
// Settings
const settings = useSharedAppSettings();

// Live backend reachability (Connection Doctor), re-probed on pull-to-refresh.
const { healthStatus, checkBackendHealth } = useBackendHealth(settings.webSocketUrl, settings.jwtToken);
const [refreshing, setRefreshing] = useState(false);

// Audio
const audioStreamer = useAudioStreamer({
autoReconnectEnabled: settings.autoReconnectEnabled,
onTokenRefreshed: (newToken) => {
// Update app-level auth state when auto-re-login refreshes the token
if (settings.currentUserEmail) {
settings.handleAuthStatusChange(true, settings.currentUserEmail, newToken);
}
},
});
const audioStreamer = useAudioStreamer();
const phoneAudioRecorder = usePhoneAudioRecorder();

const { isListeningAudio: isOmiAudioListenerActive, audioPacketsReceived, startAudioListener: originalStartAudioListener, stopAudioListener: originalStopAudioListener, isRetrying: isAudioListenerRetrying, retryAttempts: audioListenerRetryAttempts } = useAudioListener(omiConnection, () => !!deviceConnection.connectedDeviceId);

// Refs for disconnect cleanup
const isOmiAudioListenerActiveRef = useRef(isOmiAudioListenerActive);
const isAudioStreamingRef = useRef(audioStreamer.isStreaming);
// Track if audio pipeline was active before BLE disconnect (for auto-restart on reconnect)
const wasStreamingBeforeDisconnectRef = useRef(false);
useEffect(() => { isOmiAudioListenerActiveRef.current = isOmiAudioListenerActive; }, [isOmiAudioListenerActive]);
useEffect(() => { isAudioStreamingRef.current = audioStreamer.isStreaming; }, [audioStreamer.isStreaming]);

// Refs to break the declaration-order cycle:
// onDeviceConnect/onDeviceDisconnect need orchestrator + autoReconnect,
Expand Down Expand Up @@ -99,22 +89,13 @@ export default function App() {
}, [omiConnection]);

const onDeviceDisconnect = useCallback(async () => {
// Remember if audio was active so we can auto-restart on reconnect
if (isOmiAudioListenerActiveRef.current || isAudioStreamingRef.current) {
// BLE disconnect only owns the wearable pipeline. Phone capture is independent.
if (isOmiAudioListenerActiveRef.current) {
wasStreamingBeforeDisconnectRef.current = true;
await originalStopAudioListener();
await audioStreamer.stopStreaming();
}

// Stop audio listener (BLE is gone, can't read audio)
if (isOmiAudioListenerActiveRef.current) await originalStopAudioListener();

// Keep WebSocket alive — it will reconnect or idle until BLE comes back.
// Only stop WebSocket for phone audio mode (no BLE needed there).
if (phoneAudioRecorder.isRecording) {
audioStreamer.stopStreaming();
await phoneAudioRecorder.stopRecording();
orchestratorRef.current?.setIsPhoneAudioMode(false);
}
}, [originalStopAudioListener, audioStreamer.stopStreaming, phoneAudioRecorder.stopRecording, phoneAudioRecorder.isRecording]);
}, [originalStopAudioListener, audioStreamer.stopStreaming]);

const deviceConnection = useDeviceConnection(omiConnection, bleManager, onDeviceDisconnect, onDeviceConnect);

Expand Down
7 changes: 7 additions & 0 deletions app/app/settings.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import AuthSection from '@/components/AuthSection';
import BackendStatus from '@/components/BackendStatus';
import NetworkOverview from '@/components/NetworkOverview';
import NotificationsSection from '@/components/NotificationsSection';
import PhoneAudioDiagnosticsSection from '@/components/PhoneAudioDiagnosticsSection';
import SystemAdminControls from '@/components/SystemAdminControls';
import { Screen, SectionLabel } from '@/components/ui';
import { useSharedAppSettings } from '@/contexts/AppSettingsContext';
Expand All @@ -30,6 +31,12 @@ export default function SettingsScreen() {
authenticated={settings.isAuthenticated}
/>

<SectionLabel>Diagnostics</SectionLabel>
<PhoneAudioDiagnosticsSection
backendUrl={settings.webSocketUrl}
jwtToken={settings.jwtToken}
/>

<SectionLabel>Administration</SectionLabel>
<SystemAdminControls backendUrl={settings.webSocketUrl} jwtToken={settings.jwtToken} />
<NetworkOverview backendUrl={settings.webSocketUrl} />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ class ChronicleDuplexAudioModule : Module() {
"sampleRate" to 16_000,
"channels" to 1,
"frameDurationMs" to durationMs,
"audioLevel" to DuplexAudioPolicy.audioLevel(frame, count),
"opusBase64" to Base64.encodeToString(packet, Base64.NO_WRAP),
),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,21 @@ internal object DuplexAudioPolicy {
): Boolean = current != null &&
(responseId == "*" || responseId == current.id) &&
cancellationGeneration >= current.generation

fun audioLevel(pcm: ByteArray, byteCount: Int): Double {
val boundedBytes = minOf(byteCount, pcm.size)
val usableBytes = boundedBytes - (boundedBytes % 2)
if (usableBytes <= 0) return 0.0
var sumOfSquares = 0.0
var index = 0
while (index < usableBytes) {
val sample = ((pcm[index].toInt() and 0xff) or (pcm[index + 1].toInt() shl 8)).toShort()
val normalized = sample.toDouble() / 32_768.0
sumOfSquares += normalized * normalized
index += 2
}
return kotlin.math.min(1.0, kotlin.math.sqrt(sumOfSquares / (usableBytes / 2).toDouble()))
}
}

internal data class EpochResponse(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,14 @@ class DuplexAudioPolicyTest {
gate.schedule(EpochResponse("two", 1, 4))
}
}

@Test fun audioMeterReportsSilenceAndNormalizedPeak() {
assertEquals(0.0, DuplexAudioPolicy.audioLevel(ByteArray(640), 640), 0.001)
val halfScale = ByteArray(640)
for (index in halfScale.indices step 2) {
halfScale[index] = 0
halfScale[index + 1] = 64
}
assertEquals(0.5, DuplexAudioPolicy.audioLevel(halfScale, 640), 0.001)
}
}
63 changes: 63 additions & 0 deletions app/modules/chronicle-duplex-audio/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ import type { VoiceCapabilities } from '../../src/protocol/audioCapabilities';

export interface StartVoiceSessionOptions {
captureEpoch: number;
diagnosticProfile?:
| 'production'
| 'voice_processing_hold'
| 'plain_capture_hold'
| 'system_tap_format_hold';
}

export interface NativeOpusFrame {
Expand All @@ -18,9 +23,29 @@ export interface NativeOpusFrame {
sampleRate: 16000;
channels: 1;
frameDurationMs: number;
audioLevel: number;
opusBase64: string;
}

export interface NativeCaptureDiagnostic {
captureEpoch: number;
stage:
| 'tap_received'
| 'pcm_converted'
| 'pcm_conversion_failed'
| 'pcm_empty'
| 'opus_encoded'
| 'opus_encode_failed'
| 'voice_processing_fallback'
| 'capture_failed'
| 'system_change'
| 'watchdog_evaluated';
monotonicTimestampMs: number;
frameCount?: number;
byteCount?: number;
detail?: string;
}

export interface NativeResponse {
responseId: string;
generation: number;
Expand Down Expand Up @@ -48,15 +73,43 @@ export interface NativeStopResult {
failureCode: 'far_field_restore_failed' | 'permission_denied' | 'engine_unavailable' | null;
}

export interface NativeVoiceSessionDiagnostics {
diagnosticProfile: NonNullable<StartVoiceSessionOptions['diagnosticProfile']>;
captureEpoch: number;
engineRunning: boolean;
sessionRunning: boolean;
tapInstalled: boolean;
tapFrameCount: number;
convertedFrameCount: number;
opusPacketCount: number;
opusByteCount: number;
peakAudioLevel: number;
systemChangeCount: number;
lastSystemChangeReason: string;
watchdogEvaluationCount: number;
voiceProcessingEnabled: boolean;
audioSessionCategory: string;
audioSessionMode: string;
audioSessionSampleRate: number;
audioSessionIOBufferDurationMs: number;
inputFormat: string;
outputFormat: string;
}

type ChronicleDuplexAudioNative = NativeModule & {
startVoiceSession(options: StartVoiceSessionOptions): Promise<VoiceCapabilities>;
getVoiceSessionDiagnostics(): Promise<NativeVoiceSessionDiagnostics>;
scheduleResponse(response: NativeResponse): Promise<void>;
cancelResponse(responseId: string, generation: number): Promise<void>;
stopVoiceSession(): Promise<NativeStopResult>;
addListener(
eventName: 'onOpusFrame',
listener: (event: NativeOpusFrame) => void
): EventSubscription;
addListener(
eventName: 'onCaptureDiagnostic',
listener: (event: NativeCaptureDiagnostic) => void
): EventSubscription;
addListener(
eventName: 'onPlaybackState',
listener: (event: NativePlaybackState) => void
Expand Down Expand Up @@ -94,6 +147,16 @@ export function addOpusFrameListener(
return requireNative().addListener('onOpusFrame', listener);
}

export function getVoiceSessionDiagnostics(): Promise<NativeVoiceSessionDiagnostics> {
return requireNative().getVoiceSessionDiagnostics();
}

export function addCaptureDiagnosticListener(
listener: (event: NativeCaptureDiagnostic) => void
): EventSubscription {
return requireNative().addListener('onCaptureDiagnostic', listener);
}

export function addPlaybackStateListener(
listener: (event: NativePlaybackState) => void
): EventSubscription {
Expand Down
Loading
Loading