From bd8c6267bc0011609738fcf3d9db4372144eef30 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 21:58:31 +0100 Subject: [PATCH 01/10] Initial version 0.1.0. --- Makefile | 52 +- README.md | 14 + examples/example2.wav | 3 + examples/voice/README.md | 89 ++ examples/voice/file_transcription.py | 188 ++++ examples/voice/microphone_transcription.py | 140 +++ examples/voice/requirements-examples.txt | 2 + sdk/batch/speechmatics/batch/_transport.py | 6 +- sdk/voice/OVERVIEW.md | 281 ++++++ sdk/voice/README.md | 105 +++ sdk/voice/pyproject.toml | 67 ++ sdk/voice/speechmatics/__init__.py | 1 + sdk/voice/speechmatics/voice/__init__.py | 59 ++ sdk/voice/speechmatics/voice/_client.py | 945 +++++++++++++++++++++ sdk/voice/speechmatics/voice/_logging.py | 52 ++ sdk/voice/speechmatics/voice/_models.py | 793 +++++++++++++++++ tests/voice/test_setup.py | 2 + 17 files changed, 2784 insertions(+), 15 deletions(-) create mode 100644 examples/example2.wav create mode 100644 examples/voice/README.md create mode 100644 examples/voice/file_transcription.py create mode 100644 examples/voice/microphone_transcription.py create mode 100644 examples/voice/requirements-examples.txt create mode 100644 sdk/voice/OVERVIEW.md create mode 100644 sdk/voice/README.md create mode 100644 sdk/voice/pyproject.toml create mode 100644 sdk/voice/speechmatics/__init__.py create mode 100644 sdk/voice/speechmatics/voice/__init__.py create mode 100644 sdk/voice/speechmatics/voice/_client.py create mode 100644 sdk/voice/speechmatics/voice/_logging.py create mode 100644 sdk/voice/speechmatics/voice/_models.py create mode 100644 tests/voice/test_setup.py diff --git a/Makefile b/Makefile index 575f3514..7bb6694b 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ # Makefile for Speechmatics Python SDKs .PHONY: help -.PHONY: test-all test-rt test-batch test-flow -.PHONY: format-all format-rt format-batch format-flow -.PHONY: lint-all lint-rt lint-batch lint-flow -.PHONY: type-check-all type-check-rt type-check-batch type-check-flow -.PHONY: build-all build-rt build-batch build-flow -.PHONY: clean-all clean-rt clean-batch clean-flow clean-flow +.PHONY: test-all test-rt test-batch test-flow test-voice +.PHONY: format-all format-rt format-batch format-flow format-voice +.PHONY: lint-all lint-rt lint-batch lint-flow lint-voice +.PHONY: type-check-all type-check-rt type-check-batch type-check-flow type-check-voice +.PHONY: build-all build-rt build-batch build-flow build-voice +.PHONY: clean-all clean-rt clean-batch clean-flow clean-voice help: @echo "Available commands:" @@ -16,40 +16,46 @@ help: @echo " test-rt Run tests for RT SDK" @echo " test-batch Run tests for Batch SDK" @echo " test-flow Run tests for Flow SDK" + @echo " test-voice Run tests for Voice Agent SDK" @echo "" @echo "Code formatting:" @echo " format-all Auto-fix formatting for all SDKs" @echo " format-rt Auto-fix formatting for RT SDK" @echo " format-batch Auto-fix formatting for Batch SDK" @echo " format-flow Auto-fix formatting for Flow SDK" + @echo " format-voice Auto-fix formatting for Voice Agent SDK" @echo "" @echo "Linting:" @echo " lint-all Run linting for all SDKs" @echo " lint-rt Run linting for RT SDK" @echo " lint-batch Run linting for Batch SDK" @echo " lint-flow Run linting for Flow SDK" + @echo " lint-voice Run linting for Voice Agent SDK" @echo "" @echo "Type checking:" @echo " type-check-all Run type checking for all SDKs" @echo " type-check-rt Run type checking for RT SDK" @echo " type-check-batch Run type checking for Batch SDK" @echo " type-check-flow Run type checking for Flow SDK" + @echo " type-check-voice Run type checking for Voice Agent SDK" @echo "" @echo "Building:" @echo " build-all Build all SDKs" @echo " build-rt Build RT SDK" @echo " build-batch Build Batch SDK" @echo " build-flow Build Flow SDK" + @echo " build-voice Build Voice Agent SDK" @echo "" @echo "Cleaning:" @echo " clean-all Clean all SDKs" @echo " clean-rt Clean RT SDK build artifacts" @echo " clean-batch Clean Batch SDK build artifacts" @echo " clean-flow Clean Flow SDK build artifacts" + @echo " clean-voice Clean Voice Agent SDK build artifacts" @echo "" # Testing targets -test-all: test-rt test-batch test-flow +test-all: test-rt test-batch test-flow test-voice test-rt: pytest tests/rt/ -v @@ -60,8 +66,11 @@ test-batch: test-flow: pytest tests/flow/ -v +test-voice: + pytest tests/voice/ -v + # Formatting targets -format-all: format-rt format-batch format-flow +format-all: format-rt format-batch format-flow format-voice format-rt: cd sdk/rt/speechmatics && black . @@ -75,8 +84,12 @@ format-flow: cd sdk/flow/speechmatics && black . cd sdk/flow/speechmatics && ruff check --fix . +format-voice: + cd sdk/voice/speechmatics && black . + cd sdk/voice/speechmatics && ruff check --fix . + # Linting targets -lint-all: lint-rt lint-batch lint-flow +lint-all: lint-rt lint-batch lint-flow lint-voice lint-rt: cd sdk/rt/speechmatics && ruff check . @@ -87,8 +100,11 @@ lint-batch: lint-flow: cd sdk/flow/speechmatics && ruff check . +lint-voice: + cd sdk/voice/speechmatics && ruff check . + # Type checking targets -type-check-all: type-check-rt type-check-batch type-check-flow +type-check-all: type-check-rt type-check-batch type-check-flow type-check-voice type-check-rt: cd sdk/rt/speechmatics && mypy . @@ -99,18 +115,22 @@ type-check-batch: type-check-flow: cd sdk/flow/speechmatics && mypy . +type-check-voice: + cd sdk/voice/speechmatics && mypy . + # Installation targets install-dev: python -m pip install --upgrade pip python -m pip install -e sdk/rt[dev] python -m pip install -e sdk/batch[dev] python -m pip install -e sdk/flow[dev] + python -m pip install -e sdk/voice[dev] install-build: python -m pip install --upgrade build # Building targets -build-all: build-rt build-batch build-flow +build-all: build-rt build-batch build-flow build-voice build-rt: install-build cd sdk/rt && python -m build @@ -121,8 +141,11 @@ build-batch: install-build build-flow: install-build cd sdk/flow && python -m build +build-voice: install-build + cd sdk/voice && python -m build + # Cleaning targets -clean-all: clean-rt clean-batch clean-flow +clean-all: clean-rt clean-batch clean-flow clean-voice clean-rt: rm -rf sdk/rt/dist sdk/rt/build sdk/rt/*.egg-info @@ -135,3 +158,8 @@ clean-batch: clean-flow: rm -rf sdk/flow/dist sdk/flow/build sdk/flow/*.egg-info find sdk/flow -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + +clean-voice: + rm -rf sdk/voice/dist sdk/voice/build sdk/voice/*.egg-info + find sdk/voice -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + \ No newline at end of file diff --git a/README.md b/README.md index 4d085317..bf660863 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,14 @@ An async Python client for Speechmatics Flow API. pip install speechmatics-flow ``` +### (Beta) Voice Agent Client (`speechmatics-voice`) + +A Voice Agent Python client for Speechmatics Real-Time API. + +```bash +pip install speechmatics-voice +``` + ## Development ### Repository Structure @@ -53,10 +61,15 @@ speechmatics-python-sdk/ │ │ ├── pyproject.toml │ │ └── README.md │ +│ ├── voice/ +│ │ ├── pyproject.toml +│ │ └── README.md +│ ├── tests/ │ ├── batch/ │ └── rt/ │ └── flow/ +│ └── voice/ │ ├── examples/ ├── Makefile @@ -97,6 +110,7 @@ Each package can be installed separately: pip install speechmatics-rt pip install speechmatics-batch pip install speechmatics-flow +pip install speechmatics-voice ``` ## Docs diff --git a/examples/example2.wav b/examples/example2.wav new file mode 100644 index 00000000..67209392 --- /dev/null +++ b/examples/example2.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c744f7e3f3d45fc1d64000670da99c223bcdec348cbf014385eaf98b0913016f +size 901352 diff --git a/examples/voice/README.md b/examples/voice/README.md new file mode 100644 index 00000000..c95de58a --- /dev/null +++ b/examples/voice/README.md @@ -0,0 +1,89 @@ +# Speechmatics Voice Agent SDK Examples + +This directory contains practical demonstrations of the Speechmatics Voice Agent SDK for real-time speech-to-text applications. + +## Installation + +```bash +pip install -r requirements-examples.txt +``` + +**Requirements:** + +- Python 3.9+ +- PyAudio for audio I/O +- Valid Speechmatics API key in `SPEECHMATICS_API_KEY` environment variable + +## Examples + +### File Transcription + +An example that takes an audio file and sends it in real-time to the SDK and outputs the interim and final transcription to the console. The audio will also be played through the selected audio output device. + +Audio files should be in Mono 16kHz 16-bit WAV format. + +```bash +cd examples/voice +pip install -r requirements-examples.txt + +# Transcribe the default example audio file +python file_transcription.py + +# Transcribe a custom audio file (16kHz Mono 16-bit WAV) +python file_transcription.py path/to/your/audio.wav +``` + +**Features:** + +- File streaming +- Audio playback through selected device +- Real-time transcription +- Speaker diarization +- Speaker start / stop events + +### Microphone Transcription + +An example that takes audio from a microphone and sends it in real-time to the SDK and outputs the interim and final transcription to the console. + +```bash +cd examples/voice +pip install -r requirements-examples.txt + +# Real-time microphone transcription +python microphone_transcription.py +``` + +**Features:** + +- Live microphone input with device selection +- Real-time transcription +- Speaker diarization +- Speaker start / stop events + +## Troubleshooting + +**PyAudio installation issues:** + +```bash +# macOS +brew install portaudio +pip install pyaudio + +# Ubuntu/Debian +sudo apt-get install portaudio19-dev +pip install pyaudio +``` + +**No audio devices found:** + +- Ensure PyAudio is properly installed +- Check system audio permissions +- Verify audio devices are connected and recognised + +**API key issues:** + +```bash +export SPEECHMATICS_API_KEY="your_api_key_here" +``` + +For more information, see the main SDK documentation in [OVERVIEW.md](../OVERVIEW.md). diff --git a/examples/voice/file_transcription.py b/examples/voice/file_transcription.py new file mode 100644 index 00000000..c6ea5e1f --- /dev/null +++ b/examples/voice/file_transcription.py @@ -0,0 +1,188 @@ +import asyncio +import sys +import wave +from pathlib import Path + +from colorama import init as colorama_init +from utils import AudioPlayer +from utils import CustomLevels +from utils import get_logger +from utils import select_audio_output_device + +from speechmatics.voice import AgentServerMessageType +from speechmatics.voice import DiarizationSpeakerConfig +from speechmatics.voice import DiarizationFocusMode +from speechmatics.voice import EndOfUtteranceMode +from speechmatics.voice import SpeakerSegment +from speechmatics.voice import VoiceAgentClient +from speechmatics.voice import VoiceAgentConfig + +colorama_init(autoreset=True) +logger = get_logger(__name__) + + +async def main() -> None: + """ + Transcribe audio files with real-time playback and speaker diarisation. + + This example demonstrates: + - Loading and validating WAV audio files + - Real-time audio playback during transcription + - Speaker diarisation with voice activity detection + - Comprehensive event handling and logging + """ + # Parse command line arguments for audio file path + # Default to example.wav if no file specified + audio_file_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parent / "../example2.wav" + + # Validate that the audio file exists + if not audio_file_path.exists(): + print(f"Error: Audio file not found: {audio_file_path}") + print("Usage: python file_transcription.py [audio_file.wav]") + return + + # Load and validate the audio file format + # Only mono 16-bit WAV files are supported + try: + sample_rate, channels, sample_width, duration = _load_audio_file(audio_file_path) + print(f"Loading: {audio_file_path.name} ({duration:.1f}s, {sample_rate}Hz)") + except (wave.Error, ValueError) as e: + print(f"Error: {e}") + return + + # Setup audio playback device for real-time playback + # User can select from available output devices + audio_player = _setup_audio_player(sample_rate, channels, sample_width) + + # Configure Voice Agent with transcription settings + config = VoiceAgentConfig( + sample_rate=sample_rate, + end_of_utterance_silence_trigger=0.5, + enable_diarization=True, + end_of_utterance_mode=EndOfUtteranceMode.ADAPTIVE, + speaker_config=DiarizationSpeakerConfig(focus_speakers=["S2"], focus_mode=DiarizationFocusMode.IGNORE), + ) + + # Create Voice Agent client and start transcription + async with VoiceAgentClient(config=config) as client: + # Register event handlers for transcription events + _register_event_handlers(client, logger) + + try: + print("Starting transcription... (Press Ctrl+C to stop)\n") + # Connect to the Voice Agent service + await client.connect() + + # Stream the audio file with real-time playback + await _stream_audio_file(audio_file_path, client, audio_player, sample_rate, sample_width) + except asyncio.CancelledError: + print("\nTranscription cancelled") + except Exception as e: + print(f"Error: {e}") + finally: + # Clean up audio playback resources + if audio_player: + audio_player.stop() + + +def _load_audio_file(file_path: Path) -> tuple[int, int, int, float]: + """Load and validate audio file. + + Returns: + Tuple of (sample_rate, channels, sample_width, duration) + """ + with wave.open(str(file_path), "rb") as wav_file: + sample_rate = wav_file.getframerate() + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + frames = wav_file.getnframes() + duration = frames / sample_rate + + if channels != 1: + raise ValueError("Only mono audio files are supported") + if sample_width != 2: + raise ValueError("Only 16-bit audio files are supported") + + return sample_rate, channels, sample_width, duration + + +def _setup_audio_player(sample_rate: int, channels: int, sample_width: int) -> AudioPlayer | None: + """Setup audio player with device selection.""" + print("\nSelect audio output device for playback:") + output_device = select_audio_output_device() + + audio_player = AudioPlayer(sample_rate, channels, sample_width, output_device) + if not audio_player.start(): + print("Warning: Audio playback unavailable - continuing with transcription only") + return None + return audio_player + + +def _register_event_handlers(client: VoiceAgentClient, logger) -> None: + """Register event handlers for transcription events with detailed logging.""" + + def _format_segment(segment: SpeakerSegment) -> str: + """Format speaker segment for display.""" + template = "@{speaker_id} -> {text}" if segment.is_active else "@{speaker_id} (background) -> {text}" + return segment.format_text(template) + + @client.on(AgentServerMessageType.ADD_SEGMENTS) + def handle_final_segments(message): + """Handle final transcription segments.""" + segments = [_format_segment(s) for s in message["segments"]] + logger.log(CustomLevels.FINAL, f"🚀 {segments}") + + @client.on(AgentServerMessageType.ADD_INTERIM_SEGMENTS) + def handle_partial_segments(message): + """Handle partial transcription segments.""" + segments = [_format_segment(s) for s in message["segments"]] + delay = message.get("delay_to_finalize") + if delay is not None: + logger.log(CustomLevels.PARTIAL, f"⚡ {segments} ({delay}s to finals)") + else: + logger.log(CustomLevels.PARTIAL, f"⚡ {segments}") + + @client.on(AgentServerMessageType.USER_SPEECH_STARTED) + def handle_speech_started(message): + """Handle speech start events.""" + logger.log(CustomLevels.SPEAKER, f"✅ Speech started: {message}") + + @client.on(AgentServerMessageType.USER_SPEECH_ENDED) + def handle_speech_ended(message): + """Handle speech end events.""" + logger.log(CustomLevels.SPEAKER, f"🛑 Speech ended: {message}") + + @client.on(AgentServerMessageType.TTFB_METRICS) + def handle_metrics(message): + """Handle time-to-first-byte metrics.""" + logger.debug(f"📊 Metrics: {message}") + + +async def _stream_audio_file( + file_path: Path, + client: VoiceAgentClient, + audio_player: AudioPlayer | None, + sample_rate: int, + sample_width: int, +) -> None: + """Stream audio file in real-time to client and player.""" + CHUNK_SIZE = 320 + chunk_duration = CHUNK_SIZE / sample_rate / sample_width + + with wave.open(str(file_path), "rb") as wav_file: + while True: + audio_data = wav_file.readframes(CHUNK_SIZE) + if not audio_data: + print("\nEnd of file reached") + break + + # Send to transcription and play audio simultaneously + asyncio.create_task(client.send_audio(audio_data)) + if audio_player: + audio_player.play(audio_data) + + await asyncio.sleep(chunk_duration) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/voice/microphone_transcription.py b/examples/voice/microphone_transcription.py new file mode 100644 index 00000000..6f195e96 --- /dev/null +++ b/examples/voice/microphone_transcription.py @@ -0,0 +1,140 @@ +import asyncio + +from colorama import init as colorama_init +from utils import CustomLevels +from utils import get_logger +from utils import select_audio_device + +from speechmatics.rt import Microphone +from speechmatics.voice import AgentServerMessageType +from speechmatics.voice import DiarizationSpeakerConfig +from speechmatics.voice import EndOfUtteranceMode +from speechmatics.voice import SpeakerSegment +from speechmatics.voice import VoiceAgentClient +from speechmatics.voice import VoiceAgentConfig + +colorama_init(autoreset=True) + +logger = get_logger(__name__) + + +async def main() -> None: + """ + Live microphone transcription with speaker diarisation. + + This example demonstrates: + - Real-time microphone input with device selection + - Speaker diarisation and voice activity detection + - Performance metrics (TTFB) + - Works with any PyAudio-compatible input device + """ + + # Audio configuration for real-time processing + sample_rate = 16000 + chunk_size = 320 + + # Setup microphone with user device selection + mic = _setup_microphone(sample_rate, chunk_size) + if not mic: + return + + # Configure Voice Agent with microphone-specific settings + config = VoiceAgentConfig( + sample_rate=sample_rate, + end_of_utterance_silence_trigger=0.5, + enable_diarization=True, + end_of_utterance_mode=EndOfUtteranceMode.ADAPTIVE, + speaker_config=DiarizationSpeakerConfig(focus_speakers=["S1"]), + ) + + # Create Voice Agent client and start transcription + async with VoiceAgentClient(config=config) as client: + # Register event handlers for real-time transcription events + _register_event_handlers(client, logger) + + try: + print("Microphone ready - speak now... (Press Ctrl+C to stop)\n") + # Connect to the Voice Agent service + await client.connect() + + # Start streaming microphone audio + await _stream_microphone(mic, client, chunk_size) + except asyncio.CancelledError: + print("\nTranscription cancelled") + except Exception as e: + print(f"Error: {e}") + + +def _register_event_handlers(client: VoiceAgentClient, logger) -> None: + """Register event handlers for real-time transcription events.""" + + def _format_segment(segment: SpeakerSegment) -> str: + """Format speaker segment for display.""" + template = ( + "@{speaker_id}: {text}" + if segment.is_active + else "@{speaker_id} (background): {text}" + ) + return segment.format_text(template) + + @client.on(AgentServerMessageType.ADD_SEGMENTS) + def handle_final_segments(message): + """Handle final transcription segments.""" + segments = [_format_segment(s) for s in message["segments"]] + logger.log(CustomLevels.FINAL, f"🚀 Final: {segments}") + + @client.on(AgentServerMessageType.ADD_INTERIM_SEGMENTS) + def handle_partial_segments(message): + """Handle partial transcription segments.""" + segments = [_format_segment(s) for s in message["segments"]] + delay = message.get("delay_to_finalize") + if delay is not None: + logger.log( + CustomLevels.PARTIAL, f"💬 Partial: {segments} ({delay}s to finals)" + ) + else: + logger.log(CustomLevels.PARTIAL, f"💬 Partial: {segments}") + + @client.on(AgentServerMessageType.USER_SPEECH_STARTED) + def handle_speech_started(message): + """Handle speech start events.""" + logger.log(CustomLevels.SPEAKER, f"✅ Speech started: {message}") + + @client.on(AgentServerMessageType.USER_SPEECH_ENDED) + def handle_speech_ended(message): + """Handle speech end events.""" + logger.log(CustomLevels.SPEAKER, f"🛑 Speech ended: {message}") + + @client.on(AgentServerMessageType.TTFB_METRICS) + def handle_metrics(message): + """Handle time-to-first-byte metrics.""" + logger.debug(f"📊 Metrics: {message}") + + +def _setup_microphone(sample_rate: int, chunk_size: int) -> Microphone | None: + """Setup microphone with device selection.""" + selected_device = select_audio_device() + + mic = Microphone( + sample_rate=sample_rate, + chunk_size=chunk_size, + device_index=selected_device, + ) + + if not mic.start(): + print("Error: PyAudio not available - install with: pip install pyaudio") + return None + return mic + + +async def _stream_microphone( + mic: Microphone, client: VoiceAgentClient, chunk_size: int +) -> None: + """Stream microphone audio to client.""" + while True: + frame = await mic.read(chunk_size) + await client.send_audio(frame) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/voice/requirements-examples.txt b/examples/voice/requirements-examples.txt new file mode 100644 index 00000000..593081ce --- /dev/null +++ b/examples/voice/requirements-examples.txt @@ -0,0 +1,2 @@ +pyaudio +colorama diff --git a/sdk/batch/speechmatics/batch/_transport.py b/sdk/batch/speechmatics/batch/_transport.py index cfe65ff3..bab82750 100644 --- a/sdk/batch/speechmatics/batch/_transport.py +++ b/sdk/batch/speechmatics/batch/_transport.py @@ -297,9 +297,9 @@ async def _prepare_headers(self) -> dict[str, str]: Headers dictionary with authentication and tracking info """ auth_headers = await self._auth.get_auth_headers() - auth_headers[ - "User-Agent" - ] = f"speechmatics-batch-v{get_version()} python/{sys.version_info.major}.{sys.version_info.minor}" + auth_headers["User-Agent"] = ( + f"speechmatics-batch-v{get_version()} python/{sys.version_info.major}.{sys.version_info.minor}" + ) if self._request_id: auth_headers["X-Request-Id"] = self._request_id diff --git a/sdk/voice/OVERVIEW.md b/sdk/voice/OVERVIEW.md new file mode 100644 index 00000000..4ad6cc7f --- /dev/null +++ b/sdk/voice/OVERVIEW.md @@ -0,0 +1,281 @@ +# Speechmatics Voice Agent Client Overview + +The Speechmatics Voice Agent Client provides a comprehensive set of utilities for building voice agents with real-time speech-to-text capabilities, speaker diarization, and advanced transcription processing. + +## Key Features + +- **Real-time transcription** with partial and final results +- **Speaker diarization** with configurable focus modes +- **Voice activity detection** (VAD) with adaptive end-of-utterance detection +- **Event-driven architecture** with callback registration +- **TTFB (Time to First Byte) metrics** for performance monitoring + +## Installation + +```bash +pip install speechmatics-voice +``` + +## Quick Start + +### Basic Voice Agent Setup + +```python +import asyncio +from speechmatics.rt import Microphone +from speechmatics.voice import ( + VoiceAgentClient, + VoiceAgentConfig, + EndOfUtteranceMode, + AgentServerMessageType, + SpeakerSegment, +) + +async def main(): + # Configure the voice agent + config = VoiceAgentConfig( + end_of_utterance_silence_trigger=0.5, + enable_diarization=True, + end_of_utterance_mode=EndOfUtteranceMode.ADAPTIVE, + ) + + # Initialize microphone + mic = Microphone( + sample_rate=16000, + chunk_size=160, + ) + + if not mic.start(): + print("Microphone not available") + return + + # Create client and register event handlers + async with VoiceAgentClient(config=config) as client: + + # Handle interim transcription segments + @client.on(AgentServerMessageType.ADD_INTERIM_SEGMENTS) + def handle_interim_segments(message): + segments: list[SpeakerSegment] = message["segments"] + for segment in segments: + print(f"Speaker {segment.speaker_id}: {segment.text}") + + # Handle finalized transcription segments + @client.on(AgentServerMessageType.ADD_SEGMENTS) + def handle_final_segments(message): + segments: list[SpeakerSegment] = message["segments"] + for segment in segments: + print(f"Speaker {segment.speaker_id}: {segment.text}") + + # Handle user started speaking event + @client.on(AgentServerMessageType.USER_SPEECH_STARTED) + def handle_speech_started(message): + print("User started speaking") + + # Handle user stopped speaking event + @client.on(AgentServerMessageType.USER_SPEECH_ENDED) + def handle_speech_ended(message): + print("User stopped speaking") + + # Connect and start processing audio + await client.connect() + + while True: + frame = await mic.read(160) + await client.send_audio(frame) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Core Components + +### VoiceAgentClient + +The main client class is the `VoiceAgentClient` which extends `AsyncClient` from the Speechmatics Real-Time SDK. + +```python +VoiceAgentClient( + api_key: Optional[str] = None, # Speechmatics API key + url: Optional[str] = None, # REST API endpoint URL + app: Optional[str] = None, # Optional application name + config: Optional[VoiceAgentConfig] = None, # Optional voice agent config +) +``` + +**Key Methods:** + +- `connect()` - Establish WebSocket connection +- `disconnect()` - Disconnect from WebSocket connection +- `update_diarization_config(config)` - Update the diarization configuration +- `send_audio(frame)` - Send audio data for transcription +- `on(event_type)` - Register event callbacks + +**Authentication:** + +- Uses `SPEECHMATICS_API_KEY` environment variable +- Custom API key via constructor parameter + +### VoiceAgentConfig + +Configuration class for voice agent behaviour. + +```python +from speechmatics.voice import ( + AdditionalVocabEntry, + AudioEncoding, + DiarizationSpeakerConfig, + EndOfUtteranceMode, + OperatingPoint, + VoiceAgentConfig, +) + +config = VoiceAgentConfig( + # Service Configuration + operating_point=OperatingPoint.ENHANCED, # Accuracy vs latency tradeoff + domain=None, # Optional domain specification + language="en", # Language code (ISO 639-1) + output_locale="en-GB", # Output locale format + + # Timing & Finalisation Control + max_delay=1.0, # Max delay for transcription (seconds) + end_of_utterance_silence_trigger=0.5, # Silence threshold for EOU (seconds) + end_of_utterance_mode=EndOfUtteranceMode.FIXED, # EOU detection mode + + # Vocabulary Enhancement + additional_vocab=[ # Custom vocabulary entries + AdditionalVocabEntry( + content="Speechmatics", + sounds_like=["speech matters", "speech magic"] + ) + ], + punctuation_overrides={}, # Custom punctuation rules + + # Speaker diarization + enable_diarization=True, # Enable speaker separation + speaker_sensitivity=0.5, # diarization sensitivity (0.0-1.0) + max_speakers=None, # Max speakers to detect + prefer_current_speaker=False, # Prefer grouping words to current speaker + speaker_config=DiarizationSpeakerConfig(), # Advanced speaker configuration + known_speakers=[], # Pre-identified speakers + + # Audio Configuration + sample_rate=16000, # Audio sample rate (Hz) + chunk_size=160, # Audio chunk size (samples) + audio_encoding=AudioEncoding.PCM_S16LE, # Audio encoding format +) +``` + +### Event Types + +The SDK provides comprehensive event handling through `AgentServerMessageType` (shadows `ServerMessageType` from the RT SDK): + +**API Control Messages:** + +- `RECOGNITION_STARTED` - Recognition session has started +- `END_OF_TRANSCRIPT` - Recognition session has ended +- `INFO` - Informational message +- `WARNING` - Warning message +- `ERROR` - Error message + +**Transcription Messages:** + +- `ADD_TRANSCRIPT` - Final transcription results +- `ADD_PARTIAL_TRANSCRIPT` - Partial transcription updates + +**Speaker Segments:** + +- `ADD_SEGMENTS` - Final speaker segments +- `ADD_INTERIM_SEGMENTS` - Interim speaker segments + +**Voice Activity Detection (VAD):** + +- `USER_SPEECH_STARTED` - Voice activity detected +- `USER_SPEECH_ENDED` - Voice activity ended + +**Speaker Information:** + +- `SPEAKERS_RESULT` - Speaker identification results + +**Performance Metrics:** + +- `METRICS` - General STT engine metrics +- `TTFB_METRICS` - Time to First Byte metrics + +### SpeakerSegment + +Represents a segment of speech from a specific speaker: + +```python +def format_segment(segment: SpeakerSegment) -> str: + if segment.is_active: + return segment.format_text("@{speaker_id}: {text}") + else: + return segment.format_text("@{speaker_id} (background): {text}") +``` + +The `format_text()` method uses the following placeholders: + +- `{speaker_id}`: The speaker ID (empty if no speaker) (e.g. `S1`). +- `{text}`: The formatted text of the segment (e.g. `Hello, how are you?`). +- `{ts}`: The timestamp of the segment (e.g. `2025-09-03T16:05:01.000Z`). +- `{lang}`: The language of the segment (e.g. `en`). + +## Advanced Configuration + +### End of Utterance Modes + +The end of utterance mode controls how the STT engine detects the end of a user's speech. This can either be fixed silence threshold (defaults to `0.5s`) or it can be adaptive based on speech patterns, rate, pauses, and disfluencies. + +```python +from speechmatics.voice import EndOfUtteranceMode + +# Fixed silence threshold +EndOfUtteranceMode.FIXED + +# Adaptive silence detection (recommended) +EndOfUtteranceMode.ADAPTIVE +``` + +### Speaker Diarization + +The configuration below would: + +- focus on speakers `S1` and `S2` +- ignore speakers `S3` and `S4` +- retain words from all other speakers + +```python +from speechmatics.voice import DiarizationSpeakerConfig, DiarizationKnownSpeaker + +config = VoiceAgentConfig( + enable_diarization=True, + diarization_config=DiarizationSpeakerConfig( + focus_speakers=["S1", "S2"], + ignore_speakers=["S3", "S4"], + focus_mode=DiarizationFocusMode.RETAIN, + ) +) +``` + +### Additional Vocabulary + +If you have specific words that you need to make sure are transcribed properly, you can add in up to 1,000 additional vocabulary entries. You can optionally provide a list of words that sound like the word you are adding. + +```python +from speechmatics.voice import AdditionalVocabEntry + +config = VoiceAgentConfig( + additional_vocab=[ + AdditionalVocabEntry( + content="Speechmatics", + sounds_like=["speech matters", "speech magic"] + ) + ] +) +``` + +## Environment Variables + +- `SPEECHMATICS_API_KEY` - Your Speechmatics API key +- `SPEECHMATICS_RT_URL` - Custom WebSocket endpoint (optional) +- `SPEECHMATICS_DEBUG_MORE` - Enable verbose debugging diff --git a/sdk/voice/README.md b/sdk/voice/README.md new file mode 100644 index 00000000..506b148d --- /dev/null +++ b/sdk/voice/README.md @@ -0,0 +1,105 @@ +# Voice Agent Python client for Speechmatics Real-Time API + +[![License](https://img.shields.io/badge/license-MIT-yellow.svg)](https://github.com/speechmatics/speechmatics-python-voice/blob/master/LICENSE) + +An SDK for working with the Speechmatics Real-Time API optimised for use in voice agents or transcription services. + +This uses the Python Real-Time API to process the transcription results from the STT engine and combine them into manageable segments of audio. Taking advantage of speaker diarization, the transcription is grouped into individual speakers, with advanced options to focus on and/or ignore specific speakers. + +See [OVERVIEW.md](OVERVIEW.md) for more information. + +## Installation + +```bash +pip install speechmatics-voice +``` + +## Requirements + +You must have a valid Speechmatics API key to use this SDK. You can get one from the [Speechmatics Console](https://console.speechmatics.com). + +Store this as `SPEECHMATICS_API_KEY` environment variable in your `.env` file or use `export SPEECHMATICS_API_KEY="your_api_key_here"` in your terminal. + +## Quick Start + +Below is a basic example of how to use the SDK to transcribe audio from a microphone. + +```python +import asyncio +from speechmatics.rt import Microphone +from speechmatics.voice import ( + VoiceAgentClient, + VoiceAgentConfig, + EndOfUtteranceMode, + AgentServerMessageType, + SpeakerSegment, +) + +async def main(): + # Configure the voice agent + config = VoiceAgentConfig( + end_of_utterance_silence_trigger=0.5, + enable_diarization=True, + end_of_utterance_mode=EndOfUtteranceMode.ADAPTIVE, + ) + + # Initialize microphone + mic = Microphone( + sample_rate=16000, + chunk_size=160, + ) + + if not mic.start(): + print("Microphone not available") + return + + # Create client and register event handlers + async with VoiceAgentClient(config=config) as client: + + # Handle interim transcription segments + @client.on(AgentServerMessageType.ADD_INTERIM_SEGMENTS) + def handle_interim_segments(message): + segments: list[SpeakerSegment] = message["segments"] + for segment in segments: + print(f"Speaker {segment.speaker_id}: {segment.text}") + + # Handle finalized transcription segments + @client.on(AgentServerMessageType.ADD_SEGMENTS) + def handle_final_segments(message): + segments: list[SpeakerSegment] = message["segments"] + for segment in segments: + print(f"Speaker {segment.speaker_id}: {segment.text}") + + # Handle user started speaking event + @client.on(AgentServerMessageType.USER_SPEECH_STARTED) + def handle_speech_started(message): + print("User started speaking") + + # Handle user stopped speaking event + @client.on(AgentServerMessageType.USER_SPEECH_ENDED) + def handle_speech_ended(message): + print("User stopped speaking") + + # Connect and start processing audio + await client.connect() + + while True: + frame = await mic.read(160) + await client.send_audio(frame) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Examples + +The `examples/` directory contains practical demonstrations of the Voice Agent SDK. See the README in the `examples/voice` directory for more information. + +## Documentation + +- **SDK Overview**: See [OVERVIEW.md](OVERVIEW.md) for comprehensive API documentation +- **Speechmatics API**: https://docs.speechmatics.com + +## License + +[MIT](LICENSE) diff --git a/sdk/voice/pyproject.toml b/sdk/voice/pyproject.toml new file mode 100644 index 00000000..2002efee --- /dev/null +++ b/sdk/voice/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "speechmatics-voice" +dynamic = ["version"] +description = "Speechmatics Voice Agent Python client for Real-Time API" +readme = "README.md" +authors = [{ name = "Speechmatics", email = "support@speechmatics.com" }] +license = "MIT" +requires-python = ">=3.9" +dependencies = ["websockets>=10.0"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Multimedia :: Sound/Audio :: Speech", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = [ + "speechmatics", + "conversational-ai", + "voice", + "agents", + "websocket", + "real-time", + "pipecat", + "livekit" +] + +[project.optional-dependencies] +dev = [ + "black", + "ruff", + "mypy", + "pre-commit", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "build", +] + +[project.urls] +homepage = "https://github.com/speechmatics/speechmatics-python-sdk" +documentation = "https://docs.speechmatics.com/" +repository = "https://github.com/speechmatics/speechmatics-python-sdk" +issues = "https://github.com/speechmatics/speechmatics-python-sdk/issues" + +[tool.setuptools.dynamic] +version = { attr = "speechmatics.voice.__version__" } + +[tool.setuptools.package-data] +"speechmatics.voice" = ["py.typed"] + +[tool.setuptools.packages.find] +where = ["."] + +[[tool.mypy.overrides]] +module = ["speechmatics.rt.*"] +ignore_missing_imports = true diff --git a/sdk/voice/speechmatics/__init__.py b/sdk/voice/speechmatics/__init__.py new file mode 100644 index 00000000..8db66d3d --- /dev/null +++ b/sdk/voice/speechmatics/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/voice/speechmatics/voice/__init__.py b/sdk/voice/speechmatics/voice/__init__.py new file mode 100644 index 00000000..3b9b554a --- /dev/null +++ b/sdk/voice/speechmatics/voice/__init__.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2025, Speechmatics / Cantab Research Ltd +# + + +"""Voice Agents SDK. + +A comprehensive set of utility classes tailored for Voice Agents and +using the Speechmatics Python Real-Time SDK, including the processing of +partial and final transcription from the STT engine into accumulated +transcriptions with flags to indicate changes between messages, etc. +""" + +__version__ = "0.1.0" + +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioFormat +from speechmatics.rt import OperatingPoint + +from ._client import VoiceAgentClient +from ._models import AdditionalVocabEntry +from ._models import AgentClientMessageType +from ._models import AgentServerMessageType +from ._models import AnnotationFlags +from ._models import AnnotationResult +from ._models import DiarizationFocusMode +from ._models import DiarizationKnownSpeaker +from ._models import DiarizationSpeakerConfig +from ._models import EndOfUtteranceMode +from ._models import SpeakerSegment +from ._models import SpeakerVADStatus +from ._models import SpeechFragment +from ._models import VoiceAgentConfig + +__all__ = [ + # SDK + "__version__", + # Conversation config + "VoiceAgentConfig", + "EndOfUtteranceMode", + "DiarizationSpeakerConfig", + "DiarizationFocusMode", + "AdditionalVocabEntry", + "DiarizationKnownSpeaker", + "AudioEncoding", + "AudioFormat", + "OperatingPoint", + # Transcription models + "AnnotationFlags", + "AnnotationResult", + "SpeakerSegment", + "SpeechFragment", + # Events + "SpeakerVADStatus", + # Client + "VoiceAgentClient", + "AgentClientMessageType", + "AgentServerMessageType", +] diff --git a/sdk/voice/speechmatics/voice/_client.py b/sdk/voice/speechmatics/voice/_client.py new file mode 100644 index 00000000..09c03dbc --- /dev/null +++ b/sdk/voice/speechmatics/voice/_client.py @@ -0,0 +1,945 @@ +# +# Copyright (c) 2025, Speechmatics / Cantab Research Ltd +# + +from __future__ import annotations + +import asyncio +import datetime +import os +import re +import time +from typing import Any +from typing import Optional +from urllib.parse import urlencode + +from speechmatics.rt import AsyncClient +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioFormat +from speechmatics.rt import ConversationConfig +from speechmatics.rt import ServerMessageType +from speechmatics.rt import TranscriptionConfig + +from . import __version__ +from ._logging import get_logger +from ._models import AgentServerMessageType +from ._models import AnnotationFlags +from ._models import AnnotationResult +from ._models import DiarizationFocusMode +from ._models import DiarizationSpeakerConfig +from ._models import EndOfUtteranceMode +from ._models import SpeakerFragmentView +from ._models import SpeakerVADStatus +from ._models import SpeechFragment +from ._models import VoiceAgentConfig + +DEBUG_MORE = os.getenv("SPEECHMATICS_DEBUG_MORE", "0").lower() in ["1", "true"] + +if DEBUG_MORE: + import json + + +class VoiceAgentClient(AsyncClient): + """Voice Agent client. + + This class extends the AsyncClient class from the Speechmatics Real-Time SDK + and provides additional functionality for processing partial and final + transcription from the STT engine into accumulated transcriptions with + flags to indicate changes between messages, etc. + """ + + def __init__( + self, + api_key: Optional[str] = None, + url: Optional[str] = None, + app: Optional[str] = None, + config: Optional[VoiceAgentConfig] = None, + ): + """Initialize the Voice Agent client. + + Args: + api_key: Speechmatics API key. If None, uses SPEECHMATICS_API_KEY env var. + url: REST API endpoint URL. If None, uses SPEECHMATICS_RT_URL env var + or defaults to production endpoint. + app: Optional application name to use in the endpoint URL. + config: Optional voice agent configuration. + """ + # Default URL + if not url: + url = os.getenv("SPEECHMATICS_RT_URL") or "wss://eu2.rt.speechmatics.com/v2" + + # Initialize the client + super().__init__(api_key=api_key, url=self._get_endpoint_url(url, app)) + + # Logger + self._logger = get_logger(__name__) + + # Process the config + self._config, self._transcription_config, self._audio_format = self._process_config(config) + + # Connection status + self._is_connected: bool = False + self._is_ready_for_audio: bool = False + + # Timing info + self._start_time: datetime.datetime = datetime.datetime.now(datetime.timezone.utc) + self._total_time: float = 0 + self._total_bytes: int = 0 + + # TTFB metrics + self._last_ttfb_time: Optional[float] = None + self._last_ttfb: float = 0 + + # Time to disregard speech fragments before + self._trim_before_time: float = 0 + + # Current utterance speech data + self._speech_fragments: list[SpeechFragment] = [] + self._speech_fragments_lock: asyncio.Lock = asyncio.Lock() + self._last_speech_fragments: list[SpeechFragment] = [] + self._current_view: Optional[SpeakerFragmentView] = None + + # Speaking states + self._is_speaking: bool = False + self._current_speaker: Optional[str] = None + self._last_vad_time: float = 0 + + # Diarization / speaker focus + self._end_of_utterance_mode: EndOfUtteranceMode = self._config.end_of_utterance_mode + self._end_of_utterance_delay: float = self._config.end_of_utterance_silence_trigger + self._dz_enabled: bool = self._config.enable_diarization + self._dz_config = self._config.speaker_config + + # EndOfUtterance fallback timer + self._finalize_task: Optional[asyncio.Task] = None + + # Segment processor + self._processor_wait_time: float = 0.005 + self._processor_task: Optional[asyncio.Task] = None + + # Emitter task + self._finals_emitter_task: Optional[asyncio.Task] = None + + # Metrics emitter task + self._metrics_emitter_interval: float = 10.0 + self._metrics_emitter_task: Optional[asyncio.Task] = None + + # Audi sampling info + self._audio_sample_rate: float = self._audio_format.sample_rate * 1.0 + self._audio_sample_width: float = { + AudioEncoding.PCM_F32LE: 4.0, + AudioEncoding.PCM_S16LE: 2.0, + }.get(self._audio_format.encoding, 1.0) + + # Register handlers + self._register_event_handlers() + + # def on(self, event: ServerMessageType, callback: Optional[Callable] = None) -> Callable: + + def _process_config( + self, config: Optional[VoiceAgentConfig] = None + ) -> tuple[VoiceAgentConfig, TranscriptionConfig, AudioFormat]: + """Create a formatted STT transcription and audio config. + + Creates a transcription config object based on the service parameters. Aligns + with the Speechmatics RT API transcription config. + + Args: + config: Optional VoiceAgentConfig object to process. + + Returns: + A tuple of (VoiceAgentConfig, TranscriptionConfig, AudioFormat). + """ + # Default config + if config is None: + config = VoiceAgentConfig() + + # Transcription config + transcription_config = TranscriptionConfig( + language=config.language, + domain=config.domain, + output_locale=config.output_locale, + operating_point=config.operating_point, + diarization="speaker" if config.enable_diarization else None, + enable_partials=True, + max_delay=config.max_delay, + ) + + # Additional vocab + if config.additional_vocab: + transcription_config.additional_vocab = [ + { + "content": e.content, + "sounds_like": e.sounds_like, + } + for e in config.additional_vocab + ] + + # Diarization + if config.enable_diarization: + dz_cfg: dict[str, Any] = {} + if config.speaker_sensitivity is not None: + dz_cfg["speaker_sensitivity"] = config.speaker_sensitivity + if config.prefer_current_speaker is not None: + dz_cfg["prefer_current_speaker"] = config.prefer_current_speaker + if config.known_speakers: + dz_cfg["speakers"] = {s.label: s.speaker_identifiers for s in config.known_speakers} + if config.max_speakers is not None: + dz_cfg["max_speakers"] = config.max_speakers + if dz_cfg: + transcription_config.speaker_diarization_config = dz_cfg + + # End of Utterance (for fixed) + if config.end_of_utterance_silence_trigger and config.end_of_utterance_mode == EndOfUtteranceMode.FIXED: + transcription_config.conversation_config = ConversationConfig( + end_of_utterance_silence_trigger=config.end_of_utterance_silence_trigger, + ) + + # Punctuation overrides + if config.punctuation_overrides: + transcription_config.punctuation_overrides = config.punctuation_overrides + + # Configure the audio + audio_format = AudioFormat( + encoding=config.audio_encoding, + sample_rate=config.sample_rate, + chunk_size=config.chunk_size, + ) + + # Return the config objects + return config, transcription_config, audio_format + + def _register_event_handlers(self) -> None: + """Register event handlers. + + Specific event handlers that we need to deal with. All other events + from the STT API will be available to clients to use themselves. + """ + + # Recognition started event + @self.once(ServerMessageType.RECOGNITION_STARTED) + def _evt_on_recognition_started(message: dict[str, Any]): + self._start_time = datetime.datetime.now(datetime.timezone.utc) + self._is_ready_for_audio = True + + # Partial transcript event + @self.on(ServerMessageType.ADD_PARTIAL_TRANSCRIPT) + def _evt_on_partial_transcript(message: dict[str, Any]): + self._handle_transcript(message, is_final=False) + + # Final transcript event + @self.on(ServerMessageType.ADD_TRANSCRIPT) + def _evt_on_final_transcript(message: dict[str, Any]): + self._handle_transcript(message, is_final=True) + + # End of utterance event + if self._end_of_utterance_mode == EndOfUtteranceMode.FIXED: + + @self.on(ServerMessageType.END_OF_UTTERANCE) + def _evt_on_end_of_utterance(message: dict[str, Any]): + self._logger.debug("End of utterance") + self.finalize_segments() + + async def connect(self) -> None: + """Connect to the Speechmatics API. + + Args: + transcription_config: Transcription configuration. + audio_format: Audio format. + """ + # Check if we are already connected + if self._is_connected: + self.emit( + AgentServerMessageType.ERROR, + {"reason": "Already connected"}, + ) + return + + # Connect to API + try: + await self.start_session( + transcription_config=self._transcription_config, + audio_format=self._audio_format, + ) + self._is_connected = True + self._start_metrics_task() + except Exception as e: + self._logger.error(f"Exception: {e}") + raise + + async def disconnect(self) -> None: + """Disconnect from the Speechmatics API.""" + # Check if we are already connected + if not self._is_connected: + return + + # Disconnect from API + try: + await asyncio.wait_for(self.close(), timeout=5.0) + except asyncio.TimeoutError: + self._logger.warning(f"{self} Timeout while closing Speechmatics client connection") + raise + except Exception as e: + self._logger.error(f"{self} Error closing Speechmatics client: {e}") + raise + finally: + self._is_connected = False + self._is_ready_for_audio = False + self._stop_metrics_task() + + def update_diarization_config(self, config: DiarizationSpeakerConfig) -> None: + """Update the diarization configuration. + + You can update the speakers that needs to be focussed on or ignored during + a session. The new config will overwrite the existing configuration and become + active immediately. + + Args: + config: The new diarization configuration. + """ + self._dz_config = config + + async def send_audio(self, payload: bytes) -> None: + """Send an audio frame through the WebSocket. + + Args: + payload: The audio frame to send. + + Examples: + >>> audio_chunk = b"" + >>> await client.send_audio(audio_chunk) + """ + # Skip if not ready for audio + if not self._is_ready_for_audio: + return + + # Send to the AsyncClient + await super().send_audio(payload) + + # Calculate the time (in seconds) for the payload + if self._audio_format is not None: + self._total_bytes += len(payload) + self._total_time += len(payload) / self._audio_sample_rate / self._audio_sample_width + + def _start_metrics_task(self) -> None: + """Start the metrics task.""" + + # Task to send metrics + async def emit_metrics() -> None: + while True: + # Interval between emitting metrics + await asyncio.sleep(self._metrics_emitter_interval) + + # Check if there are any listeners for AgentServerMessageType.METRICS + if not self.listeners(AgentServerMessageType.METRICS): + break + + # Calculations + time_s = round(self._total_time, 3) + + # Emit metrics + self.emit( + AgentServerMessageType.METRICS, + { + "total_time": time_s, + "total_time_str": time.strftime("%H:%M:%S", time.gmtime(time_s)), + "total_bytes": self._total_bytes, + "last_ttfb": self._last_ttfb, + }, + ) + + # Trigger the task + self._metrics_emitter_task = asyncio.create_task(emit_metrics()) + + def _stop_metrics_task(self) -> None: + """Stop the metrics task.""" + if self._metrics_emitter_task: + self._metrics_emitter_task.cancel() + self._metrics_emitter_task = None + + def _handle_transcript(self, message: dict[str, Any], is_final: bool) -> None: + """Handle the partial and final transcript events. + + Args: + message: The new Partial or Final from the STT engine. + is_final: Whether the data is final or partial. + """ + # Handle async + asyncio.create_task(self._handle_transcript_async(message, is_final)) + + async def _handle_transcript_async(self, message: dict[str, Any], is_final: bool) -> None: + """Handle the partial and final transcript events (async). + + Args: + message: The new Partial or Final from the STT engine. + is_final: Whether the data is final or partial. + """ + + # Add the speech fragments + fragments_available = await self._add_speech_fragments( + message=message, + is_final=is_final, + ) + + # Skip if unchanged + if not fragments_available: + return + + # Clear any existing timer + if self._processor_task is not None: + self._processor_task.cancel() + + # Send transcription frames after delay + async def process_after_delay(delay: float): + await asyncio.sleep(delay) + await self._process_speech_fragments() + self._processor_task = None + + # Send frames after delay + self._processor_task = asyncio.create_task(process_after_delay(self._processor_wait_time)) + + def _calculate_ttfb(self, end_time: float) -> None: + """Calculate the time to first text. + + The TTFB is calculated by taking the end time of the payload from the STT + engine and then calculating the difference between the total time of bytes + sent to the engine from the client. + + Args: + end_time: The end time of the payload from the STT engine. + """ + # Skip if no fragments are words + if len(self._speech_fragments) == 0 or all(f._type != "word" for f in self._speech_fragments): + return + + # Get start of the first fragment + fragments_start_time = self._speech_fragments[0].start_time + + # Skip if no partial word or if we have already calculated the TTFB for this word + if self._last_ttfb_time and fragments_start_time <= self._last_ttfb_time: + return + + # Calculate the time difference (convert to ms) + ttfb = (self._total_time - end_time) * 1000.0 + + # Skip if zero or less + if ttfb <= 0: + return + + # Save TTFB and end time + self._last_ttfb = ttfb + self._last_ttfb_time = end_time + + # Emit the TTFB + self.emit( + AgentServerMessageType.TTFB_METRICS, + { + "ttfb": self._last_ttfb, + }, + ) + + async def _add_speech_fragments(self, message: dict[str, Any], is_final: bool = False) -> bool: + """Takes a new Partial or Final from the STT engine. + + Accumulates it into the _speech_data list. As new final data is added, all + partials are removed from the list. + + Note: If a known speaker is `__[A-Z0-9_]{2,}__`, then the words are skipped, + as this is used to protect against self-interruption by the assistant or to + block out specific known voices. + + Args: + message: The new Partial or Final from the STT engine. + is_final: Whether the data is final or partial. + + Returns: + True if the speech fragments were updated, False otherwise. + """ + async with self._speech_fragments_lock: + # Parsed new speech data from the STT engine + fragments: list[SpeechFragment] = [] + + # Metadata + metadata = message.get("metadata", {}) + payload_start_time = metadata.get("start_time", 0) + payload_end_time = metadata.get("end_time", 0) + + # Iterate over the results in the payload + for result in message.get("results", []): + alt = result.get("alternatives", [{}])[0] + if alt.get("content", None): + # Create the new fragment + fragment = SpeechFragment( + start_time=result.get("start_time", 0), + end_time=result.get("end_time", 0), + language=alt.get("language", "en"), + _type=result.get("type", "word"), + is_eos=result.get("is_eos", False), + is_disfluency="disfluency" in alt.get("tags", []), + is_punctuation=result.get("type", "") == "punctuation", + is_final=is_final, + attaches_to=result.get("attaches_to", ""), + content=alt.get("content", ""), + speaker=alt.get("speaker", None), + confidence=alt.get("confidence", 1.0), + result=result, + ) + + # Check fragment is after trim time + if fragment.start_time < self._trim_before_time: + continue + + # Speaker filtering + if fragment.speaker: + # Drop `__XX__` speakers + if re.match(r"^__[A-Z0-9_]{2,}__$", fragment.speaker): + continue + + # Drop speakers not focussed on + if ( + self._dz_config.focus_mode == DiarizationFocusMode.IGNORE + and self._dz_config.focus_speakers + and fragment.speaker not in self._dz_config.focus_speakers + ): + continue + + # Drop ignored speakers + if self._dz_config.ignore_speakers and fragment.speaker in self._dz_config.ignore_speakers: + continue + + # Add the fragment + fragments.append(fragment) + + # Evaluate for VAD (only done on partials) + if not is_final: + self._vad_evaluation(fragments) + + # Fragments to retain + # - Outside of the bounds of the payload + # - Not a trailing fragment with same start time as payload end time (end of sentence etc.) + # retained_fragments = [ + # frag + # for frag in self._speech_fragments + # if not ( + # frag.start_time >= payload_start_time + # and frag.end_time <= payload_end_time + # and frag.is_final is False + # # and frag.start_time != payload_end_time + # ) + # ] + retained_fragments = [frag for frag in self._speech_fragments if frag.is_final] + + # Re-structure the speech fragments + self._speech_fragments = retained_fragments.copy() + self._speech_fragments.extend(fragments) + self._speech_fragments.sort(key=lambda x: x.start_time) + + # Debug the fragments + if DEBUG_MORE: + debug_payload = { + "final": is_final, + "start_time": payload_start_time, + "end_time": payload_end_time, + "keeping": [f.content for f in retained_fragments], + "adding": [f.content for f in fragments], + "transcript": metadata.get("transcript", ""), + "full": [[f.content, f.start_time, f.end_time, f.is_final] for f in self._speech_fragments], + } + self._logger.debug(json.dumps(debug_payload)) + + # Update TTFB + if not is_final: + self._calculate_ttfb(end_time=payload_end_time) + + # Fragments available + return len(self._speech_fragments) > 0 + + async def _process_speech_fragments(self) -> None: + """Process the speech fragments. + + Compares the current speech fragments against the last set of speech fragments. + When segments are emitted, they are then removed from the buffer of fragments + so the next comparison is based on the remaining + new fragments. + """ + async with self._speech_fragments_lock: + # Current transcription + last_complete_view = SpeakerFragmentView( + fragments=self._last_speech_fragments.copy(), + base_time=self._start_time, + focus_speakers=self._dz_config.focus_speakers, + annotate_segments=False, + ) + last_active_view = SpeakerFragmentView( + fragments=[f for s in last_complete_view.segments if s.is_active for f in s.fragments], + base_time=self._start_time, + focus_speakers=self._dz_config.focus_speakers, + annotate_segments=False, + ) + + # New transcript + current_complete_view = SpeakerFragmentView( + fragments=self._speech_fragments.copy(), + base_time=self._start_time, + focus_speakers=self._dz_config.focus_speakers, + ) + current_active_view = SpeakerFragmentView( + fragments=[f for s in current_complete_view.segments if s.is_active for f in s.fragments], + base_time=self._start_time, + focus_speakers=self._dz_config.focus_speakers, + ) + + # If no segments, return + if current_complete_view.segment_count == 0: + return + + # Compare this against the previous transcript + complete_annotation_result = current_complete_view.compare_and_annotate(last_complete_view) + active_annotation_result = current_active_view.compare_and_annotate(last_active_view) + + # TODO - Consider a config option to split segments into smaller chunks for long transcripts? + + # Determine if there has been a change and delay before finalizing + active_view_changed, delay_to_finalize = self._calc_delay_to_finalize( + view=current_complete_view, view_changes=active_annotation_result + ) + + # Emit partial segments + if complete_annotation_result.any(AnnotationFlags.NEW, AnnotationFlags.UPDATED_FULL_LCASE): + asyncio.create_task(self._emit_partial_segments(current_complete_view, delay_to_finalize)) + + # Emit segments + if active_view_changed: + # Update the active view + self._current_view = current_complete_view + + # Stop any existing emitter task + if self._finals_emitter_task is not None: + self._finals_emitter_task.cancel() + + # Emit final segments + if delay_to_finalize is not None: + asyncio.create_task(self._emit_final_segments(delay=delay_to_finalize)) + + # Copy the data + self._last_speech_fragments = self._speech_fragments.copy() + + def _calc_delay_to_finalize( + self, + view: SpeakerFragmentView, + view_changes: AnnotationResult, + ) -> tuple[bool, float | None]: + """Calculate the delay before finalizing segments. + + Process the most recent segment and view to determine how long to delay before emitting + the segments to the client. + + Args: + view: The speaker fragment to evaluate. + view_changes: The annotation result to use for evaluation. + + Returns: + Tuple of (should_emit, emit_final_delay) + """ + # Skip if no segments + if view.segment_count == 0: + return False, None + + # Emit segments + emit_final_delay: Optional[float] = None + should_emit: bool = False + + # Last active segment + last_active_segment_index = view.last_active_segment_index + last_active_segment = view.segments[last_active_segment_index] if last_active_segment_index > -1 else None + + # Elapsed time from last segment to now + # elapsed_time = ( + # self._total_time - last_active_segment.end_time + # if last_active_segment + # else view.end_time + # ) + + # If this is NEW or UPDATED_FULL_LCASE + if view_changes.any(AnnotationFlags.NEW, AnnotationFlags.UPDATED_FULL_LCASE): + """Process the annotation flags to determine how long before sending a final segment.""" + + # Fallback when using FIXED + if self._end_of_utterance_mode == EndOfUtteranceMode.FIXED: + emit_final_delay = self._end_of_utterance_delay * 5.0 + + # Timer for when ADAPTIVE + elif self._end_of_utterance_mode == EndOfUtteranceMode.ADAPTIVE and last_active_segment: + """Check the contents of the last segment." + + Check for: + - ends with a disfluency + """ + + # Delay multiplier + multiplier = 1.5 + + # Very speaking + if last_active_segment.annotation.has(AnnotationFlags.VERY_SLOW_SPEAKER): + multiplier *= 3.0 + + # Slow speaking + if last_active_segment.annotation.has(AnnotationFlags.SLOW_SPEAKER): + multiplier *= 1.5 + + # Has a disfluency + if last_active_segment.annotation.has(AnnotationFlags.HAS_DISFLUENCY): + multiplier *= 1.5 + + # Ends with a disfluency + if last_active_segment.annotation.has(AnnotationFlags.ENDS_WITH_DISFLUENCY): + multiplier *= 4.0 + + # Calculate the delay + emit_final_delay = self._end_of_utterance_delay * multiplier + + # TODO - Other checks / end of turn detection + + # Emit segments + should_emit = True + + # Remove any elapsed time from the delay + # if emit_final_delay is not None: + # emit_final_delay = max( + # emit_final_delay - elapsed_time, + # self._end_of_utterance_delay - elapsed_time, + # 0.001, + # ) + + # Return the result + return should_emit, emit_final_delay + + def finalize_segments(self, ttl: Optional[float] = None): + """Finalize segments. + + This function will emit segments in the buffer without any further checks + on the contents of the segments. If the ttl is set to zero, then finalization + will be forced through without yielding for any remaining STT messages. + + Args: + ttl: Optional delay before finalizing partial segments (defaults to 0.01 seconds). + """ + + async def _emit(): + await asyncio.sleep(ttl or 0.01) + await self._emit_final_segments() + + asyncio.create_task(_emit()) + + async def _emit_partial_segments( + self, view: SpeakerFragmentView, delay_to_finalize: Optional[float] = None + ) -> None: + """Emit partial segments to listeners. + + This function will emit segments in the view without any further checks + on the contents of the segments. + + Args: + view: The speaker fragment view to emit. + delay_to_finalize: Optional delay before finalizing the segments. + """ + # Skip if no segments + if view.segment_count == 0: + return + + # Emit partial segments + self.emit( + AgentServerMessageType.ADD_INTERIM_SEGMENTS, + {"segments": view.segments, "delay_to_finalize": delay_to_finalize}, + ) + + async def _emit_final_segments(self, delay: float = 0) -> None: + """Emit final segments to listeners. + + At the point of emitting the result, this will use the most current view, + unlike emit_partial_segments which uses the view that was passed in. + + Args: + delay: Delay before finalizing partial segments. + """ + # Check delay within bounds + if delay < 0: + raise ValueError("Delay must be non-negative") + elif delay > 30: + raise ValueError("Delay must be less than 30 seconds") + + # Skip if no segments + if self._current_view is None or self._current_view.segment_count == 0: + return + + # Stop any existing emitter task + if self._finals_emitter_task is not None: + self._finals_emitter_task.cancel() + + async def emit_finals_now(): + """Emit final segments.""" + + # Skip if no segments + if self._current_view is None or self._current_view.segment_count == 0: + return + + # Get the active view + view = self._current_view + + # Find out if we have segments to emit + segments_to_emit = view.last_active_segment_index + 1 + + # Only finalize if there are valid segments + if segments_to_emit > 0: + # Accumulate all of the fragments for segments up to and including the last_active_index + fragments_to_emit = [] + for i in range(segments_to_emit): + fragments_to_emit.extend(view.segments[i].fragments) + + # Only include segments up to the last active segment + trimmed_view = SpeakerFragmentView( + fragments=fragments_to_emit, + base_time=view.base_time, + focus_speakers=view.focus_speakers, + annotate_segments=False, + ) + + # Emit the segments + self.emit( + AgentServerMessageType.ADD_SEGMENTS, + {"segments": trimmed_view.segments}, + ) + + # Last end time + self._trim_before_time = trimmed_view.end_time + + # Filter segments after the trim time + async with self._speech_fragments_lock: + self._speech_fragments = [ + fragment for fragment in self._speech_fragments if fragment.start_time >= self._trim_before_time + ] + + # Update the current view + self._current_view = SpeakerFragmentView( + fragments=self._speech_fragments, + base_time=self._trim_before_time, + focus_speakers=self._dz_config.focus_speakers, + annotate_segments=False, + ) + + # Reset previous fragments + self._last_speech_fragments = self._speech_fragments.copy() + + async def emit_finals_after_delay(_delay: float): + """Yield before emitting final segments after a delay.""" + + # Wait for the delay (can be cancelled) + if _delay > 0: + await asyncio.sleep(_delay) + + # Emit the segments + await emit_finals_now() + + # Reset the task + self._finals_emitter_task = None + + if DEBUG_MORE: + self._logger.debug(f"Will emit final segments after {delay} seconds") + + # Emit with delay or immediately + if delay > 0: + self._finals_emitter_task = asyncio.create_task(emit_finals_after_delay(delay)) + else: + self._finals_emitter_task = asyncio.create_task(emit_finals_now()) + + def _vad_evaluation(self, fragments: list[SpeechFragment]): + """Emit a VAD event. + + This will emit `SPEECH_STARTED` and `SPEECH_ENDED` events to the client and is + based on valid transcription for active speakers. Ignored or speakers not in + focus will not be considered an active participant. + + This should only run on partial / non-final words. + + Args: + fragments: The list of fragments to use for evaluation. + """ + # Find the valid list of partial words + if self._dz_enabled and self._dz_config.focus_speakers: + partial_words = [ + frag + for frag in fragments + if frag.speaker in self._dz_config.focus_speakers and frag._type == "word" and not frag.is_final + ] + else: + partial_words = [frag for frag in fragments if frag._type == "word" and not frag.is_final] + + # Evaluate if any valid partial words exist + has_valid_partial = len(partial_words) > 0 + + # Are we already speaking + already_speaking = self._is_speaking + + # Speakers + current_speaker = self._current_speaker + speaker = partial_words[-1].speaker if has_valid_partial else self._current_speaker + speaker_changed = speaker != current_speaker and current_speaker is not None + + # If diarization is enabled, indicate speaker switching + if self._dz_enabled and speaker is not None: + """When enabled, we send a speech events if the speaker has changed. + + For any client that wishes to show _which_ speaker is speaking, this will + emit events to indicate when speakers switch. + """ + + # Check if speaker is different to the current speaker + if already_speaking and speaker_changed: + self.emit( + AgentServerMessageType.USER_SPEECH_ENDED, + {"status": SpeakerVADStatus(speaker_id=current_speaker, is_active=False)}, + ) + self.emit( + AgentServerMessageType.USER_SPEECH_STARTED, + {"status": SpeakerVADStatus(speaker_id=speaker, is_active=True)}, + ) + + # Update current speaker + self._current_speaker = speaker + + # No change required + if has_valid_partial == already_speaking: + return + + # Set the speaking state + self._is_speaking = not self._is_speaking + + # Emit the event for latest speaker + self.emit( + ( + AgentServerMessageType.USER_SPEECH_STARTED + if self._is_speaking + else AgentServerMessageType.USER_SPEECH_ENDED + ), + {"status": SpeakerVADStatus(speaker_id=speaker, is_active=self._is_speaking)}, + ) + + # Reset the current speaker + if not self._is_speaking: + self._current_speaker = None + + def _get_endpoint_url(self, url: str, app: Optional[str] = None) -> str: + """Format the endpoint URL with the SDK and app versions. + + Args: + url: The base URL for the endpoint. + app: The application name to use in the endpoint URL. + + Returns: + str: The formatted endpoint URL. + """ + query_params = {} + query_params["sm-app"] = app or f"voice-sdk/{__version__}" + query_params["sm-voice-sdk"] = f"{__version__}" + query = urlencode(query_params) + + return f"{url}?{query}" diff --git a/sdk/voice/speechmatics/voice/_logging.py b/sdk/voice/speechmatics/voice/_logging.py new file mode 100644 index 00000000..18f7419f --- /dev/null +++ b/sdk/voice/speechmatics/voice/_logging.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2025, Speechmatics / Cantab Research Ltd +# + +import logging + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + +def get_logger(name: str) -> logging.Logger: + """Get a logger that stays silent by default. + + The logger uses Python's standard logging module and includes NullHandler + by default to avoid unwanted output. Users can configure logging levels + and handlers as needed. + + Args: + name: Logger name, typically __name__ from the calling module. + + Returns: + Configured logger instance. + + Examples: + Basic usage in SDK modules: + logger = get_logger(__name__) + logger.debug("HTTP request sent %s %s", method, url) + logger.info("Job submitted (job_id=%s)", job_id) + logger.warning("Job failed (job_id=%s): %s", job_id, error) + logger.error("Connection failed: %s", e) + + Enable debug logging in user code: + import logging + logging.basicConfig(level=logging.DEBUG) + # Now all SDK debug messages will be visible + + Custom logging configuration: + import logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # Or for specific components: + logging.getLogger('speechmatics.batch').setLevel(logging.DEBUG) + """ + module_logger = logging.getLogger(name) + module_logger.addHandler(logging.NullHandler()) + return module_logger + + +__all__ = ["get_logger"] diff --git a/sdk/voice/speechmatics/voice/_models.py b/sdk/voice/speechmatics/voice/_models.py new file mode 100644 index 00000000..a79a34b0 --- /dev/null +++ b/sdk/voice/speechmatics/voice/_models.py @@ -0,0 +1,793 @@ +# +# Copyright (c) 2025, Speechmatics / Cantab Research Ltd +# + + +import datetime +from dataclasses import dataclass +from dataclasses import field +from enum import Enum +from enum import IntFlag +from enum import auto +from typing import Any +from typing import Optional + +from speechmatics.rt import AudioEncoding +from speechmatics.rt import OperatingPoint + +__version__ = "0.1.0" + + +class EndOfUtteranceMode(str, Enum): + """End of turn delay options for transcription.""" + + NONE = "none" + FIXED = "fixed" + ADAPTIVE = "adaptive" + + +class DiarizationFocusMode(str, Enum): + """Speaker focus mode for diarization.""" + + RETAIN = "retain" + IGNORE = "ignore" + + +@dataclass +class AdditionalVocabEntry: + """Additional vocabulary entry. + + Parameters: + content: The word to add to the dictionary. + sounds_like: Similar words to the word. + """ + + content: str + sounds_like: list[str] = field(default_factory=list) + + +@dataclass +class DiarizationKnownSpeaker: + """Known speakers for speaker diarization. + + Parameters: + label: The label of the speaker. + speaker_identifiers: One or more data strings for the speaker. + """ + + label: str + speaker_identifiers: list[str] + + +@dataclass +class DiarizationSpeakerConfig: + """Speaker Diarization Config. + + List of speakers to focus on, ignore and how to deal with speakers that are not + in focus. These settings can be changed during a session. Other changes may require + a new session. + + Parameters: + focus_speakers: List of speaker IDs to focus on. When enabled, only these speakers are + emitted as finalized frames and other speakers are considered passive. Words from + other speakers are still processed, but only emitted when a focussed speaker has + also said new words. A list of labels (e.g. `S1`, `S2`) or identifiers of known + speakers (e.g. `speaker_1`, `speaker_2`) can be used. + Defaults to []. + + ignore_speakers: List of speaker IDs to ignore. When enabled, these speakers are + excluded from the transcription and their words are not processed. Their speech + will not trigger any VAD or end of utterance detection. By default, any speaker + with a label starting and ending with double underscores will be excluded (e.g. + `__ASSISTANT__`). + Defaults to []. + + focus_mode: Speaker focus mode for diarization. When set to `DiarizationFocusMode.RETAIN`, + the STT engine will retain words spoken by other speakers (not listed in `ignore_speakers`) + and process them as passive speaker frames. When set to `DiarizationFocusMode.IGNORE`, + the STT engine will ignore words spoken by other speakers and they will not be processed. + Defaults to `DiarizationFocusMode.RETAIN`. + """ + + focus_speakers: list[str] = field(default_factory=list) + ignore_speakers: list[str] = field(default_factory=list) + focus_mode: DiarizationFocusMode = DiarizationFocusMode.RETAIN + + +@dataclass +class VoiceAgentConfig: + """Voice Agent configuration. + + A framework-independent configuration object for the Speechmatics Voice Agent. This uses + utility functions to create `TranscriptionConfig` and `AudioConfig` objects and also retain + agent configuration for the `VoiceAgentClient`. + + Parameters: + operating_point: Operating point for transcription accuracy vs. latency tradeoff. It is + recommended to use OperatingPoint.ENHANCED for most use cases. Defaults to + OperatingPoint.ENHANCED. + + domain: Domain for Speechmatics API. Defaults to None. + + language: Language code for transcription. Defaults to `en`. + + output_locale: Output locale for transcription, e.g. `en-GB`. Defaults to None. + + max_delay: Maximum delay in seconds for transcription. This forces the STT engine to + speed up the processing of transcribed words and reduces the interval between partial + and final results. Lower values can have an impact on accuracy. Defaults to 1.0. + + end_of_utterance_silence_trigger: Maximum delay in seconds for end of utterance trigger. + The delay is used to wait for any further transcribed words before emitting the final + word frames. The value must be lower than max_delay. + Defaults to 0.5. + + end_of_utterance_mode: End of utterance delay mode. When ADAPTIVE is used, the delay + can be adjusted on the content of what the most recent speaker has said, such as + rate of speech and whether they have any pauses or disfluencies. When FIXED is used, + the delay is fixed to the value of `end_of_utterance_delay`. Use of NONE disables + end of utterance detection and uses a fallback timer. + Defaults to `EndOfUtteranceMode.FIXED`. + + additional_vocab: List of additional vocabulary entries. If you supply a list of + additional vocabulary entries, the this will increase the weight of the words in the + vocabulary and help the STT engine to better transcribe the words. + Defaults to []. + + punctuation_overrides: Punctuation overrides. This allows you to override the punctuation + in the STT engine. This is useful for languages that use different punctuation + than English. See documentation for more information. + Defaults to None. + + enable_diarization: Enable speaker diarization. When enabled, the STT engine will + determine and attribute words to unique speakers. The speaker_sensitivity + parameter can be used to adjust the sensitivity of diarization. + Defaults to False. + + speaker_sensitivity: Diarization sensitivity. A higher value increases the sensitivity + of diarization and helps when two or more speakers have similar voices. + Defaults to 0.5. + + max_speakers: Maximum number of speakers to detect. This forces the STT engine to cluster + words into a fixed number of speakers. It should not be used to limit the number of + speakers, unless it is clear that there will only be a known number of speakers. + Defaults to None. + + prefer_current_speaker: Prefer current speaker ID. When set to true, groups of words close + together are given extra weight to be identified as the same speaker. + Defaults to False. + + speaker_config: DiarizationSpeakerConfig to configure the speakers to focus on, ignore and + how to deal with speakers that are not in focus. + + known_speakers: List of known speaker labels and identifiers. If you supply a list of + labels and identifiers for speakers, then the STT engine will use them to attribute + any spoken words to that speaker. This is useful when you want to attribute words + to a specific speaker, such as the assistant or a specific user. Labels and identifiers + can be obtained from a running STT session and then used in subsequent sessions. + Identifiers are unique to each Speechmatics account and cannot be used across accounts. + Refer to our examples on the format of the known_speakers parameter. + Defaults to []. + + sample_rate: Audio sample rate for streaming. Defaults to 16000. + chunk_size: Audio chunk size for streaming. Defaults to 160. + audio_encoding: Audio encoding format. Defaults to AudioEncoding.PCM_S16LE. + """ + + # Service configuration + operating_point: OperatingPoint = OperatingPoint.ENHANCED + domain: Optional[str] = None + language: str = "en" + output_locale: Optional[str] = None + + # Features + max_delay: float = 1.0 + end_of_utterance_silence_trigger: float = 0.5 + end_of_utterance_mode: EndOfUtteranceMode = EndOfUtteranceMode.FIXED + additional_vocab: list[AdditionalVocabEntry] = field(default_factory=list) + punctuation_overrides: Optional[dict] = None + + # Diarization + enable_diarization: bool = False + speaker_sensitivity: float = 0.5 + max_speakers: Optional[int] = None + prefer_current_speaker: bool = False + speaker_config: DiarizationSpeakerConfig = field(default_factory=DiarizationSpeakerConfig) + known_speakers: list[DiarizationKnownSpeaker] = field(default_factory=list) + + # Audio + sample_rate: int = 16000 + chunk_size: int = 160 + audio_encoding: AudioEncoding = AudioEncoding.PCM_S16LE + + +class AgentClientMessageType(str, Enum): + """Message types that can be sent from client to server. + + These enum values represent the different types of messages that the + client can send to the Speechmatics RT API during a transcription session. + + Attributes: + EndOfStream: Signals that no more audio data will be sent. + GetSpeakers: Internal, Speechmatics only message. Allows the client to request speaker data. + + Examples: + >>> # Ending the session + >>> end_message = { + ... "message": AgentClientMessageType.END_OF_STREAM, + ... "last_seq_no": sequence_number + ... } + """ + + END_OF_STREAM = "EndOfStream" + GET_SPEAKERS = "GetSpeakers" + + +class AgentServerMessageType(str, Enum): + """Message types that can be received from the server / agent. + + These enum values represent the different types of messages that the + Speechmatics RT API / Voice Agent SDK can send to the client. + + Attributes: + RecognitionStarted: The recognition session has started. + EndOfTranscript: The recognition session has ended. + Info: Informational message. + Warning: Warning message. + Error: Error message. + AddPartialTranscript: Partial transcript has been added. + AddTranscript: Transcript has been added. + UserSpeechStarted: Speech has started. + UserSpeechEnded: Speech has ended. + TurnDetected: A turn has been detected. + AddSegments: A final segment has been detected. + AddInterimSegments: An interim segment has been detected. + SpeakersResult: Speakers result has been detected. + Metrics: Metrics for the STT engine. + SpeakerMetrics: Metrics relating to speakers. + + Examples: + >>> # Register event handlers for different message types + >>> @client.on(AgentServerMessageType.INTERIM_SEGMENTS) + >>> def handle_interim(message): + ... segments: list[SpeakerSegment] = message['segments'] + ... print(f"Interim: {segments}") + >>> + >>> @client.on(AgentServerMessageType.FINAL_SEGMENTS) + >>> def handle_final(message): + ... segments: list[SpeakerSegment] = message['segments'] + ... print(f"Final: {segments}") + >>> + >>> @client.on(AgentServerMessageType.ERROR) + >>> def handle_error(message): + ... print(f"Error: {message['reason']}") + """ + + # API messages + RECOGNITION_STARTED = "RecognitionStarted" + END_OF_TRANSCRIPT = "EndOfTranscript" + INFO = "Info" + WARNING = "Warning" + ERROR = "Error" + + # Raw transcription messages + ADD_PARTIAL_TRANSCRIPT = "AddPartialTranscript" + ADD_TRANSCRIPT = "AddTranscript" + + # VAD messages + USER_SPEECH_STARTED = "UserSpeechStarted" + USER_SPEECH_ENDED = "UserSpeechEnded" + + # Turn / segment messages + ADD_SEGMENTS = "AddSegments" + ADD_INTERIM_SEGMENTS = "AddInterimSegments" + TURN_DETECTED = "TurnDetected" + + # Speaker messages + SPEAKERS_RESULT = "SpeakersResult" + + # Metrics + METRICS = "Metrics" + TTFB_METRICS = "TTFBMetrics" + SPEAKER_METRICS = "SpeakerMetrics" + + +class AnnotationFlags(IntFlag): + """Flags to apply when processing speech / objects.""" + + # High-level segment updates + NEW = auto() + UPDATED_FULL = auto() + UPDATED_FULL_LCASE = auto() + UPDATED_STRIPPED = auto() + UPDATED_STRIPPED_LCASE = auto() + UPDATED_FINALS = auto() + UPDATED_PARTIALS = auto() + UPDATED_SPEAKERS = auto() + + # Content of segments + ONLY_ACTIVE_SPEAKERS = auto() + CONTAINS_INACTIVE_SPEAKERS = auto() + + # More granular details on the word content + HAS_PARTIAL = auto() + HAS_FINAL = auto() + STARTS_WITH_FINAL = auto() + ENDS_WITH_FINAL = auto() + HAS_EOS = auto() + ENDS_WITH_EOS = auto() + HAS_DISFLUENCY = auto() + STARTS_WITH_DISFLUENCY = auto() + ENDS_WITH_DISFLUENCY = auto() + HIGH_DISFLUENCY_COUNT = auto() + ENDS_WITH_PUNCTUATION = auto() + VERY_SLOW_SPEAKER = auto() + SLOW_SPEAKER = auto() + FAST_SPEAKER = auto() + ONLY_PUNCTUATION = auto() + MULTIPLE_SPEAKERS = auto() + NO_TEXT = auto() + + # End of utterance detection + END_OF_UTTERANCE = auto() + + +@dataclass +class AnnotationResult: + """Processing result.""" + + flags: int = 0 + + def __init__(self, *flags: AnnotationFlags): + """Initialize the object. + + Args: + flags: The initial flags to set. + """ + self.flags = 0 + for flag in flags: + self.add(flag) + + def has(self, *flags: AnnotationFlags) -> bool: + """Check if the object has all given flags.""" + return all(self.flags & flag == flag for flag in flags) + + def any(self, *flags: AnnotationFlags) -> bool: + """Check if the object has any of the given flags.""" + return any(self.flags & flag == flag for flag in flags) + + def add(self, flag: AnnotationFlags) -> None: + """Add a flag to the object.""" + self.flags |= flag + + def remove(self, flag: AnnotationFlags) -> None: + """Remove a flag from the object.""" + self.flags &= ~flag + + def __eq__(self, o: object) -> bool: + """Check if the object is equal to another.""" + if isinstance(o, AnnotationResult): + return self.flags == o.flags + elif isinstance(o, (AnnotationFlags, int)): + return self.flags == o + return False + + def __str__(self) -> str: + """String representation of the flags.""" + flags: list[str] = [] + for flag in AnnotationFlags: + if self.flags & flag == flag and flag.name is not None: + flags.append(flag.name) + return f"{type(self).__name__}({', '.join(flags)})" + + +@dataclass +class SpeechFragment: + """Fragment of a speech event. + + As the transcript is processed (partials and finals), a list of SpeechFragments + objects are accumulated and then used to form SpeechSegments objects. + + Parameters: + start_time: Start time of the fragment in seconds (from session start). + end_time: End time of the fragment in seconds (from session start). + language: Language of the fragment. Defaults to `en`. + _type: Type of the fragment. Defaults to `word`. + is_eos: Whether the fragment is the end of a sentence. Defaults to `False`. + is_final: Whether the fragment is the final fragment. Defaults to `False`. + is_disfluency: Whether the fragment is a disfluency. Defaults to `False`. + is_punctuation: Whether the fragment is a punctuation. Defaults to `False`. + attaches_to: Whether the fragment attaches to the previous or next fragment (punctuation). Defaults to empty string. + content: Content of the fragment. Defaults to empty string. + speaker: Speaker of the fragment (if diarization is enabled). Defaults to `None`. + confidence: Confidence of the fragment (0.0 to 1.0). Defaults to `1.0`. + result: Raw result of the fragment from the TTS. + annotation: Annotation for the fragment. + """ + + start_time: float + end_time: float + language: str = "en" + _type: str = "word" + is_eos: bool = False + is_final: bool = False + is_disfluency: bool = False + is_punctuation: bool = False + attaches_to: str = "" + content: str = "" + speaker: Optional[str] = None + confidence: float = 1.0 + result: Optional[Any] = None + annotation: Optional[AnnotationResult] = None + + +@dataclass +class SpeakerSegment: + """SpeechFragment items grouped by speaker_id and whether the speaker is active. + + Parameters: + speaker_id: The ID of the speaker. + is_active: Whether the speaker is active (emits frame). + timestamp: The timestamp of the frame. + language: The language of the frame. + fragments: The list of SpeechFragment items. + annotation: The annotation associated with the segment. + """ + + speaker_id: Optional[str] = None + is_active: bool = False + timestamp: Optional[str] = None + language: Optional[str] = None + fragments: list[SpeechFragment] = field(default_factory=list) + annotation: AnnotationResult = field(default_factory=AnnotationResult) + + @property + def start_time(self) -> float: + """Return the start time of the segment.""" + return self.fragments[0].start_time + + @property + def end_time(self) -> float: + """Return the end time of the segment.""" + return self.fragments[-1].end_time + + def __str__(self): + """Return a string representation of the object.""" + meta = { + "speaker_id": self.speaker_id, + "timestamp": self.timestamp, + "language": self.language, + "annotation": str(self.annotation), + "text": self.format_text(), + } + return f"SpeakerSegment({', '.join(f'{k}={v}' for k, v in meta.items())})" + + def format_text(self, format: Optional[str] = None, words_only: bool = False) -> str: + """Wrap text with speaker ID in an optional f-string format. + + Supported format variables: + speaker_id: The ID of the speaker. + text: The text of the fragment. + ts: The timestamp of the fragment. + lang: The language of the fragment. + + Args: + format: Format to wrap the text with. + words_only: Whether to include only word fragments. + + Returns: + str: The wrapped text. + """ + # Cumulative contents + content = "" + + # Select fragments to include + if not words_only: + fragments = self.fragments + else: + fragments = [frag for frag in self.fragments if frag._type == "word"] + + # Assemble the text + for frag in fragments: + if content == "" or frag.attaches_to == "previous": + content += frag.content + else: + content += " " + frag.content + + # Format the text, if format is provided + if format is None: + return content + return format.format( + **{ + "speaker_id": self.speaker_id, + "text": content, + "ts": self.timestamp, + "lang": self.language, + } + ) + + +@dataclass +class SpeakerFragmentView: + """View for speaker fragments. + + Parameters: + fragments: List of fragments. + base_time: Base time for the fragments. + focus_speakers: List of speakers to focus on or None. + """ + + fragments: list[SpeechFragment] + segments: list[SpeakerSegment] + + def __init__( + self, + fragments: list[SpeechFragment], + base_time: datetime.datetime, + focus_speakers: Optional[list[str]] = None, + annotate_segments: bool = True, + ): + self.fragments = fragments + self.base_time = base_time + self.focus_speakers = focus_speakers + self.segments = self._get_segments_from_fragments(annotate_segments=annotate_segments) + + @property + def start_time(self) -> float: + return self.fragments[0].start_time + + @property + def end_time(self) -> float: + return self.fragments[-1].end_time + + @property + def final_count(self) -> int: + return sum(1 for frag in self.fragments if frag.is_final) + + @property + def partial_count(self) -> int: + return sum(1 for frag in self.fragments if not frag.is_final) + + @property + def segment_count(self) -> int: + return len(self.segments) + + @property + def last_active_segment_index(self) -> int: + idx = next( + (i for i, segment in enumerate(reversed(self.segments)) if segment.is_active), + None, + ) + if idx is None: + return -1 + return len(self.segments) - idx - 1 + + def format_text( + self, + format: str = "|{speaker_id}|{text}|", + separator: str = "", + words_only: bool = False, + ) -> str: + return separator.join(segment.format_text(format, words_only) for segment in self.segments) + + def _get_segments_from_fragments(self, annotate_segments: bool = True) -> list[SpeakerSegment]: + # Speaker groups + current_speaker: Optional[str] = None + speaker_groups: list[list[SpeechFragment]] = [[]] + + # Group by speakers + for frag in self.fragments: + if frag.speaker != current_speaker: + current_speaker = frag.speaker + if speaker_groups[-1]: + speaker_groups.append([]) + speaker_groups[-1].append(frag) + + # Create SpeakerFragments objects + segments: list[SpeakerSegment] = [] + for group in speaker_groups: + sd = self._get_speaker_segment_from_fragment_group(group, annotate=annotate_segments) + if sd: + segments.append(sd) + + # Return the grouped SpeakerFragments objects + return segments + + def _get_speaker_segment_from_fragment_group( + self, + group: list[SpeechFragment], + annotate: bool = True, + ) -> Optional[SpeakerSegment]: + """Take a group of fragments and piece together into SpeakerSegment. + + Each fragment for a given speaker is assembled into a string, + taking into consideration whether words are attached to the + previous or next word (notably punctuation). This ensures that + the text does not have extra spaces. This will also check for + any straggling punctuation from earlier utterances that should + be removed. + + Args: + group: List of SpeechFragment objects. + annotate: Whether to annotate the segment. + + Returns: + SpeakerSegment: The object for the group. + """ + # Check for starting fragments that are attached to previous + if group and group[0].attaches_to == "previous": + group = group[1:] + + # Check for trailing fragments that are attached to next + if group and group[-1].attaches_to == "next": + group = group[:-1] + + # Check there are results + if not group: + return None + + # Get the timing extremes + start_time = min(frag.start_time for frag in group) + + # Timestamp + ts = (self.base_time + datetime.timedelta(seconds=start_time)).isoformat(timespec="milliseconds") + + # Determine if the speaker is considered active + is_active = True + if self.focus_speakers: + is_active = group[0].speaker in self.focus_speakers + + # New SpeakerSegment + segment = SpeakerSegment( + speaker_id=group[0].speaker, + timestamp=ts, + language=group[0].language, + fragments=group, + is_active=is_active, + ) + + # Annotate + if annotate: + segment.annotation = self._annotate_segment(segment) + + # Return the SpeakerSegment object + return segment + + def _annotate_segment(self, segment: SpeakerSegment) -> AnnotationResult: + """Annotate the segment. + + This will annotate the segment with any additional information. + """ + # Annotation result + result = AnnotationResult() + + # References + segment_length: int = len(segment.fragments) + first_fragment: SpeechFragment = segment.fragments[0] + last_fragment: SpeechFragment = segment.fragments[-1] + penultimate_fragment: Optional[SpeechFragment] = segment.fragments[-2] if segment_length > 1 else None + + # Count of words + words = [frag for frag in segment.fragments if frag._type == "word"] + word_count = len(words) + if word_count == 0: + result.add(AnnotationFlags.NO_TEXT) + + # Only punctuation + if all(frag.is_punctuation for frag in segment.fragments): + result.add(AnnotationFlags.ONLY_PUNCTUATION) + + # Partials and finals + if any(not frag.is_final for frag in segment.fragments): + result.add(AnnotationFlags.HAS_PARTIAL) + + # Finals + if any(frag.is_final for frag in segment.fragments): + result.add(AnnotationFlags.HAS_FINAL) + if first_fragment.is_final: + result.add(AnnotationFlags.STARTS_WITH_FINAL) + if last_fragment.is_final: + result.add(AnnotationFlags.ENDS_WITH_FINAL) + + # End of sentence + if last_fragment.is_eos: + result.add(AnnotationFlags.ENDS_WITH_EOS) + + # Punctuation + if last_fragment.is_punctuation: + result.add(AnnotationFlags.ENDS_WITH_PUNCTUATION) + + # Disfluency + if any(frag.is_disfluency for frag in segment.fragments): + result.add(AnnotationFlags.HAS_DISFLUENCY) + if first_fragment.is_disfluency: + result.add(AnnotationFlags.STARTS_WITH_DISFLUENCY) + if last_fragment.is_disfluency: + result.add(AnnotationFlags.ENDS_WITH_DISFLUENCY) + if ( + penultimate_fragment + and result.any(AnnotationFlags.ENDS_WITH_EOS, AnnotationFlags.ENDS_WITH_PUNCTUATION) + and penultimate_fragment.is_disfluency + ): + result.add(AnnotationFlags.ENDS_WITH_DISFLUENCY) + + # Rate of speech + if len(words) > 1: + # Calculate the approximate words-per-minute (for last 5 words) + last_5_words = words[-5:] + wpm = len(last_5_words) / ((last_5_words[-1].end_time - last_5_words[0].start_time) / 60.0) + + # Categorize the speaker + if wpm < 100: + result.add(AnnotationFlags.VERY_SLOW_SPEAKER) + elif wpm < 200: + result.add(AnnotationFlags.SLOW_SPEAKER) + elif wpm > 350: + result.add(AnnotationFlags.FAST_SPEAKER) + + # Return the annotation result + return result + + def compare_and_annotate(self, other: Optional["SpeakerFragmentView"]) -> AnnotationResult: + """Compare two SpeakerFragmentView objects and return the differences. + + Args: + other: The other SpeakerFragmentView object to compare to or None. + + Returns: + AnnotationResult: The annotation result. + """ + # Result + result = AnnotationResult() + + # If we have a previous view, compare it + if other and other.segment_count > 0: + # Compare full string + self_full_str: str = self.format_text() + other_full_str: str = other.format_text() + if self_full_str != other_full_str: + result.add(AnnotationFlags.UPDATED_FULL) + if self_full_str.lower() != other_full_str.lower(): + result.add(AnnotationFlags.UPDATED_FULL_LCASE) + + # Stripped string (without punctuation) + self_stripped_str: str = self.format_text(words_only=True) + other_stripped_str: str = other.format_text(words_only=True) + if self_stripped_str != other_stripped_str: + result.add(AnnotationFlags.UPDATED_STRIPPED) + if self_stripped_str.lower() != other_stripped_str.lower(): + result.add(AnnotationFlags.UPDATED_STRIPPED_LCASE) + + # Partials, finals and speakers + if self.final_count != other.final_count: + result.add(AnnotationFlags.UPDATED_FINALS) + if self.partial_count != other.partial_count: + result.add(AnnotationFlags.UPDATED_PARTIALS) + if self.segment_count != other.segment_count: + result.add(AnnotationFlags.UPDATED_SPEAKERS) + + # Assume this is new + elif self.segment_count > 0: + result.add(AnnotationFlags.NEW) + + # TODO - contents of LAST segment? + + # Return the result + self.annotation = result + return result + + +@dataclass +class SpeakerVADStatus: + """Emitted when a speaker starts or ends speaking. + + The speaker id is taken from the last word in the segment when + the event is emitted. + + Parameters: + is_active: Whether the speaker is active. + speaker_id: The ID of the speaker. + """ + + is_active: bool + speaker_id: Optional[str] = None diff --git a/tests/voice/test_setup.py b/tests/voice/test_setup.py new file mode 100644 index 00000000..4fc590d2 --- /dev/null +++ b/tests/voice/test_setup.py @@ -0,0 +1,2 @@ +def test_setup(): + assert True From 03165ad6349aaabbd83d5e1916c925e0316e285c Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 22:02:28 +0100 Subject: [PATCH 02/10] missing type fix for rt and batch --- sdk/batch/pyproject.toml | 4 ++++ sdk/rt/pyproject.toml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/sdk/batch/pyproject.toml b/sdk/batch/pyproject.toml index 7c10a8f1..7d5f9ea7 100644 --- a/sdk/batch/pyproject.toml +++ b/sdk/batch/pyproject.toml @@ -50,3 +50,7 @@ version = { attr = "speechmatics.batch.__version__" } [tool.setuptools.packages.find] where = ["."] + +[[tool.mypy.overrides]] +module = ["aiofiles.*"] +ignore_missing_imports = true diff --git a/sdk/rt/pyproject.toml b/sdk/rt/pyproject.toml index eca9c4c9..ab707803 100644 --- a/sdk/rt/pyproject.toml +++ b/sdk/rt/pyproject.toml @@ -56,3 +56,7 @@ version = { attr = "speechmatics.rt.__version__" } [tool.setuptools.packages.find] where = ["."] + +[[tool.mypy.overrides]] +module = ["pyaudio.*"] +ignore_missing_imports = true From a353400cf654b176f7d5910b77c309ff88808722 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 22:08:31 +0100 Subject: [PATCH 03/10] Fix to README in voice examples --- examples/voice/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/voice/README.md b/examples/voice/README.md index c95de58a..2d84090d 100644 --- a/examples/voice/README.md +++ b/examples/voice/README.md @@ -86,4 +86,4 @@ pip install pyaudio export SPEECHMATICS_API_KEY="your_api_key_here" ``` -For more information, see the main SDK documentation in [OVERVIEW.md](../OVERVIEW.md). +For more information, see the main SDK documentation in [OVERVIEW.md](../../OVERVIEW.md). From 39d8eb4aca551a21659dac59b45aaec5641d9d14 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 22:18:43 +0100 Subject: [PATCH 04/10] added missing github tests --- .github/workflows/release-voice.yaml | 90 ++++++++++++++++++++++++++++ .github/workflows/test.yaml | 20 +++++++ 2 files changed, 110 insertions(+) create mode 100644 .github/workflows/release-voice.yaml diff --git a/.github/workflows/release-voice.yaml b/.github/workflows/release-voice.yaml new file mode 100644 index 00000000..c80b36c4 --- /dev/null +++ b/.github/workflows/release-voice.yaml @@ -0,0 +1,90 @@ +name: Release Voice Agent SDK + +on: + push: + tags: + - "voice/v*" + +permissions: + contents: read + id-token: write + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.extract.outputs.version }} + steps: + - name: Extract version from tag + id: extract + run: | + # Extract version from tag (voice/v1.0.0 -> 1.0.0) + VERSION=${GITHUB_REF#refs/tags/voice/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + test-voice: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Test Voice Agent SDK + run: | + make install-dev + make lint-voice + make test-voice + + release-build: + runs-on: ubuntu-latest + needs: [extract-version, test-voice] + outputs: + version: ${{ needs.extract-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Update package version in sdk/voice/speechmatics/voice/__init__.py + run: | + VERSION="${{ needs.extract-version.outputs.version }}" + sed -i "s/0\.0\.0/$VERSION/g" ./sdk/voice/speechmatics/voice/__init__.py + echo "Updated version to: $VERSION" + cat ./sdk/voice/speechmatics/voice/__init__.py | grep __version__ + + - name: Build Voice Agent SDK + run: | + make install-dev + make build-voice + + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: voice-release-dist + path: sdk/voice/dist/ + + pypi-publish: + runs-on: ubuntu-latest + needs: [release-build] + environment: + name: pypi-voice + url: https://pypi.org/project/speechmatics-voice/${{ needs.release-build.outputs.version }} + + steps: + - name: Retrieve release dist + uses: actions/download-artifact@v4 + with: + name: voice-release-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.PYPI_ORG_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b0ca76c2..49d61fe7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,3 +69,23 @@ jobs: run: make test-flow - name: Build Flow SDK run: make build-flow + + test-voice: + name: Test Voice Agent SDK + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make install-dev + - name: Lint Voice Agent SDK + run: make lint-voice + - name: Test Voice Agent SDK + run: make test-voice + - name: Build Voice Agent SDK + run: make build-voice From 1f8cd405d6c662828c9ca3958cfaa92794d468e4 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 22:25:55 +0100 Subject: [PATCH 05/10] missing release info in `RELEASE.md` --- .github/RELEASE.md | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/RELEASE.md b/.github/RELEASE.md index adccaec6..9b6ef51f 100644 --- a/.github/RELEASE.md +++ b/.github/RELEASE.md @@ -5,9 +5,11 @@ This document outlines the release process for the Speechmatics Python SDK packa ## Overview The Speechmatics Python SDK repository contains two separate packages: + - `speechmatics-rt` - Real-Time API Client - `speechmatics-batch` - Batch API Client - `speechmatics-flow` - Flow API Client +- `speechmatics-voice` - Voice Agent API Client Each package is released independently with its own versioning and release workflow. @@ -16,6 +18,7 @@ Each package is released independently with its own versioning and release workf Before creating a release, ensure the following steps are completed: ### Code Quality + - [ ] All tests pass locally (`make test-all`) - [ ] Linting passes (`make lint-all`) - [ ] Type checking passes (`make type-check-all`) @@ -23,10 +26,12 @@ Before creating a release, ensure the following steps are completed: - [ ] Documentation is up to date ### Version Management + - [ ] Review and update README files if needed - [ ] Verify dependencies are correct in `pyproject.toml` ### Testing + - [ ] Test examples with fresh installations - [ ] Verify environment variables work correctly - [ ] Test error handling scenarios @@ -39,6 +44,7 @@ Before creating a release, ensure the following steps are completed: To release a new version of the RT SDK: 1. **Create a Release Tag** + ```bash git tag rt/v1.0.0 git push origin rt/v1.0.0 @@ -46,6 +52,7 @@ To release a new version of the RT SDK: 2. **Automated Workflow** The `release-rt.yaml` workflow will automatically: + - Extract version from tag (e.g., `rt/v1.0.0` → `1.0.0`) - Run comprehensive tests across Python versions - Update version in `sdk/rt/speechmatics/rt/__init__.py` @@ -63,6 +70,7 @@ To release a new version of the RT SDK: To release a new version of the Batch SDK: 1. **Create a Release Tag** + ```bash git tag batch/v1.0.0 git push origin batch/v1.0.0 @@ -70,6 +78,7 @@ To release a new version of the Batch SDK: 2. **Automated Workflow** The `release-batch.yaml` workflow will automatically: + - Extract version from tag (e.g., `batch/v1.0.0` → `1.0.0`) - Run comprehensive tests across Python versions - Update version in `sdk/batch/speechmatics/batch/__init__.py` @@ -87,6 +96,7 @@ To release a new version of the Batch SDK: To release a new version of the Flow SDK: 1. **Create a Release Tag** + ```bash git tag flow/v1.0.0 git push origin flow/v1.0.0 @@ -94,6 +104,7 @@ To release a new version of the Flow SDK: 2. **Automated Workflow** The `release-flow.yaml` workflow will automatically: + - Extract version from tag (e.g., `flow/v1.0.0` → `1.0.0`) - Run comprehensive tests across Python versions - Update version in `sdk/flow/speechmatics/flow/__init__.py` @@ -106,36 +117,72 @@ To release a new version of the Flow SDK: - Update GitHub release notes - Announce the release +### 4. Voice Agent SDK Release + +To release a new version of the Voice Agent SDK: + +1. **Create a Release Tag** + + ```bash + git tag voice/v1.0.0 + git push origin voice/v1.0.0 + ``` + +2. **Automated Workflow** + The `release-voice.yaml` workflow will automatically: + + - Extract version from tag (e.g., `voice/v1.0.0` → `1.0.0`) + - Run comprehensive tests across Python versions + - Update version in `sdk/voice/speechmatics/voice/__init__.py` + - Build the package + - Publish to PyPI + +3. **Manual Steps After Release** + - Verify the package is available on PyPI + - Test installation: `pip install speechmatics-voice==1.0.0` + - Update GitHub release notes + - Announce the release + ## Version Management ### Version Format + Both packages follow semantic versioning (SemVer): + - `MAJOR.MINOR.PATCH` (e.g., `1.2.3`) - `MAJOR.MINOR.PATCH-beta.N` for beta releases (e.g., `1.2.3-beta.1`) ### Version Update Process + 1. **Development**: Versions remain as `0.0.0` in `__init__.py` files 2. **Release**: GitHub Actions automatically updates the version during release 3. **Post-Release**: The updated version remains in the repository ### Tag Naming Convention + - RT SDK: `rt/v{version}` (e.g., `rt/v1.0.0`) - Batch SDK: `batch/v{version}` (e.g., `batch/v1.0.0`) - Flow SDK: `flow/v{version}` (e.g., `flow/v1.0.0`) +- Voice Agent SDK: `voice/v{version}` (e.g., `voice/v1.0.0`) ## Environment Setup ### PyPI Configuration + Both packages are published to PyPI using GitHub Actions with OpenID Connect (OIDC): + - RT SDK: Uses `pypi-rt` environment - Batch SDK: Uses `pypi-batch` environment - Flow SDK: Uses `pypi-flow` environment +- Voice Agent SDK: Uses `pypi-voice` environment ### Required Secrets + No manual secrets are required as the workflows use OIDC for PyPI authentication. ## Testing Matrix Both packages are tested against: + - Python versions: 3.9, 3.10, 3.11, 3.12, 3.13 - Operating system: Ubuntu (latest) From 37c379ea9ec21458b5d59b5d3deeedee6fda1e00 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 22:57:47 +0100 Subject: [PATCH 06/10] fix missing utils package in voice examples --- examples/voice/file_transcription.py | 8 +- examples/voice/microphone_transcription.py | 14 +- examples/voice/utils.py | 222 +++++++++++++++++++++ 3 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 examples/voice/utils.py diff --git a/examples/voice/file_transcription.py b/examples/voice/file_transcription.py index c6ea5e1f..941afeca 100644 --- a/examples/voice/file_transcription.py +++ b/examples/voice/file_transcription.py @@ -10,8 +10,8 @@ from utils import select_audio_output_device from speechmatics.voice import AgentServerMessageType -from speechmatics.voice import DiarizationSpeakerConfig from speechmatics.voice import DiarizationFocusMode +from speechmatics.voice import DiarizationSpeakerConfig from speechmatics.voice import EndOfUtteranceMode from speechmatics.voice import SpeakerSegment from speechmatics.voice import VoiceAgentClient @@ -166,12 +166,12 @@ async def _stream_audio_file( sample_width: int, ) -> None: """Stream audio file in real-time to client and player.""" - CHUNK_SIZE = 320 - chunk_duration = CHUNK_SIZE / sample_rate / sample_width + chunk_size = 320 + chunk_duration = chunk_size / sample_rate / sample_width with wave.open(str(file_path), "rb") as wav_file: while True: - audio_data = wav_file.readframes(CHUNK_SIZE) + audio_data = wav_file.readframes(chunk_size) if not audio_data: print("\nEnd of file reached") break diff --git a/examples/voice/microphone_transcription.py b/examples/voice/microphone_transcription.py index 6f195e96..57fba260 100644 --- a/examples/voice/microphone_transcription.py +++ b/examples/voice/microphone_transcription.py @@ -70,11 +70,7 @@ def _register_event_handlers(client: VoiceAgentClient, logger) -> None: def _format_segment(segment: SpeakerSegment) -> str: """Format speaker segment for display.""" - template = ( - "@{speaker_id}: {text}" - if segment.is_active - else "@{speaker_id} (background): {text}" - ) + template = "@{speaker_id}: {text}" if segment.is_active else "@{speaker_id} (background): {text}" return segment.format_text(template) @client.on(AgentServerMessageType.ADD_SEGMENTS) @@ -89,9 +85,7 @@ def handle_partial_segments(message): segments = [_format_segment(s) for s in message["segments"]] delay = message.get("delay_to_finalize") if delay is not None: - logger.log( - CustomLevels.PARTIAL, f"💬 Partial: {segments} ({delay}s to finals)" - ) + logger.log(CustomLevels.PARTIAL, f"💬 Partial: {segments} ({delay}s to finals)") else: logger.log(CustomLevels.PARTIAL, f"💬 Partial: {segments}") @@ -127,9 +121,7 @@ def _setup_microphone(sample_rate: int, chunk_size: int) -> Microphone | None: return mic -async def _stream_microphone( - mic: Microphone, client: VoiceAgentClient, chunk_size: int -) -> None: +async def _stream_microphone(mic: Microphone, client: VoiceAgentClient, chunk_size: int) -> None: """Stream microphone audio to client.""" while True: frame = await mic.read(chunk_size) diff --git a/examples/voice/utils.py b/examples/voice/utils.py new file mode 100644 index 00000000..03fc7c01 --- /dev/null +++ b/examples/voice/utils.py @@ -0,0 +1,222 @@ +import logging + +import pyaudio + +from speechmatics.rt import Microphone + + +def select_audio_device() -> int | None: + """Interactive microphone device selection. + + Returns: + Device index or None for default device + """ + devices = Microphone.list_devices() + if not devices: + return None + + print("Available microphones:") + for device in devices: + print(f" [{device['index']}] {device['name']} ({device['channels']} channels)") + print() + + return _get_device_choice( + [d["index"] for d in devices], + "Enter device index (or press Enter for default): ", + ) + + +def select_audio_output_device() -> int | None: + """Interactive audio output device selection. + + Returns: + Device index or None for default device + """ + try: + output_devices = _get_output_devices() + if not output_devices: + print("No audio output devices found.") + return None + + print("Available audio output devices:") + for device in output_devices: + print( + f" [{device['index']}] {device['name']} ({device['channels']} channels, {device['sample_rate']}Hz)" + ) + print() + + return _get_device_choice( + [d["index"] for d in output_devices], + "Enter output device index (or press Enter for default): ", + ) + + except Exception as e: + print(f"Error listing audio devices: {e}") + return None + + +def _get_output_devices() -> list[dict]: + """Get list of available output devices.""" + p = pyaudio.PyAudio() + try: + devices = [] + for i in range(p.get_device_count()): + info = p.get_device_info_by_index(i) + if info["maxOutputChannels"] > 0: + devices.append( + { + "index": i, + "name": info["name"], + "channels": info["maxOutputChannels"], + "sample_rate": int(info["defaultSampleRate"]), + } + ) + return devices + finally: + p.terminate() + + +def _get_device_choice(valid_indices: list[int], prompt: str) -> int | None: + """Get user device choice with validation.""" + while True: + try: + choice = input(prompt).strip() + if not choice: + return None + + device_index = int(choice) + if device_index in valid_indices: + return device_index + + print(f"Invalid device index. Choose from: {valid_indices}") + + except ValueError: + print("Please enter a valid number.") + except KeyboardInterrupt: + return None + + +class AudioPlayer: + """Real-time audio player using PyAudio.""" + + def __init__( + self, + sample_rate: int, + channels: int = 1, + sample_width: int = 2, + device_index: int | None = None, + ): + """Initialize audio player. + + Args: + sample_rate: Audio sample rate in Hz + channels: Number of audio channels (1 for mono) + sample_width: Sample width in bytes (2 for 16-bit) + device_index: Output device index (None for default) + """ + self.sample_rate = sample_rate + self.channels = channels + self.sample_width = sample_width + self.device_index = device_index + self.p = None + self.stream = None + + def start(self) -> bool: + """Start audio playback stream.""" + try: + self.p = pyaudio.PyAudio() + if not self.p: + return False + + audio_format = self._get_audio_format() + if not audio_format: + return False + + self.stream = self.p.open( + format=audio_format, + channels=self.channels, + rate=self.sample_rate, + output=True, + output_device_index=self.device_index, + ) + return True + + except Exception as e: + print(f"Error starting audio player: {e}") + return False + + def play(self, audio_data: bytes) -> None: + """Play audio data chunk.""" + if self.stream: + self.stream.write(audio_data) + + def stop(self) -> None: + """Stop and cleanup audio player.""" + if self.stream: + self.stream.stop_stream() + self.stream.close() + self.stream = None + + if self.p: + self.p.terminate() + self.p = None + + def _get_audio_format(self) -> int | None: + """Get PyAudio format from sample width.""" + format_map = {1: pyaudio.paInt8, 2: pyaudio.paInt16, 4: pyaudio.paInt32} + if self.sample_width not in format_map: + print(f"Unsupported sample width: {self.sample_width}") + return None + return format_map[self.sample_width] + + +class CustomLevels: + """Custom logging levels for transcription events.""" + + PARTIAL = 11 # Partial transcription results + FINAL = 12 # Final transcription results + SPEAKER = 15 # Speech activity events + + +class CustomTextFormatter(logging.Formatter): + """Coloured logging formatter for transcription events.""" + + # FORMAT = "%(asctime)s.%(msecs)03d %(levelname)-8s (%(filename)s:%(lineno)d) %(message)s" + FORMAT = "%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s" + + # ANSI colour codes + COLOURS = { + logging.DEBUG: "\033[90m", # Gray + CustomLevels.PARTIAL: "\033[32m", # Green + CustomLevels.FINAL: "\033[33m", # Yellow + CustomLevels.SPEAKER: "\033[36m", # Cyan + } + RESET = "\033[0m" + + def format(self, record): + colour = self.COLOURS.get(record.levelno, self.RESET) + message = super().format(record) + return f"{colour}{message}{self.RESET}" + + +def get_logger(name: str) -> logging.Logger: + """Setup coloured logger for transcription events.""" + logging.basicConfig(level=logging.INFO) + + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logging.getLogger("speechmatics.voice").setLevel(logging.DEBUG) + + # Register custom level names + logging.addLevelName(CustomLevels.PARTIAL, "PARTIAL") + logging.addLevelName(CustomLevels.FINAL, "FINAL") + logging.addLevelName(CustomLevels.SPEAKER, "SPEAKER") + + # Apply custom formatter to all handlers + formatter = CustomTextFormatter( + CustomTextFormatter.FORMAT, datefmt="%Y-%m-%d %H:%M:%S" + ) + for handler in logging.root.handlers: + handler.setFormatter(formatter) + + return logger From f3cb7be9f54b0c2279be8112547ca58674b329d0 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 3 Sep 2025 23:01:25 +0100 Subject: [PATCH 07/10] typo fix in README --- sdk/voice/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/voice/README.md b/sdk/voice/README.md index 506b148d..849e5441 100644 --- a/sdk/voice/README.md +++ b/sdk/voice/README.md @@ -4,7 +4,7 @@ An SDK for working with the Speechmatics Real-Time API optimised for use in voice agents or transcription services. -This uses the Python Real-Time API to process the transcription results from the STT engine and combine them into manageable segments of audio. Taking advantage of speaker diarization, the transcription is grouped into individual speakers, with advanced options to focus on and/or ignore specific speakers. +This uses the Python Real-Time API to process the transcription results from the STT engine and combine them into manageable segments of text. Taking advantage of speaker diarization, the transcription is grouped into individual speakers, with advanced options to focus on and/or ignore specific speakers. See [OVERVIEW.md](OVERVIEW.md) for more information. From f8bea005b32f6f2e56db22360a3265b9ca6d0bc8 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Thu, 4 Sep 2025 00:20:10 +0100 Subject: [PATCH 08/10] support a max delay to emitting finals --- sdk/voice/speechmatics/voice/_client.py | 30 +++++++++++++++---------- sdk/voice/speechmatics/voice/_models.py | 8 ++++++- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/sdk/voice/speechmatics/voice/_client.py b/sdk/voice/speechmatics/voice/_client.py index 09c03dbc..4f46258e 100644 --- a/sdk/voice/speechmatics/voice/_client.py +++ b/sdk/voice/speechmatics/voice/_client.py @@ -218,25 +218,25 @@ def _register_event_handlers(self) -> None: # Recognition started event @self.once(ServerMessageType.RECOGNITION_STARTED) - def _evt_on_recognition_started(message: dict[str, Any]): + def _evt_on_recognition_started(message: dict[str, Any]) -> None: self._start_time = datetime.datetime.now(datetime.timezone.utc) self._is_ready_for_audio = True # Partial transcript event @self.on(ServerMessageType.ADD_PARTIAL_TRANSCRIPT) - def _evt_on_partial_transcript(message: dict[str, Any]): + def _evt_on_partial_transcript(message: dict[str, Any]) -> None: self._handle_transcript(message, is_final=False) # Final transcript event @self.on(ServerMessageType.ADD_TRANSCRIPT) - def _evt_on_final_transcript(message: dict[str, Any]): + def _evt_on_final_transcript(message: dict[str, Any]) -> None: self._handle_transcript(message, is_final=True) # End of utterance event if self._end_of_utterance_mode == EndOfUtteranceMode.FIXED: @self.on(ServerMessageType.END_OF_UTTERANCE) - def _evt_on_end_of_utterance(message: dict[str, Any]): + def _evt_on_end_of_utterance(message: dict[str, Any]) -> None: self._logger.debug("End of utterance") self.finalize_segments() @@ -390,7 +390,7 @@ async def _handle_transcript_async(self, message: dict[str, Any], is_final: bool self._processor_task.cancel() # Send transcription frames after delay - async def process_after_delay(delay: float): + async def process_after_delay(delay: float) -> None: await asyncio.sleep(delay) await self._process_speech_fragments() self._processor_task = None @@ -699,6 +699,12 @@ def _calc_delay_to_finalize( # TODO - Other checks / end of turn detection + # Constrain to `end_of_utterance_max_delay` + if emit_final_delay is not None: + emit_final_delay = min(emit_final_delay, self._config.end_of_utterance_max_delay) + else: + emit_final_delay = self._config.end_of_utterance_max_delay + # Emit segments should_emit = True @@ -713,7 +719,7 @@ def _calc_delay_to_finalize( # Return the result return should_emit, emit_final_delay - def finalize_segments(self, ttl: Optional[float] = None): + def finalize_segments(self, ttl: Optional[float] = None) -> None: """Finalize segments. This function will emit segments in the buffer without any further checks @@ -724,7 +730,7 @@ def finalize_segments(self, ttl: Optional[float] = None): ttl: Optional delay before finalizing partial segments (defaults to 0.01 seconds). """ - async def _emit(): + async def _emit() -> None: await asyncio.sleep(ttl or 0.01) await self._emit_final_segments() @@ -775,7 +781,7 @@ async def _emit_final_segments(self, delay: float = 0) -> None: if self._finals_emitter_task is not None: self._finals_emitter_task.cancel() - async def emit_finals_now(): + async def emit_finals_now() -> None: """Emit final segments.""" # Skip if no segments @@ -798,7 +804,7 @@ async def emit_finals_now(): # Only include segments up to the last active segment trimmed_view = SpeakerFragmentView( fragments=fragments_to_emit, - base_time=view.base_time, + base_time=self._start_time, focus_speakers=view.focus_speakers, annotate_segments=False, ) @@ -821,7 +827,7 @@ async def emit_finals_now(): # Update the current view self._current_view = SpeakerFragmentView( fragments=self._speech_fragments, - base_time=self._trim_before_time, + base_time=self._start_time, focus_speakers=self._dz_config.focus_speakers, annotate_segments=False, ) @@ -829,7 +835,7 @@ async def emit_finals_now(): # Reset previous fragments self._last_speech_fragments = self._speech_fragments.copy() - async def emit_finals_after_delay(_delay: float): + async def emit_finals_after_delay(_delay: float) -> None: """Yield before emitting final segments after a delay.""" # Wait for the delay (can be cancelled) @@ -851,7 +857,7 @@ async def emit_finals_after_delay(_delay: float): else: self._finals_emitter_task = asyncio.create_task(emit_finals_now()) - def _vad_evaluation(self, fragments: list[SpeechFragment]): + def _vad_evaluation(self, fragments: list[SpeechFragment]) -> None: """Emit a VAD event. This will emit `SPEECH_STARTED` and `SPEECH_ENDED` events to the client and is diff --git a/sdk/voice/speechmatics/voice/_models.py b/sdk/voice/speechmatics/voice/_models.py index a79a34b0..05ebf48d 100644 --- a/sdk/voice/speechmatics/voice/_models.py +++ b/sdk/voice/speechmatics/voice/_models.py @@ -122,6 +122,11 @@ class VoiceAgentConfig: word frames. The value must be lower than max_delay. Defaults to 0.5. + end_of_utterance_max_delay: Maximum delay in seconds for end of utterance delay. + The delay is used to wait for any further transcribed words before emitting the final + word frames. The value must be greater than end_of_utterance_silence_trigger. + Defaults to 10.0. + end_of_utterance_mode: End of utterance delay mode. When ADAPTIVE is used, the delay can be adjusted on the content of what the most recent speaker has said, such as rate of speech and whether they have any pauses or disfluencies. When FIXED is used, @@ -183,6 +188,7 @@ class VoiceAgentConfig: # Features max_delay: float = 1.0 end_of_utterance_silence_trigger: float = 0.5 + end_of_utterance_max_delay: float = 10.0 end_of_utterance_mode: EndOfUtteranceMode = EndOfUtteranceMode.FIXED additional_vocab: list[AdditionalVocabEntry] = field(default_factory=list) punctuation_overrides: Optional[dict] = None @@ -451,7 +457,7 @@ def end_time(self) -> float: """Return the end time of the segment.""" return self.fragments[-1].end_time - def __str__(self): + def __str__(self) -> str: """Return a string representation of the object.""" meta = { "speaker_id": self.speaker_id, From c417df0f5753ab5175e3455699ab5117e4777f81 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 9 Sep 2025 12:26:15 +0100 Subject: [PATCH 09/10] add RT as dependency --- sdk/voice/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/voice/pyproject.toml b/sdk/voice/pyproject.toml index 2002efee..629e27ee 100644 --- a/sdk/voice/pyproject.toml +++ b/sdk/voice/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" authors = [{ name = "Speechmatics", email = "support@speechmatics.com" }] license = "MIT" requires-python = ">=3.9" -dependencies = ["websockets>=10.0"] +dependencies = ["speechmatics-rt"] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", From 9ea7199c05f939236ea9d779fa9245d4b3370f34 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Tue, 9 Sep 2025 12:33:50 +0100 Subject: [PATCH 10/10] add raw results to payload --- sdk/voice/speechmatics/voice/_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/voice/speechmatics/voice/_client.py b/sdk/voice/speechmatics/voice/_client.py index 4f46258e..a7403164 100644 --- a/sdk/voice/speechmatics/voice/_client.py +++ b/sdk/voice/speechmatics/voice/_client.py @@ -375,6 +375,10 @@ async def _handle_transcript_async(self, message: dict[str, Any], is_final: bool is_final: Whether the data is final or partial. """ + # Debug + if DEBUG_MORE: + self._logger.debug(json.dumps(message)) + # Add the speech fragments fragments_available = await self._add_speech_fragments( message=message, @@ -482,7 +486,7 @@ async def _add_speech_fragments(self, message: dict[str, Any], is_final: bool = content=alt.get("content", ""), speaker=alt.get("speaker", None), confidence=alt.get("confidence", 1.0), - result=result, + result={"final": is_final, **result}, ) # Check fragment is after trim time