From 13307eafd52d388fb733ff8e45af709367f3e97c Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Fri, 2 Oct 2026 00:49:18 +0200 Subject: [PATCH] fix(whisperx): keep the transcript when diarization fails, report real errors AudioTranscription caught every exception and returned an empty TranscriptResult. A failed diarization step therefore discarded a transcript that was already finished: with an HF token that has not accepted the terms of the gated pyannote pipeline, the download fails with 403 and every transcription came back as an empty text with HTTP 200. Diarization now degrades: if it fails, the transcript is returned without speaker labels and the reason is logged. Any other failure aborts the call with INTERNAL instead of pretending success with an empty text. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Stefan Walcz --- backend/python/whisperx/backend.py | 21 +++++++++++-------- .../python/whisperx/test_transcript_utils.py | 18 ++++++++++++++++ backend/python/whisperx/transcript_utils.py | 14 +++++++++++++ docs/content/features/audio-to-text.md | 2 ++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/backend/python/whisperx/backend.py b/backend/python/whisperx/backend.py index dc62022862b2..c3f900e3e59c 100644 --- a/backend/python/whisperx/backend.py +++ b/backend/python/whisperx/backend.py @@ -16,7 +16,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) from grpc_auth import get_auth_interceptors -from transcript_utils import require_diarization_token, seconds_to_nanoseconds +from transcript_utils import diarize_or_keep, require_diarization_token, seconds_to_nanoseconds @@ -112,13 +112,15 @@ def AudioTranscription(self, request, context): # Diarize if requested and HF token is available if request.diarize and self.hf_token: - if self.diarize_pipeline is None: - self.diarize_pipeline = DiarizationPipeline( - token=self.hf_token, - device=self.device, - ) - diarize_segments = self.diarize_pipeline(audio) - transcript = whisperx.assign_word_speakers(diarize_segments, transcript) + def _diarize(t): + if self.diarize_pipeline is None: + self.diarize_pipeline = DiarizationPipeline( + token=self.hf_token, + device=self.device, + ) + return whisperx.assign_word_speakers(self.diarize_pipeline(audio), t) + + transcript = diarize_or_keep(transcript, _diarize, lambda m: print(m, file=sys.stderr)) # Build result segments for idx, seg in enumerate(transcript["segments"]): @@ -137,8 +139,9 @@ def AudioTranscription(self, request, context): text += seg_text except Exception as err: + # Report the failure instead of an empty, successful-looking result. print(f"Unexpected {err=}, {type(err)=}", file=sys.stderr) - return backend_pb2.TranscriptResult(segments=[], text="") + context.abort(grpc.StatusCode.INTERNAL, f"transcription failed: {err}") return backend_pb2.TranscriptResult(segments=resultSegments, text=text) diff --git a/backend/python/whisperx/test_transcript_utils.py b/backend/python/whisperx/test_transcript_utils.py index debe2ea6ecf7..708d4bde2f8a 100644 --- a/backend/python/whisperx/test_transcript_utils.py +++ b/backend/python/whisperx/test_transcript_utils.py @@ -21,5 +21,23 @@ def test_seconds_are_serialized_as_nanoseconds(self): ) + def test_failed_diarization_keeps_the_transcript(self): + transcript = {"segments": [{"text": "Die Rechnung"}]} + logged = [] + + def refused(_): + raise RuntimeError("403 Client Error: gated repo") + + result = transcript_utils.diarize_or_keep(transcript, refused, logged.append) + self.assertIs(result, transcript) + self.assertIn("403", logged[0]) + + def test_successful_diarization_is_returned(self): + transcript = {"segments": [{"text": "Die Rechnung"}]} + with_speakers = {"segments": [{"text": "Die Rechnung", "speaker": "SPEAKER_00"}]} + result = transcript_utils.diarize_or_keep(transcript, lambda _: with_speakers, lambda _: None) + self.assertIs(result, with_speakers) + + if __name__ == "__main__": unittest.main() diff --git a/backend/python/whisperx/transcript_utils.py b/backend/python/whisperx/transcript_utils.py index a8ac5751057e..2211cf670ae6 100644 --- a/backend/python/whisperx/transcript_utils.py +++ b/backend/python/whisperx/transcript_utils.py @@ -10,3 +10,17 @@ def require_diarization_token(diarize, token): def seconds_to_nanoseconds(seconds): """Convert WhisperX timestamps to the duration unit used by LocalAI.""" return int(seconds * 1_000_000_000) + + +def diarize_or_keep(transcript, diarize, log): + """Run diarization; if it fails, keep the transcript without speakers. + + Diarization is an add-on to a finished transcript. A refused download of + the gated pyannote pipeline (403) or any other diarization error must not + throw the transcript away. + """ + try: + return diarize(transcript) + except Exception as err: # noqa: BLE001 - any diarization failure degrades + log(f"Diarization failed, returning transcript without speakers: {err!r}") + return transcript diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 0a4b6ee654e5..e90423749cdf 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -112,6 +112,8 @@ In addition to `file` and `model`, the endpoint accepts the following multipart | `stream` | When `true`, the endpoint emits an SSE stream of `transcript.text.delta` events followed by a final `transcript.text.done` event. | | `diarize` | LocalAI extension - speaker diarization. WhisperX requires `HF_TOKEN`; requests fail with `FailedPrecondition` when it is missing. | +If speaker diarization fails after transcription succeeded, the WhisperX backend logs the error and returns the transcript without speaker labels. Other transcription failures return an error instead of an empty transcript. Diarization still requires `HF_TOKEN`. + The response body for `verbose_json` includes `text`, `language`, `duration`, and `segments[]` (with `speaker` populated when diarization is enabled). ## Streaming transcriptions