feat(audio): remember speakers from diarization

backend/backend.proto

@@ -830,6 +830,7 @@ message DiarizeRequest {
   // Registered voices the backend may use to name speakers. Only backends that
   // identify speakers themselves read this; others ignore it.
   repeated KnownVoice known_voices = 12;
+  bool include_speaker_profiles = 13; // opt-in sensitive embeddings; unsupported backends must reject
 }
 
 message DiarizeSegment {
@@ -846,12 +847,14 @@ message DiarizeSegment {
 // names the encoder that produced it, so a backend with a different encoder can
 // refuse vectors that are not comparable.
 message KnownVoice {
+  string id = 4; // registration ID; native keys must not aggregate duplicate display names
   string name = 1;
   repeated float embedding = 2;
   string model = 3;
 }
 
 message DiarizeResponse {
+  string speaker_profiles_json = 5; // versioned speaker_profiles object only; absent by default
   repeated DiarizeSegment segments = 1;
   int32  num_speakers = 2;             // count of distinct speaker labels in `segments`
   float  duration = 3;                 // total audio duration in seconds (0 if unknown)
@@ -907,6 +910,11 @@ message MemoryUsageData {
   map<string, uint64> breakdown = 2;
 }
 
+message SpeakerEncoder {
+  string identity = 1; // sha256 of loaded GGUF bytes
+  int32 dimension = 2;
+}
+
 message StatusResponse {
   enum State {
     UNINITIALIZED = 0;
@@ -916,6 +924,7 @@ message StatusResponse {
   }
   State state = 1;
   MemoryUsageData memory = 2;
+  SpeakerEncoder speaker_encoder = 3; // trusted metadata from the loaded server encoder, never request data
 }
 
 message Message {

backend/go/parakeet-cpp/Makefile

@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb
+# Upstream pin lives below as PARAKEET_VERSION?=bee7c14dfcc23613df58176c59a40459e7b47095
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=8c8cec0c4564610a0a4b30a8a6f2ead15d1a76fb
+PARAKEET_VERSION?=bee7c14dfcc23613df58176c59a40459e7b47095
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go

backend/go/parakeet-cpp/diarize.go

@@ -109,6 +109,11 @@ func unsupportedDiarizeFields(req *pb.DiarizeRequest) []string {
 // turn); otherwise, or when no ASR companion is loaded, segments carry no
 // text (parakeet_capi_diarize_pcm) and no error is raised.
 func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error) {
+	if req.GetIncludeSpeakerProfiles() {
+		if CppDiarizeProfilesPCMJSON == nil || CppSpeakerIdentity == nil || CppSpeakerDim == nil || p.spkCtx == 0 {
+			return pb.DiarizeResponse{}, status.Error(codes.Unimplemented, "parakeet-cpp: speaker profiles require a loaded speaker encoder and profile-capable library")
+		}
+	}
 	if p.diarCtx == 0 {
 		return pb.DiarizeResponse{}, status.Error(codes.FailedPrecondition,
 			"parakeet-cpp: model is not a diarization model")
@@ -136,6 +141,11 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error
 	}
 
 	wantText := req.GetIncludeText() && p.ctxPtr != 0 && CppTranscribeAndDiarizeJSON != nil
+	if req.GetIncludeSpeakerProfiles() {
+		// Preserve the no-ASR fallback, but never silently omit text when an
+		// ASR companion is loaded and its timestamped PCM API is unavailable.
+		wantText = req.GetIncludeText() && p.ctxPtr != 0
+	}
 
 	var reg uintptr
 	if len(req.GetKnownVoices()) > 0 && p.spkCtx != 0 {
@@ -146,43 +156,68 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error
 		defer p.freeSpeakerRegistry(reg)
 	}
 
-	raw, err := p.diarizeCall(pcm, wantText, reg)
+	raw, err := p.diarizeCall(pcm, wantText, reg, req.GetIncludeSpeakerProfiles())
 	if err != nil {
 		return pb.DiarizeResponse{}, err
 	}
+	var profiles string
+	if req.GetIncludeSpeakerProfiles() {
+		var doc struct {
+			Profiles json.RawMessage `json:"speaker_profiles"`
+		}
+		if err := json.Unmarshal([]byte(raw), &doc); err != nil || len(doc.Profiles) == 0 || string(doc.Profiles) == "null" {
+			return pb.DiarizeResponse{}, status.Error(codes.Internal, "parakeet-cpp: missing speaker profiles")
+		}
+		profiles = string(doc.Profiles)
+	}
 	segments, err := parseDiarizeDoc(raw, wantText)
 	if err != nil {
 		return pb.DiarizeResponse{}, err
 	}
 
+	displayNames := voiceNames(req.GetKnownVoices())
+	for _, segment := range segments {
+		if name, ok := displayNames[segment.Name]; ok {
+			segment.Name = name
+		}
+	}
+
 	segments = applyDurationFilters(segments, req.GetMinDurationOn(), req.GetMinDurationOff())
 	renumberDiarizeSegments(segments)
 
 	return pb.DiarizeResponse{
-		Segments:    segments,
-		NumSpeakers: distinctDiarizeSpeakers(segments),
-		Duration:    duration,
+		SpeakerProfilesJson: profiles,
+		Segments:            segments,
+		NumSpeakers:         distinctDiarizeSpeakers(segments),
+		Duration:            duration,
 	}, nil
 }
 
-// diarizeCall runs the single C call Diarize needs (transcribe_and_diarize_json
-// when wantText, else diarize_pcm) under engineMu, and returns the raw JSON
+// diarizeCall runs diarization and optional ASR under engineMu, returning a JSON
 // document. p.diarCtx (and, on the include_text path, p.ctxPtr) is re-checked
 // under the lock before the C call: Diarize's own p.diarCtx==0/wantText checks
 // run before this lock is taken, so a Free() racing in between (which zeroes
 // those fields under the same engineMu) would otherwise reach the C side with
 // a freed context. last_error is ctx-shared, so it is read under the same
 // lock as the failing call.
-func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool, reg uintptr) (string, error) {
+func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool, reg uintptr, wantProfiles bool) (string, error) {
 	p.engineMu.Lock()
 	defer p.engineMu.Unlock()
 
 	if p.diarCtx == 0 || (wantText && p.ctxPtr == 0) || (reg != 0 && p.spkCtx == 0) {
 		return "", grpcerrors.ModelNotLoaded("parakeet-cpp")
 	}
 
+	if wantProfiles && (p.spkCtx == 0 || CppDiarizeProfilesPCMJSON == nil) {
+		return "", status.Error(codes.Unimplemented, "parakeet-cpp: speaker profile capability unavailable")
+	}
+	if wantProfiles && wantText && CppTranscribePcmBatchJSON == nil {
+		return "", status.Error(codes.Unimplemented, "parakeet-cpp: combined profile export requires timestamped PCM transcription")
+	}
 	var cstr uintptr
 	switch {
+	case wantProfiles:
+		cstr = CppDiarizeProfilesPCMJSON(p.diarCtx, p.spkCtx, reg, &pcm[0], int32(len(pcm)), 16000, p.speakerAccept, p.speakerMargin)
 	case reg != 0 && wantText:
 		if CppTranscribeAndDiarizeNamedJSON == nil {
 			return "", status.Error(codes.Unimplemented,
@@ -203,13 +238,63 @@ func (p *ParakeetCpp) diarizeCall(pcm []float32, wantText bool, reg uintptr) (st
 		cstr = CppDiarizePCM(p.diarCtx, &pcm[0], int32(len(pcm)), 16000)
 	}
 	if cstr == 0 {
-		return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText, reg != 0))
+		return "", fmt.Errorf("parakeet-cpp: diarize failed: %s", diarizeLastError(p, wantText, reg != 0 || wantProfiles))
 	}
 	raw := goStringFromCPtr(cstr)
 	CppFreeString(cstr)
+	if wantProfiles && wantText {
+		// Keep both contexts alive and last_error protected through both calls.
+		// Only the profile call diarizes: ASR contributes words, never slots.
+		cstr = CppTranscribePcmBatchJSON(p.ctxPtr, pcm, []int32{int32(len(pcm))}, 1, 16000, 0)
+		if cstr == 0 {
+			return "", fmt.Errorf("parakeet-cpp: transcribe failed: %s", CppLastError(p.ctxPtr))
+		}
+		asr := goStringFromCPtr(cstr)
+		CppFreeString(cstr)
+		return composeProfileTranscript(raw, asr)
+	}
 	return raw, nil
 }
 
+// composeProfileTranscript preserves the native profiles and slot-keyed names,
+// adding utterances from timestamped words on that same diarization timeline.
+func composeProfileTranscript(raw, asr string) (string, error) {
+	var doc diarizePCMDoc
+	if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+		return "", fmt.Errorf("parakeet-cpp: decode profile diarization: %w", err)
+	}
+	var transcripts []transcriptJSON
+	if err := json.Unmarshal([]byte(asr), &transcripts); err != nil {
+		return "", fmt.Errorf("parakeet-cpp: decode transcript: %w", err)
+	}
+	if len(transcripts) != 1 {
+		return "", fmt.Errorf("parakeet-cpp: expected one transcript, got %d", len(transcripts))
+	}
+	t := transcripts[0]
+	if len(t.Words) == 0 && strings.TrimSpace(t.Text) != "" {
+		return "", fmt.Errorf("parakeet-cpp: transcript has no timestamped words")
+	}
+	groups, slots := splitAtSpeakerChanges([][]transcriptWord{t.Words}, assignSpeakers(t.Words, doc.Segments))
+	utterances := make([]diarizeUtteranceJSON, 0, len(groups))
+	for i, group := range groups {
+		parts := make([]string, len(group))
+		for j, word := range group {
+			parts[j] = word.W
+		}
+		utterances = append(utterances, diarizeUtteranceJSON{
+			Speaker: slots[i], Start: group[0].Start, End: group[len(group)-1].End,
+			Text: strings.TrimSpace(strings.Join(parts, " ")),
+		})
+	}
+	var fields map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(raw), &fields); err != nil {
+		return "", err
+	}
+	fields["utterances"], _ = json.Marshal(utterances)
+	combined, err := json.Marshal(fields)
+	return string(combined), err
+}
+
 // diarizeLastError reads last_error off p.diarCtx and, on the include_text
 // path, p.ctxPtr too — the failing call is CppTranscribeAndDiarizeJSON there,
 // and either side of the pairing may be the one that set it — then joins

backend/go/parakeet-cpp/diarize_test.go

@@ -259,7 +259,7 @@ var _ = Describe("ParakeetCpp.Diarize", func() {
 		// Simulate a Free() racing between Diarize's own diarCtx==0 check and
 		// diarizeCall's lock, exactly as it zeroes diarCtx under engineMu.
 		p.diarCtx = 0
-		_, err := p.diarizeCall(make([]float32, 10), false, 0)
+		_, err := p.diarizeCall(make([]float32, 10), false, 0, false)
 		Expect(grpcerrors.IsModelNotLoaded(err)).To(BeTrue())
 		Expect(called).To(BeFalse(), "no C call once diarCtx was cleared")
 	})
@@ -311,6 +311,24 @@ var _ = Describe("ParakeetCpp.Diarize", func() {
 			}
 		})
 
+		It("replays distinct IDs and translates duplicate display names offline", func() {
+			vectors := map[string]float32{}
+			CppSpeakerRegistryAddEmbedding = func(_ uintptr, key string, emb *float32, _ int32) int32 { vectors[key] = *emb; return 0 }
+			CppDiarizeNamedPCMJSON = func(_, _, _ uintptr, _ *float32, _, _ int32, _, _ float32) uintptr {
+				return pool.cstr(`{"segments":[{"speaker":0,"start":0,"end":2},{"speaker":1,"start":2,"end":4}],"names":{"0":{"name":"a","score":0.9},"1":{"name":"b","score":0.8}}}`)
+			}
+			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2}
+			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: []*pb.KnownVoice{
+				{Id: "a", Name: "Ada", Embedding: []float32{1, 0}}, {Id: "b", Name: "Ada", Embedding: []float32{0, 1}},
+			}})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(vectors).To(Equal(map[string]float32{"a": 1, "b": 0}))
+			Expect(res.Segments[0].Name).To(Equal("Ada"))
+			Expect(res.Segments[1].Name).To(Equal("Ada"))
+			Expect(res.Segments[0].Speaker).NotTo(Equal(res.Segments[1].Speaker))
+			Expect(res.SpeakerProfilesJson).To(BeEmpty())
+		})
+
 		It("puts the registered names on the segments and frees the registry", func() {
 			p := &ParakeetCpp{diarCtx: 1, spkCtx: 2, speakerAccept: 0.7, speakerMargin: 0.05}
 			res, err := p.Diarize(&pb.DiarizeRequest{Dst: diarizeWav(5), KnownVoices: ada})

backend/go/parakeet-cpp/goparakeetcpp.go

@@ -109,6 +109,8 @@ var (
 	// CppTranscribeAndDiarizeNamedJSON are ABI v9; CppSpeakerRegistryAddEmbedding and
 	// CppDiarizeNamedPCMJSON are ABI v10. All are nil on an older libparakeet.so, and
 	// Load refuses speaker_model: unless the v10 ones are present.
+	CppSpeakerIdentity             func(ctx uintptr) uintptr
+	CppDiarizeProfilesPCMJSON      func(diar, speaker, reg uintptr, samples *float32, n, sampleRate int32, acceptThreshold, margin float32) uintptr
 	CppSpeakerDim                  func(ctx uintptr) int32
 	CppSpeakerRegistryNew          func() uintptr
 	CppSpeakerRegistryFree         func(reg uintptr)

backend/go/parakeet-cpp/main.go

@@ -132,6 +132,15 @@ func main() {
 		purego.RegisterLibFunc(&CppDiarizeNamedPCMJSON, lib, "parakeet_capi_diarize_named_pcm_json")
 	}
 
+	for _, lf := range []LibFuncs{
+		{&CppSpeakerIdentity, "parakeet_capi_speaker_identity"},
+		{&CppSpeakerDim, "parakeet_capi_speaker_dim"},
+		{&CppDiarizeProfilesPCMJSON, "parakeet_capi_diarize_profiles_pcm_json"},
+	} {
+		if sym, err := purego.Dlsym(lib, lf.Name); err == nil && sym != 0 {
+			purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name)
+		}
+	}
 	fmt.Fprintf(os.Stderr, "[parakeet-cpp] ABI=%d\n", CppAbiVersion())
 
 	flag.Parse()

backend/go/parakeet-cpp/profiles.go

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: MIT
+package main
+
+import pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+
+// Status exposes only metadata derived from the loaded encoder. The native
+// identity is borrowed, so copy it while holding the context lifetime lock.
+func (p *ParakeetCpp) Status() (pb.StatusResponse, error) {
+	result, err := p.Base.Status()
+	if err != nil {
+		return pb.StatusResponse{}, err
+	}
+	p.engineMu.Lock()
+	defer p.engineMu.Unlock()
+	if p.spkCtx != 0 && CppSpeakerIdentity != nil && CppSpeakerDim != nil {
+		identity := CppSpeakerIdentity(p.spkCtx)
+		dim := CppSpeakerDim(p.spkCtx)
+		if identity != 0 && dim > 0 {
+			result.SpeakerEncoder = &pb.SpeakerEncoder{Identity: goStringFromCPtr(identity), Dimension: dim}
+		}
+	}
+	return pb.StatusResponse{
+		State:          result.GetState(),
+		Memory:         result.GetMemory(),
+		SpeakerEncoder: result.GetSpeakerEncoder(),
+	}, nil
+}