From 6211442ee016fbeed7b1beacc672808b42595c71 Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Mon, 19 Oct 2026 23:38:00 -0500 Subject: [PATCH] fix(router): SmartChatResponse.Model names the model that answered When the gateway reroutes (free health breaker open, or a free-cascade fallback) Model reported the requested id. It is now the response body's model, else the X-Served-Model / X-Fallback-Model header, else the requested id. New SmartChatResponse.RequestedModel keeps the router's pick; existing fields are unchanged. --- CHANGELOG.md | 8 +++ README.md | 7 +- client.go | 26 ++++++-- router.go | 53 +++++++++++---- router_served_model_test.go | 124 ++++++++++++++++++++++++++++++++++++ 5 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 router_served_model_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 90cb11e..de4e023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to blockrun-llm-go will be documented in this file. ## Unreleased +- **fix(router): `SmartChatResponse.Model` names the model that answered.** + When the gateway reroutes a call (a free model whose health breaker is open, + or a free-cascade fallback), `Model` used to report the requested id. It is + now read from the response body's `model`, else the `X-Served-Model` / + `X-Fallback-Model` header, else the requested id. New + `SmartChatResponse.RequestedModel` (`json:"requested_model"`) keeps the + router's pick, as `Routing.Model` already did. Additive; no field changed + name or type. - **fix(router): SmartChat no longer routes to ids the gateway does not serve.** The free profile named `nvidia/nemotron-3-super-120b` (gone from `/v1/models`) and `nvidia/nemotron-3-nano-omni-30b-a3b-reasoning` diff --git a/README.md b/README.md index a86a5e8..8b43f05 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ reply, _ := client.Chat(ctx, "nvidia/nemotron-3.5-lightning", "Explain x402 in 1 result, _ := client.SmartChat(ctx, "What is 2+2?", &blockrun.SmartChatOptions{ RoutingProfile: blockrun.RoutingFree, }) -fmt.Println(result.Model) // e.g. "nvidia/nemotron-3.5-lightning" +fmt.Println(result.RequestedModel) // e.g. "nvidia/nemotron-3.5-lightning" (router's pick) +fmt.Println(result.Model) // the model that actually answered fmt.Println(result.Response) // "4" ``` @@ -523,8 +524,8 @@ resp, err := client.SmartChat(ctx, "Prove P != NP", &blockrun.SmartChatOptions{ > (nano-omni for Simple, nemotron-3.5-lightning for the rest). When a primary's > health breaker is open, the gateway reroutes the call along its free ladder > (nemotron-3-super-120b → nemotron-3-nano-omni-30b-a3b-reasoning → -> llama-3.2-11b-vision), so `result.Model` names the requested primary and -> the model that served can differ. `nvidia/nemotron-3-super-120b` is hidden +> llama-3.2-11b-vision). `result.Model` reports the model that served the +> call and `result.RequestedModel` the primary the router chose. `nvidia/nemotron-3-super-120b` is hidden > from the catalog and is never routed to directly. > `moonshot/kimi-k2.7` is hidden from the catalog: **eco** Simple moved to > `google/gemini-2.5-flash-lite` and **auto** Simple to `google/gemini-2.5-flash`. diff --git a/client.go b/client.go index 08138bb..b8ec94b 100644 --- a/client.go +++ b/client.go @@ -136,22 +136,34 @@ func (c *LLMClient) ChatWithSystem(ctx context.Context, model, prompt, system st if err != nil { return "", err } + return firstChoiceContent(resp) +} +// firstChoiceContent returns the first choice's text, or an error when the +// gateway answered with no choices. +func firstChoiceContent(resp *ChatResponse) (string, error) { if len(resp.Choices) == 0 { return "", &APIError{Message: "No choices in response"} } - return resp.Choices[0].Message.Content, nil } // ChatCompletion sends a full chat completion request (OpenAI-compatible). func (c *LLMClient) ChatCompletion(ctx context.Context, model string, messages []ChatMessage, opts *ChatCompletionOptions) (*ChatResponse, error) { + resp, _, err := c.chatCompletionHeaders(ctx, model, messages, opts) + return resp, err +} + +// chatCompletionHeaders is ChatCompletion plus the final HTTP response +// headers, which carry the gateway's reroute metadata (X-Served-Model, +// X-Fallback-Model). Headers are nil when the response came from the cache. +func (c *LLMClient) chatCompletionHeaders(ctx context.Context, model string, messages []ChatMessage, opts *ChatCompletionOptions) (*ChatResponse, http.Header, error) { // Validate inputs if model == "" { - return nil, &ValidationError{Field: "model", Message: "Model is required"} + return nil, nil, &ValidationError{Field: "model", Message: "Model is required"} } if len(messages) == 0 { - return nil, &ValidationError{Field: "messages", Message: "At least one message is required"} + return nil, nil, &ValidationError{Field: "messages", Message: "At least one message is required"} } // Build request body @@ -197,17 +209,17 @@ func (c *LLMClient) ChatCompletion(ctx context.Context, model string, messages [ body["max_tokens"] = maxTokens // Make request with payment handling - respBytes, err := c.doRequest(ctx, "/v1/chat/completions", body) + respBytes, headers, err := c.doRequestHeaders(ctx, "/v1/chat/completions", body) if err != nil { - return nil, err + return nil, nil, err } var chatResp ChatResponse if err := json.Unmarshal(respBytes, &chatResp); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) + return nil, nil, fmt.Errorf("failed to decode response: %w", err) } - return &chatResp, nil + return &chatResp, headers, nil } // GetCostSummary returns an aggregate summary of all costs logged to the persistent JSONL file. diff --git a/router.go b/router.go index 9a259e6..d5c6d8b 100644 --- a/router.go +++ b/router.go @@ -3,6 +3,7 @@ package blockrun import ( "context" "fmt" + "net/http" "regexp" "strings" ) @@ -37,10 +38,32 @@ type RoutingDecision struct { } // SmartChatResponse wraps the chat response with routing metadata. +// +// Model is the model that actually answered. The gateway can serve a +// different model than the one requested (a free model whose health breaker +// is open is rerouted along the free ladder), so Model is read from the +// response: the body's "model" field, else the X-Served-Model or +// X-Fallback-Model header, else the requested id. RequestedModel (and +// Routing.Model) is the id the router chose and sent. type SmartChatResponse struct { - Response string `json:"response"` - Model string `json:"model"` - Routing RoutingDecision `json:"routing"` + Response string `json:"response"` + Model string `json:"model"` + RequestedModel string `json:"requested_model"` + Routing RoutingDecision `json:"routing"` +} + +// servedModel names the model that answered a chat call: the response body's +// model field, then the gateway's reroute headers, then the requested id. +func servedModel(requested string, resp *ChatResponse, headers http.Header) string { + if resp != nil && resp.Model != "" { + return resp.Model + } + for _, h := range []string{"X-Served-Model", "X-Fallback-Model"} { + if v := headers.Get(h); v != "" { + return v + } + } + return requested } // SmartChatOptions configures a SmartChat request. @@ -121,8 +144,8 @@ type SmartChatOptions struct { // llama-3.2-11b-vision within a 90s budget. A client retry on top of that // would only stack waits. So each tier names the sibling SDKs' primary, and // the gateway's ladder supplies the fallbacks, ending at llama-3.2-11b-vision. -// When the gateway reroutes, SmartChatResponse.Model still names the -// requested primary, not the model that served the call. +// When the gateway reroutes, SmartChatResponse.Model reports the model that +// served the call (see servedModel) and RequestedModel keeps the primary. var routingTable = map[RoutingProfile]map[RoutingTier]string{ RoutingFree: { TierSimple: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", TierMedium: "nvidia/nemotron-3.5-lightning", @@ -276,19 +299,25 @@ func (c *LLMClient) SmartChat(ctx context.Context, prompt string, opts *SmartCha } // Make the chat call - var response string + var messages []ChatMessage if opts != nil && opts.System != "" { - response, err = c.ChatWithSystem(ctx, decision.Model, prompt, opts.System) - } else { - response, err = c.Chat(ctx, decision.Model, prompt) + messages = append(messages, ChatMessage{Role: "system", Content: opts.System}) + } + messages = append(messages, ChatMessage{Role: "user", Content: prompt}) + + resp, headers, err := c.chatCompletionHeaders(ctx, decision.Model, messages, nil) + if err != nil { + return nil, err } + response, err := firstChoiceContent(resp) if err != nil { return nil, err } return &SmartChatResponse{ - Response: response, - Model: decision.Model, - Routing: decision, + Response: response, + Model: servedModel(decision.Model, resp, headers), + RequestedModel: decision.Model, + Routing: decision, }, nil } diff --git a/router_served_model_test.go b/router_served_model_test.go new file mode 100644 index 0000000..0edf3f0 --- /dev/null +++ b/router_served_model_test.go @@ -0,0 +1,124 @@ +package blockrun + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// servedModelServer answers /v1/chat/completions with bodyModel in the JSON +// body (omitted when empty) and the given headers. +func servedModelServer(t *testing.T, bodyModel string, headers map[string]string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + http.Error(w, "not found", http.StatusNotFound) + return + } + resp := map[string]any{ + "id": "test", + "object": "chat.completion", + "created": 1700000000, + "choices": []map[string]any{{ + "index": 0, + "message": map[string]string{"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + }}, + } + if bodyModel != "" { + resp["model"] = bodyModel + } + for k, v := range headers { + w.Header().Set(k, v) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + })) +} + +func TestSmartChatReportsServedModel(t *testing.T) { + const prompt = "hello" + decision, err := routePrompt(prompt, RoutingFree) + if err != nil { + t.Fatal(err) + } + requested := decision.Model + const served = "nvidia/llama-3.2-11b-vision" + if requested == served { + t.Fatalf("test needs a free primary other than %s for %q", served, prompt) + } + + tests := []struct { + name string + bodyModel string + headers map[string]string + want string + }{ + {name: "no reroute", bodyModel: requested, want: requested}, + { + name: "body model differs", + bodyModel: served, + headers: map[string]string{"X-Health-Reroute": "true", "X-Original-Model": requested, "X-Served-Model": served}, + want: served, + }, + { + name: "header only, health reroute", + headers: map[string]string{"X-Health-Reroute": "true", "X-Original-Model": requested, "X-Served-Model": served}, + want: served, + }, + { + name: "header only, cascade fallback", + headers: map[string]string{"X-Original-Model": requested, "X-Fallback-Model": served}, + want: served, + }, + {name: "no body model, no headers", want: requested}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := servedModelServer(t, tt.bodyModel, tt.headers) + defer server.Close() + client, err := NewLLMClient(routerTestKey, WithAPIURL(server.URL)) + if err != nil { + t.Fatalf("NewLLMClient: %v", err) + } + resp, err := client.SmartChat(context.Background(), prompt, &SmartChatOptions{RoutingProfile: RoutingFree}) + if err != nil { + t.Fatalf("SmartChat: %v", err) + } + if resp.Model != tt.want { + t.Errorf("Model = %q, want %q", resp.Model, tt.want) + } + if resp.RequestedModel != requested { + t.Errorf("RequestedModel = %q, want %q", resp.RequestedModel, requested) + } + if resp.Routing.Model != requested { + t.Errorf("Routing.Model = %q, want %q", resp.Routing.Model, requested) + } + if resp.Response != "OK" { + t.Errorf("Response = %q, want OK", resp.Response) + } + }) + } +} + +// The JSON shape is additive: existing keys keep their names. +func TestSmartChatResponseJSONKeys(t *testing.T) { + raw, err := json.Marshal(SmartChatResponse{Response: "r", Model: "served", RequestedModel: "req"}) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + for _, k := range []string{"response", "model", "requested_model", "routing"} { + if _, ok := got[k]; !ok { + t.Errorf("missing JSON key %q in %s", k, raw) + } + } + if got["model"] != "served" || got["requested_model"] != "req" { + t.Errorf("unexpected values: %s", raw) + } +}