From f9e5b02d5efe52fa97dbcc0964aca192b9a6c2d3 Mon Sep 17 00:00:00 2001 From: Vasil Date: Thu, 29 Oct 2026 09:32:33 +0100 Subject: [PATCH 1/4] feat: pass temperature, top_p, stop and seed to the model GenerateTextRequest, chat completions and /v1/messages carry sampling to the serving model as far as its API takes it. A per-vendor table (registry.SamplingSupport) decides what each model gets; the adapter leaves out the rest and the response names it in ignored_parameters and X-KTMS-Ignored-Params. The no_sampling capability sends a model none. Sampling joins the response cache key. --- CHANGELOG.md | 7 ++ README.md | 38 ++++++++- ROADMAP.md | 1 - gen/ktms/v1/gateway.pb.go | 112 +++++++++++++++++++------ internal/cache/hash.go | 13 +++ internal/cache/hash_test.go | 21 +++++ internal/grpcapi/batch.go | 4 + internal/grpcapi/handler.go | 66 +++++++++------ internal/grpcapi/sampling.go | 44 ++++++++++ internal/grpcapi/sampling_test.go | 92 ++++++++++++++++++++ internal/llm/adapters/anthropic.go | 12 +++ internal/llm/adapters/bedrock.go | 20 ++++- internal/llm/adapters/google.go | 30 ++++++- internal/llm/adapters/open_ai.go | 18 ++++ internal/llm/adapters/sampling_test.go | 89 ++++++++++++++++++++ internal/llm/context.go | 14 +++- internal/llm/sampling.go | 78 +++++++++++++++++ internal/llm/sampling_test.go | 45 ++++++++++ internal/registry/build.go | 4 + internal/registry/sampling.go | 74 ++++++++++++++++ internal/registry/sampling_test.go | 46 ++++++++++ internal/server/anthropic.go | 7 ++ internal/server/openai.go | 32 +++++++ internal/server/openai_test.go | 38 +++++++++ plugin/skills/assess/SKILL.md | 3 +- plugin/skills/migrate/SKILL.md | 6 +- proto/ktms/v1/gateway.proto | 10 +++ 27 files changed, 861 insertions(+), 63 deletions(-) create mode 100644 internal/grpcapi/sampling.go create mode 100644 internal/grpcapi/sampling_test.go create mode 100644 internal/llm/adapters/sampling_test.go create mode 100644 internal/llm/sampling.go create mode 100644 internal/llm/sampling_test.go create mode 100644 internal/registry/sampling.go create mode 100644 internal/registry/sampling_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4d9b5..fc72e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,16 @@ All notable changes to KTMS are listed here. The format follows events, and returns errors in the Messages shape. `POST /v1/messages/count_tokens` returns an estimate. Claude Code's session-scoped `metadata.user_id` becomes its account or device as the user, with the session in `attributes["session_id"]`. +- Sampling: `temperature`, `top_p`, `stop` and `seed` on `GenerateTextRequest`, chat completions + and `/v1/messages` (`stop_sequences`) reach the serving model as far as its API takes them. A + field the model does not take is left out of its request, and the response names it in + `ignored_parameters` and the `X-KTMS-Ignored-Params` header. The new `no_sampling` capability + sends a model none of them. Sampling is part of the response cache key. ### Changed - Token budget reservations count tool definitions and the tool calls in the conversation. +- Chat completions and `/v1/messages` no longer ignore `temperature`, `top_p` and stop sequences, + and chat completions no longer ignore `seed`: a client that sends them changes the answer. ## [1.4.0] - 2026-10-27 diff --git a/README.md b/README.md index c3cd36e..6c896a0 100644 --- a/README.md +++ b/README.md @@ -696,6 +696,7 @@ Capabilities a model may declare: | `batch` | Can serve asynchronous batch text generation requests (`SubmitBatch`). | | `google_search` | Can serve requests with `enable_google_search`. | | `no_thinking` | The model reasons by default and bills the reasoning as output (DeepSeek V4, Gemma 4 on Ollama). The proxy turns it off: `thinking: {"type": "disabled"}` for DeepSeek, `reasoning_effort: "none"` for Ollama, and for `local` models when the serving node runs Ollama (other engines get no switch). OpenAI-compatible vendors only. | +| `no_sampling` | The model's API rejects sampling fields Keystone would otherwise send it (see [Sampling](#sampling)): it gets none of `temperature`, `top_p`, `stop` and `seed`, and responses report them as ignored. | | `prompt_cache` | Anthropic only: puts `cache_control` breakpoints after the tools, the system prompt and, in a multi-turn conversation, the latest turn, so repeated prefixes are read at 0.1x input. A new prefix is written at 1.25x, so set it on models whose instructions or conversations repeat. | | `vision` | Can process image parts in multimodal messages (`ImagePart`). | | `audio_input` | Can process audio parts in multimodal messages (`AudioPart`). | @@ -1122,6 +1123,7 @@ make proto-ts # TypeScript stubs in gen/ts/ | `tools`, `tool_choice` | Function/tool declarations with JSON Schema parameters. `tool_choice` specifies auto, required, none, or a specific named function. In non-streaming calls, arguments of `strict` tools are checked against their parameters, since not every vendor enforces strict; a call outside them is retried once and then falls back. Streamed tool calls reach the caller as they arrive and are not checked. | | `messages` | Chat history. Supports `MESSAGE_ROLE_USER`, `MESSAGE_ROLE_ASSISTANT`, and `MESSAGE_ROLE_TOOL` (with `tool_call_id`). User messages support multimodal `parts` (`ContentPart`: `text`, `image`, `audio`, `document`). Assistant messages can carry `tool_calls`. | | `max_output_tokens` | Ceiling, `0` = none. Output tokens are the expensive ones; set it. | +| `temperature`, `top_p`, `stop`, `seed` | Sampling, passed to the model that serves the call; unset leaves its default. `temperature` 0..2, `top_p` 0..1, up to 16 non-empty `stop` sequences; other values get `INVALID_ARGUMENT`. See [Sampling](#sampling). | | `enable_google_search` | Only Google models declaring `google_search` can serve it. | | `action_id` | Free-form label for usage attribution. | | `model_override` | Exact model id; bypasses rules but **not** throttles. | @@ -1131,14 +1133,39 @@ make proto-ts # TypeScript stubs in gen/ts/ The response carries `choices`, token `usage`, `resolved_model`, `resolved_vendor`, `matched_rule` (empty on override), structured `details` (`Choice` objects with text, tool calls, and finish reasons), -`cached` status, and routing metadata: `route_mode`, `placement`, `judge_score`, and `judge_model`. +`cached` status, routing metadata: `route_mode`, `placement`, `judge_score`, and `judge_model`, and +`ignored_parameters`: the sampling fields the resolved model did not get. + +### Sampling + +`temperature`, `top_p`, `stop` and `seed` reach the model that serves the call, as far as its API +takes them. A field the model does not take is left out of its request rather than failing the call, +and the response names it: `ignored_parameters` in gRPC (on the first chunk of a stream), the +`X-KTMS-Ignored-Params` header over HTTP. Fallback models are judged one by one, so the list +describes the model that answered. + +| Vendor | Takes | +| --- | --- | +| `openai`, `azure` | All four, up to 4 `stop` sequences. Reasoning models (`o*`, `gpt-5*` except `gpt-5-chat`) take none. | +| `groq` | All four, up to 4 `stop` sequences. | +| `google` | All four, up to 5 `stop` sequences. | +| `ollama`, `local` | All four. | +| `deepseek` | `temperature`, `top_p`, `stop` (up to 16). | +| `mistral` | `temperature` (capped at 1.5), `top_p`, `stop`. | +| `anthropic`, Claude on `bedrock` | `stop`. Claude 4.6 and earlier also take `temperature` (capped at 1) or `top_p`, and when both are set, only `temperature`. Claude 4.7 and later accept only the default sampling, so they get neither. | +| Other `bedrock` models | `temperature` (capped at 1), `top_p`, up to 4 `stop` sequences. | + +A temperature above a vendor's cap is lowered to it, not reported. When a vendor takes fewer stop +sequences than the call has, it gets the first ones and `stop` is reported. For a model whose API +rejects a field this table sends, declare the `no_sampling` capability. Sampling is part of the +response cache key. ### GenerateTextStream Server-streaming RPC `rpc GenerateTextStream(GenerateTextStreamRequest) returns (stream GenerateTextChunk)`. - Takes a `GenerateTextStreamRequest` containing the standard `GenerateTextRequest`. -- Streams incremental `GenerateTextChunk` items containing `delta` (token text), `tool_call_chunks`, `resolved_model`, `resolved_vendor`, `matched_rule`, `cached`, `route_mode`, `placement`, `judge_score`, `judge_model`, and final `finish_reason` with token `usage`. +- Streams incremental `GenerateTextChunk` items containing `delta` (token text), `tool_call_chunks`, `resolved_model`, `resolved_vendor`, `matched_rule`, `cached`, `route_mode`, `placement`, `judge_score`, `judge_model`, `ignored_parameters`, and final `finish_reason` with token `usage`. - **Fallback guarantee:** If upstream models fail before the first chunk is sent, the router falls back to the next model in the rule's chain. Once the first chunk is transmitted to the caller, the stream is locked to that model to guarantee integrity. - Usage events and daily token budgets are settled on stream completion. @@ -1308,6 +1335,7 @@ Authentication is `Authorization: Bearer ktms_…` (legacy `llm_…` keys are al - `X-KTMS-Placement`: infrastructure placement (`nodes_first`, `nodes_only`, `cloud_first`, `cloud_only`). - `X-KTMS-Route-Mode`: routing mechanism (`direct`, `override`, `auto-judge`, `auto-default`, `auto-effort`). - `X-KTMS-Judge-Score` and `X-KTMS-Judge-Model`: evaluation confidence and judge model ID when judge evaluates task complexity. +- `X-KTMS-Ignored-Params`: the sampling fields the serving model did not get, comma-separated (see [Sampling](#sampling)). It has no `X-LLM-Proxy-*` twin. > [!NOTE] > **Backwards Compatibility**: All legacy `X-LLM-Proxy-*` response headers are dual-emitted alongside `X-KTMS-*` with standard RFC 8594 `Deprecation: true`, `Sunset`, and `Warning` headers to provide seamless zero-downtime migration for existing clients. @@ -1367,7 +1395,8 @@ fields: `metadata` is passed through as attributes, `user` becomes `attributes[" | `tools`, `tool_choice` | Supported. Function declarations with JSON Schema parameters. `tool_choice` supports `"auto"`, `"none"`, `"required"`, or specific function names. | | `role: "tool"` | Supported. Tool responses with `tool_call_id`. | | Image/audio content parts | Rejected with `400`. | -| `temperature`, `top_p`, `stop`, `seed`, `logprobs`, penalties… | Accepted and ignored. | +| `temperature`, `top_p`, `stop`, `seed` | Passed to the model as far as it takes them; the rest is named in `X-KTMS-Ignored-Params` (see [Sampling](#sampling)). `stop` is a string or an array. | +| `logprobs`, penalties… | Accepted and ignored. | The response is a standard `chat.completion` with `finish_reason` (`"stop"`, `"tool_calls"`, `"length"`, or `"content_filter"`) and token `usage`. @@ -1443,7 +1472,8 @@ print(reply.content[0].text, reply.model) # `model` is the one that answered | `stream` | SSE events `message_start`, `ping`, `content_block_start`, `content_block_delta` (`text_delta`, `input_json_delta`), `content_block_stop`, `message_delta` and `message_stop`. `message_start` carries an input token estimate; `message_delta` carries the vendor's counts. | | `metadata.user_id` | `attributes["user_id"]`, as `user` on chat completions; other `metadata` keys are attributes, and `action_id` is the usage action label. Claude Code's `user_id` names a session: Keystone keeps its account (or, without one, its device) as the user and puts the session in `attributes["session_id"]`, so per-user limits and reports span sessions. | | `enable_cache`, `placement` | Extensions, as on chat completions; so are the `X-KTMS-Cache` and `X-KTMS-Placement` request headers (set them in Claude Code with `ANTHROPIC_CUSTOM_HEADERS`). | -| `temperature`, `top_p`, `top_k`, `stop_sequences`, `thinking`, `service_tier` | Accepted and ignored. | +| `temperature`, `top_p`, `stop_sequences` | Passed to the model as on chat completions (see [Sampling](#sampling)). A stop sequence ends the answer with `stop_reason` `end_turn`. | +| `top_k`, `thinking`, `service_tier` | Accepted and ignored. | The key goes in `x-api-key` or `Authorization: Bearer`. The response's `stop_reason` is `end_turn`, `tool_use`, `max_tokens` or `refusal`, and `usage` splits `cache_read_input_tokens` and diff --git a/ROADMAP.md b/ROADMAP.md index a580ea2..8e243cc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,7 +19,6 @@ Every 1.x release is additive: new RPCs, fields, endpoints and settings. Anythin ### Coding agents and API - OpenAI Responses API at `/v1/responses` for Codex CLI and the Agents SDK. -- `temperature`, `top_p`, `stop` and `seed` passed through to the chosen model. ### Routing and providers diff --git a/gen/ktms/v1/gateway.pb.go b/gen/ktms/v1/gateway.pb.go index 53924a0..b46edbe 100644 --- a/gen/ktms/v1/gateway.pb.go +++ b/gen/ktms/v1/gateway.pb.go @@ -1304,8 +1304,15 @@ type GenerateTextRequest struct { // choice is experimental; with it true the routing is stable. AutoRoute bool `protobuf:"varint,15,opt,name=auto_route,json=autoRoute,proto3" json:"auto_route,omitempty"` EffortSpecified bool `protobuf:"varint,16,opt,name=effort_specified,json=effortSpecified,proto3" json:"effort_specified,omitempty"` // true when effort was explicitly pinned by caller - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Sampling, passed to the model that serves the call. Unset leaves the + // model's default. A model that does not take one gets the call without it, + // and the response names it in ignored_parameters. + Temperature *float64 `protobuf:"fixed64,18,opt,name=temperature,proto3,oneof" json:"temperature,omitempty"` // 0..2; capped at the vendor's maximum + TopP *float64 `protobuf:"fixed64,19,opt,name=top_p,json=topP,proto3,oneof" json:"top_p,omitempty"` // 0..1 + Stop []string `protobuf:"bytes,20,rep,name=stop,proto3" json:"stop,omitempty"` // up to 16 sequences; vendors with a lower limit get the first ones + Seed *int64 `protobuf:"varint,21,opt,name=seed,proto3,oneof" json:"seed,omitempty"` // best-effort determinism + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GenerateTextRequest) Reset() { @@ -1450,6 +1457,34 @@ func (x *GenerateTextRequest) GetEffortSpecified() bool { return false } +func (x *GenerateTextRequest) GetTemperature() float64 { + if x != nil && x.Temperature != nil { + return *x.Temperature + } + return 0 +} + +func (x *GenerateTextRequest) GetTopP() float64 { + if x != nil && x.TopP != nil { + return *x.TopP + } + return 0 +} + +func (x *GenerateTextRequest) GetStop() []string { + if x != nil { + return x.Stop + } + return nil +} + +func (x *GenerateTextRequest) GetSeed() int64 { + if x != nil && x.Seed != nil { + return *x.Seed + } + return 0 +} + type TokensUsage struct { state protoimpl.MessageState `protogen:"open.v1"` Input int64 `protobuf:"varint,1,opt,name=input,proto3" json:"input,omitempty"` @@ -1600,8 +1635,10 @@ type GenerateTextResponse struct { Placement string `protobuf:"bytes,9,opt,name=placement,proto3" json:"placement,omitempty"` // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" JudgeScore float64 `protobuf:"fixed64,10,opt,name=judge_score,json=judgeScore,proto3" json:"judge_score,omitempty"` // Experimental: score or confidence from judge evaluation JudgeModel string `protobuf:"bytes,11,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // Experimental: judge model id if judge was invoked - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Sampling fields the resolved model did not take: "temperature", "top_p", "stop", "seed". + IgnoredParameters []string `protobuf:"bytes,12,rep,name=ignored_parameters,json=ignoredParameters,proto3" json:"ignored_parameters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GenerateTextResponse) Reset() { @@ -1711,22 +1748,30 @@ func (x *GenerateTextResponse) GetJudgeModel() string { return "" } +func (x *GenerateTextResponse) GetIgnoredParameters() []string { + if x != nil { + return x.IgnoredParameters + } + return nil +} + type GenerateTextChunk struct { - state protoimpl.MessageState `protogen:"open.v1"` - Delta string `protobuf:"bytes,1,opt,name=delta,proto3" json:"delta,omitempty"` - FinishReason string `protobuf:"bytes,2,opt,name=finish_reason,json=finishReason,proto3" json:"finish_reason,omitempty"` - Usage *TokensUsage `protobuf:"bytes,3,opt,name=usage,proto3" json:"usage,omitempty"` - ResolvedModel string `protobuf:"bytes,4,opt,name=resolved_model,json=resolvedModel,proto3" json:"resolved_model,omitempty"` - ResolvedVendor string `protobuf:"bytes,5,opt,name=resolved_vendor,json=resolvedVendor,proto3" json:"resolved_vendor,omitempty"` - MatchedRule string `protobuf:"bytes,6,opt,name=matched_rule,json=matchedRule,proto3" json:"matched_rule,omitempty"` - ToolCallChunks []*ToolCallChunk `protobuf:"bytes,7,rep,name=tool_call_chunks,json=toolCallChunks,proto3" json:"tool_call_chunks,omitempty"` - Cached bool `protobuf:"varint,8,opt,name=cached,proto3" json:"cached,omitempty"` // true when served from cache - RouteMode string `protobuf:"bytes,9,opt,name=route_mode,json=routeMode,proto3" json:"route_mode,omitempty"` // "direct" | "override" | "auto-judge" | "auto-default" | "auto-effort"; "auto-judge" and "auto-default" are experimental - Placement string `protobuf:"bytes,10,opt,name=placement,proto3" json:"placement,omitempty"` // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" - JudgeScore float64 `protobuf:"fixed64,11,opt,name=judge_score,json=judgeScore,proto3" json:"judge_score,omitempty"` // Experimental: score or confidence from judge evaluation - JudgeModel string `protobuf:"bytes,12,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // Experimental: judge model id if judge was invoked - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Delta string `protobuf:"bytes,1,opt,name=delta,proto3" json:"delta,omitempty"` + FinishReason string `protobuf:"bytes,2,opt,name=finish_reason,json=finishReason,proto3" json:"finish_reason,omitempty"` + Usage *TokensUsage `protobuf:"bytes,3,opt,name=usage,proto3" json:"usage,omitempty"` + ResolvedModel string `protobuf:"bytes,4,opt,name=resolved_model,json=resolvedModel,proto3" json:"resolved_model,omitempty"` + ResolvedVendor string `protobuf:"bytes,5,opt,name=resolved_vendor,json=resolvedVendor,proto3" json:"resolved_vendor,omitempty"` + MatchedRule string `protobuf:"bytes,6,opt,name=matched_rule,json=matchedRule,proto3" json:"matched_rule,omitempty"` + ToolCallChunks []*ToolCallChunk `protobuf:"bytes,7,rep,name=tool_call_chunks,json=toolCallChunks,proto3" json:"tool_call_chunks,omitempty"` + Cached bool `protobuf:"varint,8,opt,name=cached,proto3" json:"cached,omitempty"` // true when served from cache + RouteMode string `protobuf:"bytes,9,opt,name=route_mode,json=routeMode,proto3" json:"route_mode,omitempty"` // "direct" | "override" | "auto-judge" | "auto-default" | "auto-effort"; "auto-judge" and "auto-default" are experimental + Placement string `protobuf:"bytes,10,opt,name=placement,proto3" json:"placement,omitempty"` // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" + JudgeScore float64 `protobuf:"fixed64,11,opt,name=judge_score,json=judgeScore,proto3" json:"judge_score,omitempty"` // Experimental: score or confidence from judge evaluation + JudgeModel string `protobuf:"bytes,12,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // Experimental: judge model id if judge was invoked + IgnoredParameters []string `protobuf:"bytes,13,rep,name=ignored_parameters,json=ignoredParameters,proto3" json:"ignored_parameters,omitempty"` // on the first chunk, as in GenerateTextResponse + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GenerateTextChunk) Reset() { @@ -1843,6 +1888,13 @@ func (x *GenerateTextChunk) GetJudgeModel() string { return "" } +func (x *GenerateTextChunk) GetIgnoredParameters() []string { + if x != nil { + return x.IgnoredParameters + } + return nil +} + type ListModelsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -4251,7 +4303,7 @@ const file_ktms_v1_gateway_proto_rawDesc = "" + "\x0fPropertiesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12-\n" + "\x05value\x18\x02 \x01(\v2\x17.ktms.v1.SchemaPropertyR\x05value:\x028\x01B\x18\n" + - "\x16_additional_properties\"\xb0\x06\n" + + "\x16_additional_properties\"\xc1\a\n" + "\x13GenerateTextRequest\x12'\n" + "\x06effort\x18\x01 \x01(\x0e2\x0f.ktms.v1.EffortR\x06effort\x12L\n" + "\n" + @@ -4273,10 +4325,17 @@ const file_ktms_v1_gateway_proto_rawDesc = "" + "\tplacement\x18\x0e \x01(\tR\tplacement\x12\x1d\n" + "\n" + "auto_route\x18\x0f \x01(\bR\tautoRoute\x12)\n" + - "\x10effort_specified\x18\x10 \x01(\bR\x0feffortSpecified\x1a=\n" + + "\x10effort_specified\x18\x10 \x01(\bR\x0feffortSpecified\x12%\n" + + "\vtemperature\x18\x12 \x01(\x01H\x00R\vtemperature\x88\x01\x01\x12\x18\n" + + "\x05top_p\x18\x13 \x01(\x01H\x01R\x04topP\x88\x01\x01\x12\x12\n" + + "\x04stop\x18\x14 \x03(\tR\x04stop\x12\x17\n" + + "\x04seed\x18\x15 \x01(\x03H\x02R\x04seed\x88\x01\x01\x1a=\n" + "\x0fAttributesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"{\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x0e\n" + + "\f_temperatureB\b\n" + + "\x06_top_pB\a\n" + + "\x05_seed\"{\n" + "\vTokensUsage\x12\x14\n" + "\x05input\x18\x01 \x01(\x03R\x05input\x12\x16\n" + "\x06output\x18\x02 \x01(\x03R\x06output\x12\x1d\n" + @@ -4289,7 +4348,7 @@ const file_ktms_v1_gateway_proto_rawDesc = "" + "\x04text\x18\x02 \x01(\tR\x04text\x120\n" + "\n" + "tool_calls\x18\x03 \x03(\v2\x11.ktms.v1.ToolCallR\ttoolCalls\x12#\n" + - "\rfinish_reason\x18\x04 \x01(\tR\ffinishReason\"\x91\x03\n" + + "\rfinish_reason\x18\x04 \x01(\tR\ffinishReason\"\xc0\x03\n" + "\x14GenerateTextResponse\x12\x18\n" + "\achoices\x18\x01 \x03(\tR\achoices\x12*\n" + "\x05usage\x18\x02 \x01(\v2\x14.ktms.v1.TokensUsageR\x05usage\x12%\n" + @@ -4305,7 +4364,8 @@ const file_ktms_v1_gateway_proto_rawDesc = "" + " \x01(\x01R\n" + "judgeScore\x12\x1f\n" + "\vjudge_model\x18\v \x01(\tR\n" + - "judgeModel\"\xc6\x03\n" + + "judgeModel\x12-\n" + + "\x12ignored_parameters\x18\f \x03(\tR\x11ignoredParameters\"\xf5\x03\n" + "\x11GenerateTextChunk\x12\x14\n" + "\x05delta\x18\x01 \x01(\tR\x05delta\x12#\n" + "\rfinish_reason\x18\x02 \x01(\tR\ffinishReason\x12*\n" + @@ -4322,7 +4382,8 @@ const file_ktms_v1_gateway_proto_rawDesc = "" + "\vjudge_score\x18\v \x01(\x01R\n" + "judgeScore\x12\x1f\n" + "\vjudge_model\x18\f \x01(\tR\n" + - "judgeModel\"\x13\n" + + "judgeModel\x12-\n" + + "\x12ignored_parameters\x18\r \x03(\tR\x11ignoredParameters\"\x13\n" + "\x11ListModelsRequest\"\xd0\x03\n" + "\tModelInfo\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x16\n" + @@ -4810,6 +4871,7 @@ func file_ktms_v1_gateway_proto_init() { } file_ktms_v1_gateway_proto_msgTypes[12].OneofWrappers = []any{} file_ktms_v1_gateway_proto_msgTypes[13].OneofWrappers = []any{} + file_ktms_v1_gateway_proto_msgTypes[14].OneofWrappers = []any{} file_ktms_v1_gateway_proto_msgTypes[30].OneofWrappers = []any{ (*MediaPart_Text)(nil), (*MediaPart_Image)(nil), diff --git a/internal/cache/hash.go b/internal/cache/hash.go index 9f6ab1e..c6f16b5 100644 --- a/internal/cache/hash.go +++ b/internal/cache/hash.go @@ -42,6 +42,19 @@ func writeParams(h io.Writer, req *pb.GenerateTextRequest, withRequestInstructio } fmt.Fprintf(h, "max_tokens:%d\n", req.GetMaxOutputTokens()) fmt.Fprintf(h, "search:%t\n", req.GetEnableGoogleSearch()) + // Written only when set, so keys of requests without sampling stay valid. + if req.Temperature != nil { + fmt.Fprintf(h, "temperature:%g\n", req.GetTemperature()) + } + if req.TopP != nil { + fmt.Fprintf(h, "top_p:%g\n", req.GetTopP()) + } + for _, s := range req.GetStop() { + fmt.Fprintf(h, "stop:%q\n", s) + } + if req.Seed != nil { + fmt.Fprintf(h, "seed:%d\n", req.GetSeed()) + } // Sorted attributes if attrs := req.GetAttributes(); len(attrs) > 0 { diff --git a/internal/cache/hash_test.go b/internal/cache/hash_test.go index 97cb663..b001038 100644 --- a/internal/cache/hash_test.go +++ b/internal/cache/hash_test.go @@ -3,6 +3,8 @@ package cache import ( "testing" + "google.golang.org/protobuf/proto" + pb "github.com/ai-process/ktms/gen/ktms/v1" ) @@ -132,3 +134,22 @@ func TestComputeKey_SchemaConstraints(t *testing.T) { } } } + +func TestComputeKeySampling(t *testing.T) { + base := &pb.GenerateTextRequest{Messages: []*pb.ChatMessage{{Text: "hi"}}} + key := ComputeKey("k", 1, base) + seen := map[string]bool{key: true} + for _, r := range []*pb.GenerateTextRequest{ + {Messages: base.Messages, Temperature: proto.Float64(0)}, + {Messages: base.Messages, Temperature: proto.Float64(0.7)}, + {Messages: base.Messages, TopP: proto.Float64(0.5)}, + {Messages: base.Messages, Stop: []string{"END"}}, + {Messages: base.Messages, Seed: proto.Int64(1)}, + } { + k := ComputeKey("k", 1, r) + if seen[k] { + t.Fatalf("sampling %v shares a key", r) + } + seen[k] = true + } +} diff --git a/internal/grpcapi/batch.go b/internal/grpcapi/batch.go index 4500f2c..9889bd4 100644 --- a/internal/grpcapi/batch.go +++ b/internal/grpcapi/batch.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "strconv" + "strings" "time" "github.com/gofrs/uuid" @@ -76,6 +77,9 @@ func (s *ProxyServer) SubmitBatch(ctx context.Context, req *pb.SubmitBatchReques if itemReq == nil || (len(itemReq.GetMessages()) == 0 && itemReq.GetBaseSystemInstruction() == "" && itemReq.GetRequestInstruction() == "") { return nil, status.Errorf(codes.InvalidArgument, "bad_request: item %q has no messages or instructions", cid) } + if err := validateSampling(itemReq); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "bad_request: item %q: %s", cid, strings.TrimPrefix(status.Convert(err).Message(), "bad_request: ")) + } for _, msg := range itemReq.GetMessages() { if len(msg.GetParts()) > 0 { return nil, status.Error(codes.InvalidArgument, "bad_request: multimodal batch not supported yet") diff --git a/internal/grpcapi/handler.go b/internal/grpcapi/handler.go index 2bea4a4..84b4419 100644 --- a/internal/grpcapi/handler.go +++ b/internal/grpcapi/handler.go @@ -119,6 +119,9 @@ func (s *ProxyServer) GenerateText(ctx context.Context, req *pb.GenerateTextRequ if len(req.GetMessages()) == 0 && req.GetBaseSystemInstruction() == "" && req.GetRequestInstruction() == "" { return nil, status.Error(codes.InvalidArgument, "bad_request: no messages or instructions") } + if err := validateSampling(req); err != nil { + return nil, err + } if req.GetAttributes() != nil { req.Attributes = router.SanitizeAttributes(req.GetAttributes()) @@ -192,13 +195,14 @@ func (s *ProxyServer) GenerateText(ctx context.Context, req *pb.GenerateTextRequ CacheRead: int64(resp.Usage.CacheRead), CacheWrite: int64(resp.Usage.CacheWrite), }, - ResolvedModel: model.ID, - ResolvedVendor: model.Vendor, - MatchedRule: plan.ruleName, - RouteMode: plan.routeMode, - Placement: plan.placement, - JudgeScore: plan.judgeScore, - JudgeModel: plan.judgeModel, + ResolvedModel: model.ID, + ResolvedVendor: model.Vendor, + MatchedRule: plan.ruleName, + RouteMode: plan.routeMode, + Placement: plan.placement, + JudgeScore: plan.judgeScore, + JudgeModel: plan.judgeModel, + IgnoredParameters: ignoredParameters(model, chat.GetSampling()), } for i, choice := range resp.Choices { choiceText := choice.Text @@ -249,6 +253,9 @@ func (s *ProxyServer) GenerateTextStream(req *pb.GenerateTextStreamRequest, stre if len(inner.GetMessages()) == 0 && inner.GetBaseSystemInstruction() == "" && inner.GetRequestInstruction() == "" { return status.Error(codes.InvalidArgument, "bad_request: no messages or instructions") } + if err := validateSampling(inner); err != nil { + return err + } if inner.GetAttributes() != nil { inner.Attributes = router.SanitizeAttributes(inner.GetAttributes()) @@ -293,6 +300,7 @@ func (s *ProxyServer) GenerateTextStream(req *pb.GenerateTextStreamRequest, stre streamUsage *pb.TokensUsage streamModel string streamVendor string + streamIgnored []string ) var shadowEvaluator func(context.Context, *proxydb.Model, *proxydb.Model, *llm.Response, *llm.Response) if plan.shadowEval { @@ -355,6 +363,8 @@ func (s *ProxyServer) GenerateTextStream(req *pb.GenerateTextStreamRequest, stre outChunk.Placement = plan.placement outChunk.JudgeScore = plan.judgeScore outChunk.JudgeModel = plan.judgeModel + streamIgnored = ignoredParameters(m, chat.GetSampling()) + outChunk.IgnoredParameters = streamIgnored } if chunk.Usage != nil { outChunk.Usage = &pb.TokensUsage{ @@ -431,14 +441,15 @@ func (s *ProxyServer) GenerateTextStream(req *pb.GenerateTextStreamRequest, stre FinishReason: streamFinish, }, }, - Usage: streamUsage, - ResolvedModel: streamModel, - ResolvedVendor: streamVendor, - MatchedRule: plan.ruleName, - RouteMode: plan.routeMode, - Placement: plan.placement, - JudgeScore: plan.judgeScore, - JudgeModel: plan.judgeModel, + Usage: streamUsage, + ResolvedModel: streamModel, + ResolvedVendor: streamVendor, + MatchedRule: plan.ruleName, + RouteMode: plan.routeMode, + Placement: plan.placement, + JudgeScore: plan.judgeScore, + JudgeModel: plan.judgeModel, + IgnoredParameters: streamIgnored, } s.storeCache(stream.Context(), snap, id, inner, cachedResp) } @@ -450,11 +461,12 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached for _, choice := range cached.Details { if choice.Text != "" { chunk := &pb.GenerateTextChunk{ - Delta: choice.Text, - ResolvedModel: cached.ResolvedModel, - ResolvedVendor: cached.ResolvedVendor, - MatchedRule: cached.MatchedRule, - Cached: true, + Delta: choice.Text, + ResolvedModel: cached.ResolvedModel, + ResolvedVendor: cached.ResolvedVendor, + MatchedRule: cached.MatchedRule, + Cached: true, + IgnoredParameters: cached.IgnoredParameters, } sentFirst = true if err := stream.Send(chunk); err != nil { @@ -481,6 +493,7 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached chunk.ResolvedModel = cached.ResolvedModel chunk.ResolvedVendor = cached.ResolvedVendor chunk.MatchedRule = cached.MatchedRule + chunk.IgnoredParameters = cached.IgnoredParameters } if err := stream.Send(chunk); err != nil { return err @@ -500,6 +513,7 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached finalChunk.ResolvedModel = cached.ResolvedModel finalChunk.ResolvedVendor = cached.ResolvedVendor finalChunk.MatchedRule = cached.MatchedRule + finalChunk.IgnoredParameters = cached.IgnoredParameters } if err := stream.Send(finalChunk); err != nil { return err @@ -508,11 +522,12 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached if len(cached.Details) == 0 && len(cached.Choices) > 0 { for _, text := range cached.Choices { chunk := &pb.GenerateTextChunk{ - Delta: text, - ResolvedModel: cached.ResolvedModel, - ResolvedVendor: cached.ResolvedVendor, - MatchedRule: cached.MatchedRule, - Cached: true, + Delta: text, + ResolvedModel: cached.ResolvedModel, + ResolvedVendor: cached.ResolvedVendor, + MatchedRule: cached.MatchedRule, + Cached: true, + IgnoredParameters: cached.IgnoredParameters, } if err := stream.Send(chunk); err != nil { return err @@ -641,6 +656,7 @@ func buildChat(req *pb.GenerateTextRequest, id *apikeys.Identity, ruleName, effo if req.GetEnableGoogleSearch() { chat.SetEnableGoogleSearch() } + chat.SetSampling(samplingOf(req)) actionID := req.GetActionId() if actionID == "" { diff --git a/internal/grpcapi/sampling.go b/internal/grpcapi/sampling.go new file mode 100644 index 0000000..5c245c8 --- /dev/null +++ b/internal/grpcapi/sampling.go @@ -0,0 +1,44 @@ +package grpcapi + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "github.com/ai-process/ktms/gen/ktms/v1" + "github.com/ai-process/ktms/internal/llm" + "github.com/ai-process/ktms/internal/proxydb" + "github.com/ai-process/ktms/internal/registry" +) + +const maxStopSequences = 16 + +// validateSampling rejects values outside every vendor's range, so a bad +// value fails the call rather than being dropped by whichever model serves it. +func validateSampling(req *pb.GenerateTextRequest) error { + if req.Temperature != nil && !(req.GetTemperature() >= 0 && req.GetTemperature() <= 2) { + return status.Error(codes.InvalidArgument, "bad_request: temperature must be between 0 and 2") + } + if req.TopP != nil && !(req.GetTopP() >= 0 && req.GetTopP() <= 1) { + return status.Error(codes.InvalidArgument, "bad_request: top_p must be between 0 and 1") + } + if len(req.GetStop()) > maxStopSequences { + return status.Errorf(codes.InvalidArgument, "bad_request: at most %d stop sequences", maxStopSequences) + } + for _, s := range req.GetStop() { + if s == "" { + return status.Error(codes.InvalidArgument, "bad_request: stop sequences must not be empty") + } + } + return nil +} + +func samplingOf(req *pb.GenerateTextRequest) llm.Sampling { + return llm.Sampling{Temperature: req.Temperature, TopP: req.TopP, Stop: req.GetStop(), Seed: req.Seed} +} + +// ignoredParameters names the caller's sampling the model did not get, by +// the same table its adapter filters with. +func ignoredParameters(m *proxydb.Model, s llm.Sampling) []string { + _, ignored := registry.SamplingSupport(m).Apply(s) + return ignored +} diff --git a/internal/grpcapi/sampling_test.go b/internal/grpcapi/sampling_test.go new file mode 100644 index 0000000..f9f215d --- /dev/null +++ b/internal/grpcapi/sampling_test.go @@ -0,0 +1,92 @@ +package grpcapi + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + pb "github.com/ai-process/ktms/gen/ktms/v1" + "github.com/ai-process/ktms/internal/proxydb" + "github.com/ai-process/ktms/internal/registry" +) + +func samplingRequest() *pb.GenerateTextRequest { + return &pb.GenerateTextRequest{ + Effort: pb.Effort_EFFORT_LOW, + Messages: []*pb.ChatMessage{{Role: pb.MessageRole_MESSAGE_ROLE_USER, Text: "hi"}}, + Temperature: proto.Float64(0.2), + TopP: proto.Float64(0.9), + Stop: []string{"END"}, + Seed: proto.Int64(7), + } +} + +func TestSamplingReachesTheModel(t *testing.T) { + for _, c := range []struct { + model string + sent []string + ignored []string + }{ + {"gpt-4.1-mini", []string{"temperature", "top_p", "stop", "seed"}, nil}, + {"o4-mini", nil, []string{"temperature", "top_p", "stop", "seed"}}, + } { + vendor := &fakeVendor{reply: "ok"} + srv := httptest.NewServer(http.HandlerFunc(vendor.handler)) + models := []*proxydb.Model{testModel(c.model, registry.VendorOpenAI, srv.URL, registry.EffortLow)} + s := newTestServer(t, models, catchAllRules(c.model), nil) + + resp, err := s.GenerateText(testIdentity(), samplingRequest()) + srv.Close() + if err != nil { + t.Fatalf("%s: %v", c.model, err) + } + var body map[string]any + _ = json.Unmarshal(vendor.body(), &body) + var sent []string + for _, k := range []string{"temperature", "top_p", "stop", "seed"} { + if _, ok := body[k]; ok { + sent = append(sent, k) + } + } + if !slices.Equal(sent, c.sent) || !slices.Equal(resp.GetIgnoredParameters(), c.ignored) { + t.Errorf("%s: sent %v ignored %v", c.model, sent, resp.GetIgnoredParameters()) + } + } +} + +func TestSamplingIgnoredOnTheFirstChunk(t *testing.T) { + vendor := &fakeVendor{reply: "ok"} + srv := httptest.NewServer(http.HandlerFunc(vendor.handler)) + defer srv.Close() + s := newTestServer(t, []*proxydb.Model{testModel("o4-mini", registry.VendorOpenAI, srv.URL, registry.EffortLow)}, catchAllRules("o4-mini"), nil) + + stream := &mockStreamServer{ctx: testIdentity()} + if err := s.GenerateTextStream(&pb.GenerateTextStreamRequest{Request: samplingRequest()}, stream); err != nil { + t.Fatal(err) + } + if len(stream.chunks) == 0 || len(stream.chunks[0].GetIgnoredParameters()) != 4 { + t.Fatalf("first chunk: %v", stream.chunks) + } +} + +func TestSamplingValidation(t *testing.T) { + s := newTestServer(t, []*proxydb.Model{testModel("m1", registry.VendorOpenAI, "http://127.0.0.1:1", registry.EffortLow)}, catchAllRules("m1"), nil) + for name, mutate := range map[string]func(*pb.GenerateTextRequest){ + "temperature above 2": func(r *pb.GenerateTextRequest) { r.Temperature = proto.Float64(2.5) }, + "negative top_p": func(r *pb.GenerateTextRequest) { r.TopP = proto.Float64(-0.1) }, + "empty stop": func(r *pb.GenerateTextRequest) { r.Stop = []string{""} }, + "too many stops": func(r *pb.GenerateTextRequest) { r.Stop = make([]string, maxStopSequences+1) }, + } { + req := samplingRequest() + mutate(req) + if _, err := s.GenerateText(testIdentity(), req); status.Code(err) != codes.InvalidArgument { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/internal/llm/adapters/anthropic.go b/internal/llm/adapters/anthropic.go index 9b96eb1..a6a1973 100644 --- a/internal/llm/adapters/anthropic.go +++ b/internal/llm/adapters/anthropic.go @@ -34,6 +34,7 @@ type AnthropicAdapter struct { model string softSchema bool promptCache bool + sampling llm.SamplingSupport } func NewAnthropicAdapter(client *anthropic.Client, model string, throttle *llm.ThrottleControl) *AnthropicAdapter { @@ -52,6 +53,9 @@ func (a *AnthropicAdapter) SetPromptCache(on bool) { a.promptCache = on } // SetSoftSchema switches this adapter to prompt-described schemas. func (a *AnthropicAdapter) SetSoftSchema(soft bool) { a.softSchema = soft } +// SetSampling sets the sampling fields the model takes. +func (a *AnthropicAdapter) SetSampling(s llm.SamplingSupport) { a.sampling = s } + func (a *AnthropicAdapter) GenerateText(ctx context.Context, chat *llm.ChatContext) (*llm.Response, error) { ceiling := chat.GetMaxOutputTokens() // Built once, not per attempt: buildParams consumes the chat's per-request @@ -446,6 +450,14 @@ func (a *AnthropicAdapter) buildParams(chat *llm.ChatContext, ceiling int) anthr if system != "" && !systemConsumed { params.System = []anthropic.TextBlockParam{{Text: system}} } + s, _ := a.sampling.Apply(chat.GetSampling()) + if s.Temperature != nil { + params.Temperature = anthropic.Float(*s.Temperature) + } + if s.TopP != nil { + params.TopP = anthropic.Float(*s.TopP) + } + params.StopSequences = s.Stop if chat.GetResponseSchema() != nil && !a.softSchema { schema := llm.EnforceAllRequired(chat.GetResponseSchema()) diff --git a/internal/llm/adapters/bedrock.go b/internal/llm/adapters/bedrock.go index 7aea3e0..f1fa7ae 100644 --- a/internal/llm/adapters/bedrock.go +++ b/internal/llm/adapters/bedrock.go @@ -25,6 +25,7 @@ type BedrockAdapter struct { throttle *llm.ThrottleControl model string softSchema bool + sampling llm.SamplingSupport } func NewBedrockAdapter(client BedrockRuntimeClient, model string, throttle *llm.ThrottleControl) *BedrockAdapter { @@ -38,6 +39,9 @@ func NewBedrockAdapter(client BedrockRuntimeClient, model string, throttle *llm. // SetSoftSchema switches this adapter to prompt-described schemas. func (a *BedrockAdapter) SetSoftSchema(soft bool) { a.softSchema = soft } +// SetSampling sets the sampling fields the model takes. +func (a *BedrockAdapter) SetSampling(s llm.SamplingSupport) { a.sampling = s } + func (a *BedrockAdapter) GenerateText(ctx context.Context, chat *llm.ChatContext) (*llm.Response, error) { ceiling := chat.GetMaxOutputTokens() params := a.buildConverseInput(chat, ceiling) @@ -477,10 +481,20 @@ func (a *BedrockAdapter) buildCommonConfig(chat *llm.ChatContext, ceiling int) ( } } + inf := types.InferenceConfiguration{} if ceiling > 0 { - infConfig = &types.InferenceConfiguration{ - MaxTokens: aws.Int32(int32(ceiling)), - } + inf.MaxTokens = aws.Int32(int32(ceiling)) + } + s, _ := a.sampling.Apply(chat.GetSampling()) + if s.Temperature != nil { + inf.Temperature = aws.Float32(float32(*s.Temperature)) + } + if s.TopP != nil { + inf.TopP = aws.Float32(float32(*s.TopP)) + } + inf.StopSequences = s.Stop + if inf.MaxTokens != nil || inf.Temperature != nil || inf.TopP != nil || len(inf.StopSequences) > 0 { + infConfig = &inf } if chat.GetResponseSchema() != nil && !a.softSchema { diff --git a/internal/llm/adapters/google.go b/internal/llm/adapters/google.go index ef99091..29da232 100644 --- a/internal/llm/adapters/google.go +++ b/internal/llm/adapters/google.go @@ -33,6 +33,7 @@ type GoogleAdapter struct { ttsVoice string imageModel string softSchema bool + sampling llm.SamplingSupport } func NewGoogleAdapter(geminiClient *genai.Client, model string, throttle *llm.ThrottleControl) *GoogleAdapter { @@ -48,6 +49,9 @@ func (a *GoogleAdapter) SetSoftSchema(s bool) { a.softSchema = s } +// SetSampling sets the sampling fields the model takes. +func (a *GoogleAdapter) SetSampling(s llm.SamplingSupport) { a.sampling = s } + func (a *GoogleAdapter) GenerateText(ctx context.Context, chat *llm.ChatContext) (*llm.Response, error) { // Built once, not per attempt: buildSystemInstruction consumes the chat's // per-request instruction, so rebuilding would silently drop it on the retry. @@ -389,6 +393,18 @@ func (a *GoogleAdapter) buildGenerateConfig(chat *llm.ChatContext) *genai.Genera } config.MaxOutputTokens = int32(max) } + s, _ := a.sampling.Apply(chat.GetSampling()) + if s.Temperature != nil { + config.Temperature = genai.Ptr(float32(*s.Temperature)) + } + if s.TopP != nil { + config.TopP = genai.Ptr(float32(*s.TopP)) + } + config.StopSequences = s.Stop + if s.Seed != nil { + // Gemini's seed is 32-bit; a wider one wraps, still the same per value. + config.Seed = genai.Ptr(int32(*s.Seed)) + } if chat.GetResponseSchema() != nil { if genaiSchema := convertSchemaToGenai(llm.EnforceAllRequired(chat.GetResponseSchema())); genaiSchema != nil { @@ -766,7 +782,7 @@ func (a *GoogleAdapter) SubmitBatch(ctx context.Context, batchID string, items [ lineReq := map[string]any{ "contents": inlineRequests[i].Contents, } - if inlineRequests[i].Config != nil { + if c := inlineRequests[i].Config; c != nil { cfgMap := map[string]any{} if inlineRequests[i].Config.SystemInstruction != nil { lineReq["system_instruction"] = inlineRequests[i].Config.SystemInstruction @@ -774,6 +790,18 @@ func (a *GoogleAdapter) SubmitBatch(ctx context.Context, batchID string, items [ if inlineRequests[i].Config.MaxOutputTokens > 0 { cfgMap["max_output_tokens"] = inlineRequests[i].Config.MaxOutputTokens } + if c.Temperature != nil { + cfgMap["temperature"] = *c.Temperature + } + if c.TopP != nil { + cfgMap["top_p"] = *c.TopP + } + if len(c.StopSequences) > 0 { + cfgMap["stop_sequences"] = c.StopSequences + } + if c.Seed != nil { + cfgMap["seed"] = *c.Seed + } if inlineRequests[i].Config.ResponseMIMEType != "" { cfgMap["response_mime_type"] = inlineRequests[i].Config.ResponseMIMEType } diff --git a/internal/llm/adapters/open_ai.go b/internal/llm/adapters/open_ai.go index aabf9cd..eac9846 100644 --- a/internal/llm/adapters/open_ai.go +++ b/internal/llm/adapters/open_ai.go @@ -199,6 +199,7 @@ type OpenAIAdapter struct { // verifies the reply against the schema afterwards. softSchema bool thinkingOff map[string]any + sampling llm.SamplingSupport } func NewOpenAIAdapter(client *openai.Client, model string, throttle *llm.ThrottleControl) *OpenAIAdapter { @@ -227,6 +228,9 @@ func (a *OpenAIAdapter) SetHTTPClient(client *http.Client) { // SetSoftSchema switches this adapter to prompt-described schemas. func (a *OpenAIAdapter) SetSoftSchema(soft bool) { a.softSchema = soft } +// SetSampling sets the sampling fields the model takes. +func (a *OpenAIAdapter) SetSampling(s llm.SamplingSupport) { a.sampling = s } + // SetThinkingOff sets the request fields that turn the vendor's default // reasoning off. func (a *OpenAIAdapter) SetThinkingOff(fields map[string]any) { a.thinkingOff = fields } @@ -407,6 +411,20 @@ func (a *OpenAIAdapter) buildCompletionParams(chat *llm.ChatContext, messages [] params.MaxCompletionTokens = openai.Int(budget) } + s, _ := a.sampling.Apply(chat.GetSampling()) + if s.Temperature != nil { + params.Temperature = openai.Float(*s.Temperature) + } + if s.TopP != nil { + params.TopP = openai.Float(*s.TopP) + } + if len(s.Stop) > 0 { + params.Stop = openai.ChatCompletionNewParamsStopUnion{OfStringArray: s.Stop} + } + if s.Seed != nil { + params.Seed = openai.Int(*s.Seed) + } + if chat.GetResponseSchema() != nil && !a.softSchema { params.ResponseFormat = openai.ChatCompletionNewParamsResponseFormatUnion{ OfJSONSchema: &shared.ResponseFormatJSONSchemaParam{ diff --git a/internal/llm/adapters/sampling_test.go b/internal/llm/adapters/sampling_test.go new file mode 100644 index 0000000..e30137c --- /dev/null +++ b/internal/llm/adapters/sampling_test.go @@ -0,0 +1,89 @@ +package adapters + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/ai-process/ktms/internal/llm" +) + +func samplingChat() *llm.ChatContext { + t, p, seed := 0.3, 0.8, int64(42) + chat := llm.NewChatContext() + chat.AddUserMessage("go") + chat.SetSampling(llm.Sampling{Temperature: &t, TopP: &p, Stop: []string{"END", "STOP"}, Seed: &seed}) + return chat +} + +// wire is the request as the vendor reads it. +func wire(t *testing.T, v any) map[string]any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + return m +} + +func TestOpenAISendsSampling(t *testing.T) { + a := &OpenAIAdapter{sampling: llm.AllSampling} + got := wire(t, a.buildCompletionParams(samplingChat(), nil, 0)) + if got["temperature"] != 0.3 || got["top_p"] != 0.8 || got["seed"] != 42.0 { + t.Fatalf("sampling not sent: %v", got) + } + if stop, _ := json.Marshal(got["stop"]); string(stop) != `["END","STOP"]` { + t.Fatalf("stop: %s", stop) + } + + // A reasoning model's request carries none of them. + got = wire(t, (&OpenAIAdapter{}).buildCompletionParams(samplingChat(), nil, 0)) + for _, k := range []string{"temperature", "top_p", "stop", "seed"} { + if _, ok := got[k]; ok { + t.Errorf("%s sent to a model that does not take it", k) + } + } +} + +func TestAnthropicSendsSampling(t *testing.T) { + a := &AnthropicAdapter{model: "claude-sonnet-4-6", sampling: llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, OneOfTemperatureTopP: true}} + got := wire(t, a.buildParams(samplingChat(), 0)) + if got["temperature"] != 0.3 || got["top_p"] != nil { + t.Fatalf("want temperature alone: %v", got) + } + if stop, _ := json.Marshal(got["stop_sequences"]); string(stop) != `["END","STOP"]` { + t.Fatalf("stop_sequences: %s", stop) + } + got = wire(t, (&AnthropicAdapter{model: "claude-opus-5-5", sampling: llm.SamplingSupport{Stop: true}}).buildParams(samplingChat(), 0)) + if _, ok := got["temperature"]; ok { + t.Fatalf("temperature sent to a model that rejects it: %v", got) + } +} + +func TestGoogleSendsSampling(t *testing.T) { + c := (&GoogleAdapter{sampling: llm.AllSampling}).buildGenerateConfig(samplingChat()) + if c.Temperature == nil || *c.Temperature != 0.3 || c.TopP == nil || *c.TopP != 0.8 || + c.Seed == nil || *c.Seed != 42 || !slices.Equal(c.StopSequences, []string{"END", "STOP"}) { + t.Fatalf("sampling not set: %+v", c) + } + if c := (&GoogleAdapter{}).buildGenerateConfig(samplingChat()); c.Temperature != nil || c.StopSequences != nil { + t.Fatal("sampling set without support") + } +} + +func TestBedrockSendsSampling(t *testing.T) { + a := &BedrockAdapter{model: "amazon.nova-pro-v1:0", sampling: llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4}} + in := a.buildConverseInput(samplingChat(), 0) + inf := in.InferenceConfig + if inf == nil || inf.MaxTokens != nil || *inf.Temperature != 0.3 || *inf.TopP != 0.8 || strings.Join(inf.StopSequences, ",") != "END,STOP" { + t.Fatalf("inference config: %+v", inf) + } + if in := (&BedrockAdapter{}).buildConverseInput(samplingChat(), 0); in.InferenceConfig != nil { + t.Fatalf("inference config without ceiling or sampling: %+v", in.InferenceConfig) + } +} diff --git a/internal/llm/context.go b/internal/llm/context.go index f53e24c..e4b305d 100644 --- a/internal/llm/context.go +++ b/internal/llm/context.go @@ -87,7 +87,8 @@ type ChatContext struct { messages []*Message protectedIndices []int responseSchema *ResponseSchema - maxOutputTokens int // 0 = unset, no ceiling sent to the vendor + maxOutputTokens int // 0 = unset, no ceiling sent to the vendor + sampling Sampling enableGoogleSearch bool // per-request, never persisted; see SetEnableGoogleSearch BaseSystemInstruction string // Persistent system instructions (not in messages) perRequestSystemInstruction string // Ephemeral per-request instruction (not persisted) @@ -340,6 +341,17 @@ func (c *ChatContext) GetMaxOutputTokens() int { return c.maxOutputTokens } +// SetSampling sets the caller's sampling. Each adapter passes on what its +// model takes (SamplingSupport.Apply). +func (c *ChatContext) SetSampling(s Sampling) { + c.sampling = s +} + +// GetSampling returns the caller's sampling, before any model's filter. +func (c *ChatContext) GetSampling() Sampling { + return c.sampling +} + // SetEnableGoogleSearch asks for Google Search grounding on this request. // Advisory: adapters without a search tool ignore it, so callers wanting fresh // facts should route to a Google candidate explicitly. diff --git a/internal/llm/sampling.go b/internal/llm/sampling.go new file mode 100644 index 0000000..1271ea9 --- /dev/null +++ b/internal/llm/sampling.go @@ -0,0 +1,78 @@ +package llm + +import "slices" + +// Sampling parameter names, as callers send them and responses report them. +const ( + ParamTemperature = "temperature" + ParamTopP = "top_p" + ParamStop = "stop" + ParamSeed = "seed" +) + +// Sampling is the caller's sampling settings; a nil field or an empty Stop +// leaves the model's default. +type Sampling struct { + Temperature *float64 + TopP *float64 + Stop []string + Seed *int64 +} + +// SamplingSupport is what one model takes. The zero value takes nothing, so a +// model nobody described never gets a field its API may reject. +type SamplingSupport struct { + Temperature, TopP, Stop, Seed bool + MaxTemperature float64 // 0 means no cap below the caller's range + MaxStop int // 0 means no limit + // OneOfTemperatureTopP: the API rejects both together, so top_p goes. + OneOfTemperatureTopP bool +} + +// AllSampling is an OpenAI-style API that takes every field. +var AllSampling = SamplingSupport{Temperature: true, TopP: true, Stop: true, Seed: true} + +// Apply returns the settings the model gets and the names of those it does +// not. A temperature above the cap is lowered to it rather than dropped; a +// stop list over the limit keeps its first sequences and is reported. +func (s SamplingSupport) Apply(in Sampling) (Sampling, []string) { + var out Sampling + var ignored []string + if in.Temperature != nil { + if s.Temperature { + t := *in.Temperature + if s.MaxTemperature > 0 && t > s.MaxTemperature { + t = s.MaxTemperature + } + out.Temperature = &t + } else { + ignored = append(ignored, ParamTemperature) + } + } + if in.TopP != nil { + if s.TopP && !(s.OneOfTemperatureTopP && out.Temperature != nil) { + out.TopP = in.TopP + } else { + ignored = append(ignored, ParamTopP) + } + } + if len(in.Stop) > 0 { + switch { + case !s.Stop: + ignored = append(ignored, ParamStop) + case s.MaxStop > 0 && len(in.Stop) > s.MaxStop: + out.Stop = slices.Clone(in.Stop[:s.MaxStop]) + ignored = append(ignored, ParamStop) + default: + out.Stop = in.Stop + } + } + if in.Seed != nil { + if s.Seed { + out.Seed = in.Seed + } else { + ignored = append(ignored, ParamSeed) + } + } + return out, ignored +} diff --git a/internal/llm/sampling_test.go b/internal/llm/sampling_test.go new file mode 100644 index 0000000..5dea3fd --- /dev/null +++ b/internal/llm/sampling_test.go @@ -0,0 +1,45 @@ +package llm + +import ( + "slices" + "testing" +) + +func ptr[T any](v T) *T { return &v } + +func TestSamplingSupportApply(t *testing.T) { + all := Sampling{Temperature: ptr(1.4), TopP: ptr(0.9), Stop: []string{"a", "b", "c"}, Seed: ptr(int64(7))} + cases := []struct { + name string + support SamplingSupport + in Sampling + want Sampling + ignored []string + }{ + {"takes all", AllSampling, all, all, nil}, + {"takes nothing", SamplingSupport{}, all, Sampling{}, []string{ParamTemperature, ParamTopP, ParamStop, ParamSeed}}, + {"unset stays unset", SamplingSupport{}, Sampling{}, Sampling{}, nil}, + {"temperature capped, not dropped", SamplingSupport{Temperature: true, MaxTemperature: 1}, + Sampling{Temperature: ptr(1.4)}, Sampling{Temperature: ptr(1.0)}, nil}, + {"stop over the limit keeps the first and is reported", SamplingSupport{Stop: true, MaxStop: 2}, + Sampling{Stop: []string{"a", "b", "c"}}, Sampling{Stop: []string{"a", "b"}}, []string{ParamStop}}, + {"one of temperature and top_p keeps temperature", SamplingSupport{Temperature: true, TopP: true, OneOfTemperatureTopP: true}, + Sampling{Temperature: ptr(0.2), TopP: ptr(0.9)}, Sampling{Temperature: ptr(0.2)}, []string{ParamTopP}}, + {"one of, top_p alone passes", SamplingSupport{Temperature: true, TopP: true, OneOfTemperatureTopP: true}, + Sampling{TopP: ptr(0.9)}, Sampling{TopP: ptr(0.9)}, nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ignored := c.support.Apply(c.in) + if !sameSampling(got, c.want) || !slices.Equal(ignored, c.ignored) { + t.Fatalf("got %+v %v, want %+v %v", got, ignored, c.want, c.ignored) + } + }) + } +} + +func sameSampling(a, b Sampling) bool { + eq := func(x, y *float64) bool { return (x == nil) == (y == nil) && (x == nil || *x == *y) } + seed := (a.Seed == nil) == (b.Seed == nil) && (a.Seed == nil || *a.Seed == *b.Seed) + return eq(a.Temperature, b.Temperature) && eq(a.TopP, b.TopP) && slices.Equal(a.Stop, b.Stop) && seed +} diff --git a/internal/registry/build.go b/internal/registry/build.go index 992e19d..a3f446e 100644 --- a/internal/registry/build.go +++ b/internal/registry/build.go @@ -87,6 +87,7 @@ const ( // adapter. Pool adapters pass the node's engine; direct vendors pass "". func ConfigureOpenAI(a *adapters.OpenAIAdapter, m *proxydb.Model, engine string) { a.SetSoftSchema(SoftSchema(m)) + a.SetSampling(SamplingSupport(m)) if HasCapability(m, CapabilityNoThinking) { a.SetThinkingOff(ThinkingOff(m.Vendor, engine)) } @@ -166,6 +167,7 @@ func buildAdapter(m *proxydb.Model, key string) (llm.ClientAdapter, error) { } adapter := adapters.NewGoogleAdapter(client, m.ID, noThrottle) adapter.SetSoftSchema(SoftSchema(m)) + adapter.SetSampling(SamplingSupport(m)) if HasCapability(m, CapabilityTTS) { adapter.SetTTS(m.ID, googleTTSVoice) } @@ -189,6 +191,7 @@ func buildAdapter(m *proxydb.Model, key string) (llm.ClientAdapter, error) { client := anthropic.NewClient(opts...) adapter := adapters.NewAnthropicAdapter(&client, m.ID, noThrottle) adapter.SetSoftSchema(SoftSchema(m)) + adapter.SetSampling(SamplingSupport(m)) adapter.SetPromptCache(HasCapability(m, CapabilityPromptCache)) return adapter, nil case VendorBedrock: @@ -223,6 +226,7 @@ func buildAdapter(m *proxydb.Model, key string) (llm.ClientAdapter, error) { client := bedrockruntime.NewFromConfig(cfg, optFns...) adapter := adapters.NewBedrockAdapter(client, m.ID, noThrottle) adapter.SetSoftSchema(SoftSchema(m)) + adapter.SetSampling(SamplingSupport(m)) return adapter, nil default: return nil, fmt.Errorf("unknown vendor %q", m.Vendor) diff --git a/internal/registry/sampling.go b/internal/registry/sampling.go new file mode 100644 index 0000000..cbbd798 --- /dev/null +++ b/internal/registry/sampling.go @@ -0,0 +1,74 @@ +package registry + +import ( + "regexp" + "strings" + + "github.com/ai-process/ktms/internal/llm" + "github.com/ai-process/ktms/internal/proxydb" +) + +// CapabilityNoSampling marks a model whose API rejects sampling fields the +// table below would send it: the proxy then sends none of them. +const CapabilityNoSampling = "no_sampling" + +var ( + // OpenAI reasoning models reject temperature, top_p and stop; + // gpt-5-chat is the chat variant that takes them. + openAIReasoning = regexp.MustCompile(`^(o\d|gpt-5)`) + // Claude up to 4.6 takes temperature and top_p on the 2023-06-01 API; 4.7 + // and later accept only the defaults, so unknown ids get none. + claudeWithSampling = regexp.MustCompile(`claude-(3([-.@:]|$)|(opus|sonnet|haiku)-4(-[0-6])?(-\d{8}|-v\d|[.@:]|$))`) +) + +// SamplingSupport is the sampling a model's API takes. Adapters send only +// that, and responses name the rest in ignored_parameters. +func SamplingSupport(m *proxydb.Model) llm.SamplingSupport { + if HasCapability(m, CapabilityNoSampling) { + return llm.SamplingSupport{} + } + id := strings.ToLower(m.ID) + switch m.Vendor { + case VendorOpenAI, VendorAzure: + if openAIReasoning.MatchString(id) && !strings.Contains(id, "-chat") { + return llm.SamplingSupport{} + } + s := llm.AllSampling + s.MaxStop = 4 + return s + case VendorGroq: + s := llm.AllSampling + s.MaxStop = 4 + return s + case VendorDeepSeek: + return llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxStop: 16} + case VendorMistral: + // Mistral names its seed random_seed and rejects unknown fields. + return llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1.5} + case VendorOllama, VendorLocal: + return llm.AllSampling + case VendorGoogle: + s := llm.AllSampling + s.MaxStop = 5 + return s + case VendorAnthropic: + return claudeSampling(id) + case VendorBedrock: + if strings.Contains(id, "claude") { + return claudeSampling(id) + } + // Converse takes temperature 0..1 and no seed for every model family. + return llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4} + } + return llm.SamplingSupport{} +} + +func claudeSampling(id string) llm.SamplingSupport { + s := llm.SamplingSupport{Stop: true} + if claudeWithSampling.MatchString(id) { + // Claude reads temperature 0..1, and its 4.x models reject both + // fields together. + s.Temperature, s.TopP, s.MaxTemperature, s.OneOfTemperatureTopP = true, true, 1, true + } + return s +} diff --git a/internal/registry/sampling_test.go b/internal/registry/sampling_test.go new file mode 100644 index 0000000..222427f --- /dev/null +++ b/internal/registry/sampling_test.go @@ -0,0 +1,46 @@ +package registry + +import ( + "testing" + + "github.com/ai-process/ktms/internal/llm" + "github.com/ai-process/ktms/internal/proxydb" +) + +func TestSamplingSupport(t *testing.T) { + claudeOld := llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, OneOfTemperatureTopP: true} + claudeNew := llm.SamplingSupport{Stop: true} + openAI := llm.AllSampling + openAI.MaxStop = 4 + cases := []struct { + vendor, id string + caps []string + want llm.SamplingSupport + }{ + {VendorOpenAI, "gpt-4.1-mini", nil, openAI}, + {VendorOpenAI, "gpt-5-mini", nil, llm.SamplingSupport{}}, + {VendorOpenAI, "o4-mini", nil, llm.SamplingSupport{}}, + {VendorOpenAI, "gpt-5-chat-latest", nil, openAI}, + {VendorOpenAI, "gpt-4.1-mini", []string{CapabilityNoSampling}, llm.SamplingSupport{}}, + {VendorAnthropic, "claude-sonnet-4-6", nil, claudeOld}, + {VendorAnthropic, "claude-haiku-4-5-20251001", nil, claudeOld}, + {VendorAnthropic, "claude-sonnet-4-20250514", nil, claudeOld}, + {VendorAnthropic, "claude-3-5-haiku-latest", nil, claudeOld}, + {VendorAnthropic, "claude-opus-4-7", nil, claudeNew}, + {VendorAnthropic, "claude-opus-5-5", nil, claudeNew}, + {VendorAnthropic, "claude-sonnet-4-60", nil, claudeNew}, + {VendorBedrock, "us.anthropic.claude-sonnet-4-5-20250929-v1:0", nil, claudeOld}, + {VendorBedrock, "anthropic.claude-opus-4-8", nil, claudeNew}, + {VendorBedrock, "amazon.nova-pro-v1:0", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4}}, + {VendorMistral, "mistral-small-latest", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1.5}}, + {VendorDeepSeek, "deepseek-chat", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxStop: 16}}, + {VendorLocal, "qwen3:8b", nil, llm.AllSampling}, + {VendorTypeSafe, "typesafe", nil, llm.SamplingSupport{}}, + } + for _, c := range cases { + m := &proxydb.Model{ID: c.id, Vendor: c.vendor, Capabilities: c.caps} + if got := SamplingSupport(m); got != c.want { + t.Errorf("%s %s: got %+v, want %+v", c.vendor, c.id, got, c.want) + } + } +} diff --git a/internal/server/anthropic.go b/internal/server/anthropic.go index 08aa690..7122356 100644 --- a/internal/server/anthropic.go +++ b/internal/server/anthropic.go @@ -31,6 +31,10 @@ type messagesRequest struct { Tools []anthropicTool `json:"tools"` ToolChoice *anthropicToolChoice `json:"tool_choice"` Metadata map[string]string `json:"metadata"` + // Sampling, passed on as on chat completions; top_k is not. + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"top_p"` + StopSequences []string `json:"stop_sequences"` // Proxy extensions, as on chat completions. EnableCache *bool `json:"enable_cache,omitempty"` Placement string `json:"placement,omitempty"` @@ -118,6 +122,9 @@ func (req *messagesRequest) toProto(ctx context.Context, headerPlacement string, Placement: dec.Placement, AutoRoute: dec.IsAuto, EffortSpecified: dec.EffortSpecified, + Temperature: req.Temperature, + TopP: req.TopP, + Stop: req.StopSequences, } out.Attributes, out.ActionId = attributes(req.Metadata, "") if user, session := claudeCodeUser(out.Attributes["user_id"]); user != "" { diff --git a/internal/server/openai.go b/internal/server/openai.go index 7d94482..1ceb02d 100644 --- a/internal/server/openai.go +++ b/internal/server/openai.go @@ -124,6 +124,10 @@ type chatRequest struct { MaxCompletionTokens int32 `json:"max_completion_tokens"` ResponseFormat *responseFormat `json:"response_format"` ReasoningEffort string `json:"reasoning_effort"` + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"top_p"` + Stop json.RawMessage `json:"stop"` // a string or an array of them + Seed *int64 `json:"seed"` User string `json:"user"` Metadata map[string]string `json:"metadata"` N *int `json:"n"` @@ -610,6 +614,10 @@ func (req *chatRequest) toProto(ctx context.Context, headerPlacement string) (*p if out.MaxOutputTokens == 0 { out.MaxOutputTokens = req.MaxTokens } + out.Temperature, out.TopP, out.Seed = req.Temperature, req.TopP, req.Seed + if out.Stop, err = stopSequences(req.Stop); err != nil { + return nil, err + } if out.MaxOutputTokens < 0 { return nil, invalid("max_tokens must not be negative") } @@ -958,6 +966,7 @@ type routeInfo interface { GetRouteMode() string GetJudgeScore() float64 GetJudgeModel() string + GetIgnoredParameters() []string } func setRouteHeaders(w http.ResponseWriter, r routeInfo) { @@ -968,6 +977,29 @@ func setRouteHeaders(w http.ResponseWriter, r routeInfo) { setObservabilityHeader(w, "Judge-Score", fmt.Sprintf("%.2f", r.GetJudgeScore())) } setObservabilityHeader(w, "Judge-Model", r.GetJudgeModel()) + // New with no X-LLM-Proxy- twin, so not through setObservabilityHeader. + if ignored := r.GetIgnoredParameters(); len(ignored) > 0 { + w.Header().Set(headerIgnoredParams, strings.Join(ignored, ",")) + } +} + +// headerIgnoredParams lists the sampling fields the serving model did not take. +const headerIgnoredParams = "X-KTMS-Ignored-Params" + +// stopSequences reads OpenAI's stop: one string, an array of them, or null. +func stopSequences(raw json.RawMessage) ([]string, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + var one string + if err := json.Unmarshal(raw, &one); err == nil { + return []string{one}, nil + } + var many []string + if err := json.Unmarshal(raw, &many); err != nil { + return nil, invalid("stop must be a string or an array of strings") + } + return many, nil } func setCacheHeader(w http.ResponseWriter, enabled, hit bool) { diff --git a/internal/server/openai_test.go b/internal/server/openai_test.go index 8cb799d..81aa755 100644 --- a/internal/server/openai_test.go +++ b/internal/server/openai_test.go @@ -34,6 +34,7 @@ type fakeProxy struct { respRouteMode string respJudgeScore float64 respJudgeModel string + respIgnored []string streamChunks []*pb.GenerateTextChunk streamErr error // returned after streamChunks emptyStream bool @@ -60,6 +61,7 @@ func (f *fakeProxy) GenerateText(_ context.Context, r *pb.GenerateTextRequest) ( JudgeScore: f.respJudgeScore, JudgeModel: f.respJudgeModel, } + resp.IgnoredParameters = f.respIgnored if len(f.respDetails) > 0 { resp.Details = f.respDetails } @@ -1510,3 +1512,39 @@ func TestHeaderCompatibilityAndDeprecation(t *testing.T) { t.Errorf("expected placement nodes_only, got %s", fp.text.Placement) } } + +func TestSamplingOverHTTP(t *testing.T) { + fp, h, key := newTestAPI(t) + fp.respIgnored = []string{"seed"} + post := func(path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", path, strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+key) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec + } + + rec := post("/v1/chat/completions", `{"messages":[{"role":"user","content":"x"}],"temperature":0.2,"top_p":0.9,"stop":"END","seed":7}`) + if rec.Code != 200 || fp.text.GetTemperature() != 0.2 || fp.text.GetTopP() != 0.9 || fp.text.GetSeed() != 7 || + strings.Join(fp.text.GetStop(), ",") != "END" { + t.Fatalf("chat: %d %v", rec.Code, fp.text) + } + if got := rec.Header().Get("X-KTMS-Ignored-Params"); got != "seed" { + t.Fatalf("ignored header %q", got) + } + if rec := post("/v1/chat/completions", `{"messages":[{"role":"user","content":"x"}],"stop":["A","B"]}`); rec.Code != 200 || + strings.Join(fp.text.GetStop(), ",") != "A,B" || fp.text.Temperature != nil { + t.Fatalf("stop array: %d %v", rec.Code, fp.text) + } + if rec := post("/v1/chat/completions", `{"messages":[{"role":"user","content":"x"}],"stop":3}`); rec.Code != 400 { + t.Fatalf("stop as a number: %d", rec.Code) + } + + rec = post("/v1/messages", `{"max_tokens":8,"messages":[{"role":"user","content":"x"}],"temperature":0.5,"top_p":0.8,"stop_sequences":["Q:"]}`) + if rec.Code != 200 || fp.text.GetTemperature() != 0.5 || fp.text.GetTopP() != 0.8 || strings.Join(fp.text.GetStop(), ",") != "Q:" { + t.Fatalf("messages: %d %v", rec.Code, fp.text) + } + if got := rec.Header().Get("X-KTMS-Ignored-Params"); got != "seed" { + t.Fatalf("messages ignored header %q", got) + } +} diff --git a/plugin/skills/assess/SKILL.md b/plugin/skills/assess/SKILL.md index ae6f626..b85dc73 100644 --- a/plugin/skills/assess/SKILL.md +++ b/plugin/skills/assess/SKILL.md @@ -52,7 +52,8 @@ holds it. (`user`, `metadata.action_id`). Flag what does not map: an endpoint missing from [Endpoints](https://github.com/ai-process/ktms/blob/main/README.md#endpoints) (Responses, Assistants, Files, Realtime, transcription), `n` above 1, and code that relies on - `temperature`, `top_p`, `stop` or `seed`, which `/v1` ignores. + `temperature`, `top_p`, `stop` or `seed`: Keystone 1.4 and earlier ignore them, later versions + pass them on as far as the model takes them ([Sampling](https://github.com/ai-process/ktms/blob/main/README.md#sampling)). 4. **What it brings and costs here**: fallbacks across vendors, per-key and per-user limits and budgets, vendor keys out of the services, spend per call site, local nodes; against one more hop (about a millisecond, see [Resource footprint](https://github.com/ai-process/ktms/blob/main/README.md#resource-footprint)), diff --git a/plugin/skills/migrate/SKILL.md b/plugin/skills/migrate/SKILL.md index 0a0d4df..d0a50da 100644 --- a/plugin/skills/migrate/SKILL.md +++ b/plugin/skills/migrate/SKILL.md @@ -19,8 +19,10 @@ description: Plan and prepare a migration to Keystone (KTMS) from LiteLLM Proxy | Spend logs | Usage events in light-tokenmeter | Clients stop picking aliases: they send an effort (`auto`, `auto:low|medium|high`) and -attributes. On `/v1`, `temperature`, `top_p`, `stop`, `seed` and penalties are accepted and -ignored, and `n` must be 1: flag every client that depends on them. +attributes. On `/v1`, penalties are accepted and ignored and `n` must be 1; `temperature`, +`top_p`, `stop` and `seed` reach the model as far as it takes them (Keystone 1.4 and earlier +ignore them, see [Sampling](https://github.com/ai-process/ktms/blob/main/README.md#sampling)). +Flag every client that depends on them. ## Steps diff --git a/proto/ktms/v1/gateway.proto b/proto/ktms/v1/gateway.proto index 23d8b6a..6aa22df 100644 --- a/proto/ktms/v1/gateway.proto +++ b/proto/ktms/v1/gateway.proto @@ -164,6 +164,13 @@ message GenerateTextRequest { // choice is experimental; with it true the routing is stable. bool auto_route = 15; bool effort_specified = 16; // true when effort was explicitly pinned by caller + // Sampling, passed to the model that serves the call. Unset leaves the + // model's default. A model that does not take one gets the call without it, + // and the response names it in ignored_parameters. + optional double temperature = 18; // 0..2; capped at the vendor's maximum + optional double top_p = 19; // 0..1 + repeated string stop = 20; // up to 16 sequences; vendors with a lower limit get the first ones + optional int64 seed = 21; // best-effort determinism } message TokensUsage { @@ -193,6 +200,8 @@ message GenerateTextResponse { string placement = 9; // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" double judge_score = 10; // Experimental: score or confidence from judge evaluation string judge_model = 11; // Experimental: judge model id if judge was invoked + // Sampling fields the resolved model did not take: "temperature", "top_p", "stop", "seed". + repeated string ignored_parameters = 12; } message GenerateTextChunk { @@ -208,6 +217,7 @@ message GenerateTextChunk { string placement = 10; // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" double judge_score = 11; // Experimental: score or confidence from judge evaluation string judge_model = 12; // Experimental: judge model id if judge was invoked + repeated string ignored_parameters = 13; // on the first chunk, as in GenerateTextResponse } message ListModelsRequest {} From 6f67a7882ef001002752f9d24cb68da997f1ed5a Mon Sep 17 00:00:00 2001 From: Vasil Date: Thu, 29 Oct 2026 09:45:52 +0100 Subject: [PATCH 2/4] fix(sampling): review: model ids, blank Claude stops, a sampling capability Dotted Claude ids no longer pass for 4.6 and earlier, prefixed OpenAI reasoning ids are recognised, and Claude 3 keeps top_p beside temperature. Claude gets no whitespace-only stop sequence. The sampling capability turns the vendor's fields on for a model whose id misleads. HTTP drops empty stop sequences, as clients sent them while ignored. --- CHANGELOG.md | 9 ++++++--- README.md | 14 ++++++++----- internal/llm/sampling.go | 28 +++++++++++++++++--------- internal/llm/sampling_test.go | 4 ++++ internal/registry/sampling.go | 32 +++++++++++++++++++----------- internal/registry/sampling_test.go | 17 +++++++++++++--- internal/server/anthropic.go | 2 +- internal/server/openai.go | 13 +++++++++--- internal/server/openai_test.go | 3 +++ 9 files changed, 86 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc72e3f..ea98e47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,16 @@ All notable changes to KTMS are listed here. The format follows - Sampling: `temperature`, `top_p`, `stop` and `seed` on `GenerateTextRequest`, chat completions and `/v1/messages` (`stop_sequences`) reach the serving model as far as its API takes them. A field the model does not take is left out of its request, and the response names it in - `ignored_parameters` and the `X-KTMS-Ignored-Params` header. The new `no_sampling` capability - sends a model none of them. Sampling is part of the response cache key. + `ignored_parameters` and the `X-KTMS-Ignored-Params` header. The new `no_sampling` and + `sampling` capabilities correct a model whose id misleads the per-vendor table. Sampling is part + of the response cache key. ### Changed - Token budget reservations count tool definitions and the tool calls in the conversation. - Chat completions and `/v1/messages` no longer ignore `temperature`, `top_p` and stop sequences, - and chat completions no longer ignore `seed`: a client that sends them changes the answer. + and chat completions no longer ignore `seed`: a client that sends them changes the answer. A + `temperature` outside 0..2, a `top_p` outside 0..1 or more than 16 stop sequences now gets + `400`. ## [1.4.0] - 2026-10-27 diff --git a/README.md b/README.md index 6c896a0..ded061c 100644 --- a/README.md +++ b/README.md @@ -697,6 +697,7 @@ Capabilities a model may declare: | `google_search` | Can serve requests with `enable_google_search`. | | `no_thinking` | The model reasons by default and bills the reasoning as output (DeepSeek V4, Gemma 4 on Ollama). The proxy turns it off: `thinking: {"type": "disabled"}` for DeepSeek, `reasoning_effort: "none"` for Ollama, and for `local` models when the serving node runs Ollama (other engines get no switch). OpenAI-compatible vendors only. | | `no_sampling` | The model's API rejects sampling fields Keystone would otherwise send it (see [Sampling](#sampling)): it gets none of `temperature`, `top_p`, `stop` and `seed`, and responses report them as ignored. | +| `sampling` | The model takes its vendor's sampling fields although its id says otherwise (see [Sampling](#sampling)): an Azure deployment name, a Claude alias, a reasoning model run without reasoning. | | `prompt_cache` | Anthropic only: puts `cache_control` breakpoints after the tools, the system prompt and, in a multi-turn conversation, the latest turn, so repeated prefixes are read at 0.1x input. A new prefix is written at 1.25x, so set it on models whose instructions or conversations repeat. | | `vision` | Can process image parts in multimodal messages (`ImagePart`). | | `audio_input` | Can process audio parts in multimodal messages (`AudioPart`). | @@ -1146,19 +1147,22 @@ describes the model that answered. | Vendor | Takes | | --- | --- | -| `openai`, `azure` | All four, up to 4 `stop` sequences. Reasoning models (`o*`, `gpt-5*` except `gpt-5-chat`) take none. | +| `openai`, `azure` | All four, up to 4 `stop` sequences. Reasoning models (an id with `o` or `gpt-5`, as a whole or after a `/`, `-` or `_`, except `gpt-5-chat`) take none. | | `groq` | All four, up to 4 `stop` sequences. | | `google` | All four, up to 5 `stop` sequences. | | `ollama`, `local` | All four. | | `deepseek` | `temperature`, `top_p`, `stop` (up to 16). | | `mistral` | `temperature` (capped at 1.5), `top_p`, `stop`. | -| `anthropic`, Claude on `bedrock` | `stop`. Claude 4.6 and earlier also take `temperature` (capped at 1) or `top_p`, and when both are set, only `temperature`. Claude 4.7 and later accept only the default sampling, so they get neither. | +| `anthropic`, Claude on `bedrock` | `stop`, without sequences of whitespace alone. Claude 4.6 and earlier also take `temperature` (capped at 1) and `top_p`; from Claude 4 on, not both: with both set, only `temperature`. Claude 4.7 and later accept only the default sampling, so they get neither. | | Other `bedrock` models | `temperature` (capped at 1), `top_p`, up to 4 `stop` sequences. | A temperature above a vendor's cap is lowered to it, not reported. When a vendor takes fewer stop -sequences than the call has, it gets the first ones and `stop` is reported. For a model whose API -rejects a field this table sends, declare the `no_sampling` capability. Sampling is part of the -response cache key. +sequences than the call has, it gets the first ones and `stop` is reported. The model is told apart +by its id. When the id misleads, declare a capability: `no_sampling` for a model whose API rejects a +field this table sends, `sampling` for one that takes what its vendor's table row allows, such as +an Azure deployment of a chat model named like a reasoning one. Sampling is part of the response +cache key. Batch results do not carry `ignored_parameters`. Over HTTP, empty stop sequences are +dropped before the call. ### GenerateTextStream diff --git a/internal/llm/sampling.go b/internal/llm/sampling.go index 1271ea9..9a0b69e 100644 --- a/internal/llm/sampling.go +++ b/internal/llm/sampling.go @@ -1,6 +1,9 @@ package llm -import "slices" +import ( + "slices" + "strings" +) // Sampling parameter names, as callers send them and responses report them. const ( @@ -27,6 +30,8 @@ type SamplingSupport struct { MaxStop int // 0 means no limit // OneOfTemperatureTopP: the API rejects both together, so top_p goes. OneOfTemperatureTopP bool + // NoBlankStop: the API rejects a stop sequence of whitespace alone. + NoBlankStop bool } // AllSampling is an OpenAI-style API that takes every field. @@ -34,7 +39,8 @@ var AllSampling = SamplingSupport{Temperature: true, TopP: true, Stop: true, See // Apply returns the settings the model gets and the names of those it does // not. A temperature above the cap is lowered to it rather than dropped; a -// stop list over the limit keeps its first sequences and is reported. +// stop list cut to the limit, or of blank sequences the API rejects, is +// reported. func (s SamplingSupport) Apply(in Sampling) (Sampling, []string) { var out Sampling var ignored []string @@ -57,14 +63,18 @@ func (s SamplingSupport) Apply(in Sampling) (Sampling, []string) { } } if len(in.Stop) > 0 { - switch { - case !s.Stop: - ignored = append(ignored, ParamStop) - case s.MaxStop > 0 && len(in.Stop) > s.MaxStop: - out.Stop = slices.Clone(in.Stop[:s.MaxStop]) + stop := in.Stop + if s.NoBlankStop { + stop = slices.DeleteFunc(slices.Clone(stop), func(v string) bool { return strings.TrimSpace(v) == "" }) + } + if s.Stop && s.MaxStop > 0 && len(stop) > s.MaxStop { + stop = stop[:s.MaxStop] + } + if s.Stop && len(stop) > 0 { + out.Stop = stop + } + if !s.Stop || len(stop) < len(in.Stop) { ignored = append(ignored, ParamStop) - default: - out.Stop = in.Stop } } if in.Seed != nil { diff --git a/internal/llm/sampling_test.go b/internal/llm/sampling_test.go index 5dea3fd..5a8ec0a 100644 --- a/internal/llm/sampling_test.go +++ b/internal/llm/sampling_test.go @@ -25,6 +25,10 @@ func TestSamplingSupportApply(t *testing.T) { Sampling{Stop: []string{"a", "b", "c"}}, Sampling{Stop: []string{"a", "b"}}, []string{ParamStop}}, {"one of temperature and top_p keeps temperature", SamplingSupport{Temperature: true, TopP: true, OneOfTemperatureTopP: true}, Sampling{Temperature: ptr(0.2), TopP: ptr(0.9)}, Sampling{Temperature: ptr(0.2)}, []string{ParamTopP}}, + {"blank stop sequences dropped and reported", SamplingSupport{Stop: true, NoBlankStop: true}, + Sampling{Stop: []string{"\n", "END"}}, Sampling{Stop: []string{"END"}}, []string{ParamStop}}, + {"only blank stop sequences", SamplingSupport{Stop: true, NoBlankStop: true}, + Sampling{Stop: []string{" "}}, Sampling{}, []string{ParamStop}}, {"one of, top_p alone passes", SamplingSupport{Temperature: true, TopP: true, OneOfTemperatureTopP: true}, Sampling{TopP: ptr(0.9)}, Sampling{TopP: ptr(0.9)}, nil}, } diff --git a/internal/registry/sampling.go b/internal/registry/sampling.go index cbbd798..5d917e5 100644 --- a/internal/registry/sampling.go +++ b/internal/registry/sampling.go @@ -12,13 +12,19 @@ import ( // table below would send it: the proxy then sends none of them. const CapabilityNoSampling = "no_sampling" +// CapabilitySampling marks a model that takes its vendor's sampling fields +// although its id looks like one that does not: an Azure deployment name, a +// Claude alias, a reasoning model run without reasoning. +const CapabilitySampling = "sampling" + var ( // OpenAI reasoning models reject temperature, top_p and stop; - // gpt-5-chat is the chat variant that takes them. - openAIReasoning = regexp.MustCompile(`^(o\d|gpt-5)`) + // gpt-5-chat is the chat variant that takes them. Ids may carry a prefix. + openAIReasoning = regexp.MustCompile(`(^|[/_-])(o\d|gpt-5)`) // Claude up to 4.6 takes temperature and top_p on the 2023-06-01 API; 4.7 // and later accept only the defaults, so unknown ids get none. - claudeWithSampling = regexp.MustCompile(`claude-(3([-.@:]|$)|(opus|sonnet|haiku)-4(-[0-6])?(-\d{8}|-v\d|[.@:]|$))`) + claudeWithSampling = regexp.MustCompile(`claude-(3([-.@:]|$)|(opus|sonnet|haiku)-4([-.][0-6])?(-\d{8}|-v\d|-latest|[@:]|$))`) + claude3 = regexp.MustCompile(`claude-3([-.@:]|$)`) ) // SamplingSupport is the sampling a model's API takes. Adapters send only @@ -27,10 +33,11 @@ func SamplingSupport(m *proxydb.Model) llm.SamplingSupport { if HasCapability(m, CapabilityNoSampling) { return llm.SamplingSupport{} } + declared := HasCapability(m, CapabilitySampling) id := strings.ToLower(m.ID) switch m.Vendor { case VendorOpenAI, VendorAzure: - if openAIReasoning.MatchString(id) && !strings.Contains(id, "-chat") { + if !declared && openAIReasoning.MatchString(id) && !strings.Contains(id, "-chat") { return llm.SamplingSupport{} } s := llm.AllSampling @@ -52,10 +59,10 @@ func SamplingSupport(m *proxydb.Model) llm.SamplingSupport { s.MaxStop = 5 return s case VendorAnthropic: - return claudeSampling(id) + return claudeSampling(id, declared) case VendorBedrock: if strings.Contains(id, "claude") { - return claudeSampling(id) + return claudeSampling(id, declared) } // Converse takes temperature 0..1 and no seed for every model family. return llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4} @@ -63,12 +70,13 @@ func SamplingSupport(m *proxydb.Model) llm.SamplingSupport { return llm.SamplingSupport{} } -func claudeSampling(id string) llm.SamplingSupport { - s := llm.SamplingSupport{Stop: true} - if claudeWithSampling.MatchString(id) { - // Claude reads temperature 0..1, and its 4.x models reject both - // fields together. - s.Temperature, s.TopP, s.MaxTemperature, s.OneOfTemperatureTopP = true, true, 1, true +// claudeSampling: Claude rejects a stop sequence of whitespace alone, reads +// temperature 0..1, and from 4 on rejects temperature and top_p together. +func claudeSampling(id string, declared bool) llm.SamplingSupport { + s := llm.SamplingSupport{Stop: true, NoBlankStop: true} + if declared || claudeWithSampling.MatchString(id) { + s.Temperature, s.TopP, s.MaxTemperature = true, true, 1 + s.OneOfTemperatureTopP = !claude3.MatchString(id) } return s } diff --git a/internal/registry/sampling_test.go b/internal/registry/sampling_test.go index 222427f..0dc03bc 100644 --- a/internal/registry/sampling_test.go +++ b/internal/registry/sampling_test.go @@ -8,8 +8,10 @@ import ( ) func TestSamplingSupport(t *testing.T) { - claudeOld := llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, OneOfTemperatureTopP: true} - claudeNew := llm.SamplingSupport{Stop: true} + claudeOld := llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, OneOfTemperatureTopP: true, NoBlankStop: true} + claude3 := claudeOld + claude3.OneOfTemperatureTopP = false + claudeNew := llm.SamplingSupport{Stop: true, NoBlankStop: true} openAI := llm.AllSampling openAI.MaxStop = 4 cases := []struct { @@ -22,10 +24,19 @@ func TestSamplingSupport(t *testing.T) { {VendorOpenAI, "o4-mini", nil, llm.SamplingSupport{}}, {VendorOpenAI, "gpt-5-chat-latest", nil, openAI}, {VendorOpenAI, "gpt-4.1-mini", []string{CapabilityNoSampling}, llm.SamplingSupport{}}, + {VendorOpenAI, "gpt-5.1", []string{CapabilitySampling}, openAI}, + {VendorOpenAI, "openai/gpt-5", nil, llm.SamplingSupport{}}, + {VendorAzure, "prod-o3", nil, llm.SamplingSupport{}}, + {VendorAzure, "chat-4o", nil, openAI}, {VendorAnthropic, "claude-sonnet-4-6", nil, claudeOld}, {VendorAnthropic, "claude-haiku-4-5-20251001", nil, claudeOld}, {VendorAnthropic, "claude-sonnet-4-20250514", nil, claudeOld}, - {VendorAnthropic, "claude-3-5-haiku-latest", nil, claudeOld}, + {VendorAnthropic, "claude-3-5-haiku-latest", nil, claude3}, + {VendorAnthropic, "claude-3.5-sonnet", nil, claude3}, + {VendorAnthropic, "claude-sonnet-4-5-latest", nil, claudeOld}, + {VendorAnthropic, "claude-opus-4.7", nil, claudeNew}, + {VendorAnthropic, "claude-opus-4.1", nil, claudeOld}, + {VendorAnthropic, "my-claude-alias", []string{CapabilitySampling}, claudeOld}, {VendorAnthropic, "claude-opus-4-7", nil, claudeNew}, {VendorAnthropic, "claude-opus-5-5", nil, claudeNew}, {VendorAnthropic, "claude-sonnet-4-60", nil, claudeNew}, diff --git a/internal/server/anthropic.go b/internal/server/anthropic.go index 7122356..7243e47 100644 --- a/internal/server/anthropic.go +++ b/internal/server/anthropic.go @@ -124,7 +124,7 @@ func (req *messagesRequest) toProto(ctx context.Context, headerPlacement string, EffortSpecified: dec.EffortSpecified, Temperature: req.Temperature, TopP: req.TopP, - Stop: req.StopSequences, + Stop: nonEmpty(req.StopSequences), } out.Attributes, out.ActionId = attributes(req.Metadata, "") if user, session := claudeCodeUser(out.Attributes["user_id"]); user != "" { diff --git a/internal/server/openai.go b/internal/server/openai.go index 1ceb02d..e020a5e 100644 --- a/internal/server/openai.go +++ b/internal/server/openai.go @@ -14,6 +14,7 @@ import ( "math" "net/http" "regexp" + "slices" "strconv" "strings" "time" @@ -992,14 +993,20 @@ func stopSequences(raw json.RawMessage) ([]string, error) { return nil, nil } var one string - if err := json.Unmarshal(raw, &one); err == nil { - return []string{one}, nil + if json.Unmarshal(raw, &one) == nil { + return nonEmpty([]string{one}), nil } var many []string if err := json.Unmarshal(raw, &many); err != nil { return nil, invalid("stop must be a string or an array of strings") } - return many, nil + return nonEmpty(many), nil +} + +// nonEmpty drops empty stop sequences: they stop nothing, and HTTP clients +// sent them while the field was ignored. +func nonEmpty(stop []string) []string { + return slices.DeleteFunc(slices.Clone(stop), func(s string) bool { return s == "" }) } func setCacheHeader(w http.ResponseWriter, enabled, hit bool) { diff --git a/internal/server/openai_test.go b/internal/server/openai_test.go index 81aa755..7b02b5d 100644 --- a/internal/server/openai_test.go +++ b/internal/server/openai_test.go @@ -1536,6 +1536,9 @@ func TestSamplingOverHTTP(t *testing.T) { strings.Join(fp.text.GetStop(), ",") != "A,B" || fp.text.Temperature != nil { t.Fatalf("stop array: %d %v", rec.Code, fp.text) } + if rec := post("/v1/chat/completions", `{"messages":[{"role":"user","content":"x"}],"stop":""}`); rec.Code != 200 || len(fp.text.GetStop()) != 0 { + t.Fatalf("empty stop, ignored while the field was: %d %v", rec.Code, fp.text.GetStop()) + } if rec := post("/v1/chat/completions", `{"messages":[{"role":"user","content":"x"}],"stop":3}`); rec.Code != 400 { t.Fatalf("stop as a number: %d", rec.Code) } From 5b0e3b25fbeaaf46b120b64e4762618ca578a3de Mon Sep 17 00:00:00 2001 From: Vasil Date: Thu, 29 Oct 2026 14:30:12 +0100 Subject: [PATCH 3/4] fix(sampling): report a capped temperature and top_k A temperature lowered to the vendor's cap is named in ignored_parameters, and /v1/messages names top_k, which no model gets. The handler skips the vendor table when the call sets no sampling. --- CHANGELOG.md | 5 +++-- README.md | 9 +++++---- gen/ktms/v1/gateway.pb.go | 4 ++-- internal/grpcapi/sampling.go | 3 +++ internal/llm/sampling.go | 6 +++--- internal/llm/sampling_test.go | 6 ++++-- internal/server/anthropic.go | 15 ++++++++++++--- internal/server/openai.go | 8 ++++++++ internal/server/openai_test.go | 10 ++++++++-- proto/ktms/v1/gateway.proto | 4 ++-- 10 files changed, 50 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea98e47..09c8d22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ All notable changes to KTMS are listed here. The format follows user, with the session in `attributes["session_id"]`. - Sampling: `temperature`, `top_p`, `stop` and `seed` on `GenerateTextRequest`, chat completions and `/v1/messages` (`stop_sequences`) reach the serving model as far as its API takes them. A - field the model does not take is left out of its request, and the response names it in - `ignored_parameters` and the `X-KTMS-Ignored-Params` header. The new `no_sampling` and + field the model does not take is left out of its request, and the response names it, or a value + lowered to the vendor's cap, in `ignored_parameters` and the `X-KTMS-Ignored-Params` header; + `top_k` on `/v1/messages` is named there too. The new `no_sampling` and `sampling` capabilities correct a model whose id misleads the per-vendor table. Sampling is part of the response cache key. diff --git a/README.md b/README.md index ded061c..c2b3464 100644 --- a/README.md +++ b/README.md @@ -1141,7 +1141,7 @@ The response carries `choices`, token `usage`, `resolved_model`, `resolved_vendo `temperature`, `top_p`, `stop` and `seed` reach the model that serves the call, as far as its API takes them. A field the model does not take is left out of its request rather than failing the call, -and the response names it: `ignored_parameters` in gRPC (on the first chunk of a stream), the +and the response names it, as it names a value the model got changed: `ignored_parameters` in gRPC (on the first chunk of a stream), the `X-KTMS-Ignored-Params` header over HTTP. Fallback models are judged one by one, so the list describes the model that answered. @@ -1156,8 +1156,8 @@ describes the model that answered. | `anthropic`, Claude on `bedrock` | `stop`, without sequences of whitespace alone. Claude 4.6 and earlier also take `temperature` (capped at 1) and `top_p`; from Claude 4 on, not both: with both set, only `temperature`. Claude 4.7 and later accept only the default sampling, so they get neither. | | Other `bedrock` models | `temperature` (capped at 1), `top_p`, up to 4 `stop` sequences. | -A temperature above a vendor's cap is lowered to it, not reported. When a vendor takes fewer stop -sequences than the call has, it gets the first ones and `stop` is reported. The model is told apart +A temperature above a vendor's cap is lowered to it and reported as `temperature`. When a vendor +takes fewer stop sequences than the call has, it gets the first ones and `stop` is reported. The model is told apart by its id. When the id misleads, declare a capability: `no_sampling` for a model whose API rejects a field this table sends, `sampling` for one that takes what its vendor's table row allows, such as an Azure deployment of a chat model named like a reasoning one. Sampling is part of the response @@ -1477,7 +1477,8 @@ print(reply.content[0].text, reply.model) # `model` is the one that answered | `metadata.user_id` | `attributes["user_id"]`, as `user` on chat completions; other `metadata` keys are attributes, and `action_id` is the usage action label. Claude Code's `user_id` names a session: Keystone keeps its account (or, without one, its device) as the user and puts the session in `attributes["session_id"]`, so per-user limits and reports span sessions. | | `enable_cache`, `placement` | Extensions, as on chat completions; so are the `X-KTMS-Cache` and `X-KTMS-Placement` request headers (set them in Claude Code with `ANTHROPIC_CUSTOM_HEADERS`). | | `temperature`, `top_p`, `stop_sequences` | Passed to the model as on chat completions (see [Sampling](#sampling)). A stop sequence ends the answer with `stop_reason` `end_turn`. | -| `top_k`, `thinking`, `service_tier` | Accepted and ignored. | +| `top_k` | Accepted, sent to no model, and named in `X-KTMS-Ignored-Params`. | +| `thinking`, `service_tier` | Accepted and ignored. | The key goes in `x-api-key` or `Authorization: Bearer`. The response's `stop_reason` is `end_turn`, `tool_use`, `max_tokens` or `refusal`, and `usage` splits `cache_read_input_tokens` and diff --git a/gen/ktms/v1/gateway.pb.go b/gen/ktms/v1/gateway.pb.go index b46edbe..7177233 100644 --- a/gen/ktms/v1/gateway.pb.go +++ b/gen/ktms/v1/gateway.pb.go @@ -1307,7 +1307,7 @@ type GenerateTextRequest struct { // Sampling, passed to the model that serves the call. Unset leaves the // model's default. A model that does not take one gets the call without it, // and the response names it in ignored_parameters. - Temperature *float64 `protobuf:"fixed64,18,opt,name=temperature,proto3,oneof" json:"temperature,omitempty"` // 0..2; capped at the vendor's maximum + Temperature *float64 `protobuf:"fixed64,18,opt,name=temperature,proto3,oneof" json:"temperature,omitempty"` // 0..2; capped at the vendor's maximum, then reported TopP *float64 `protobuf:"fixed64,19,opt,name=top_p,json=topP,proto3,oneof" json:"top_p,omitempty"` // 0..1 Stop []string `protobuf:"bytes,20,rep,name=stop,proto3" json:"stop,omitempty"` // up to 16 sequences; vendors with a lower limit get the first ones Seed *int64 `protobuf:"varint,21,opt,name=seed,proto3,oneof" json:"seed,omitempty"` // best-effort determinism @@ -1635,7 +1635,7 @@ type GenerateTextResponse struct { Placement string `protobuf:"bytes,9,opt,name=placement,proto3" json:"placement,omitempty"` // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" JudgeScore float64 `protobuf:"fixed64,10,opt,name=judge_score,json=judgeScore,proto3" json:"judge_score,omitempty"` // Experimental: score or confidence from judge evaluation JudgeModel string `protobuf:"bytes,11,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // Experimental: judge model id if judge was invoked - // Sampling fields the resolved model did not take: "temperature", "top_p", "stop", "seed". + // Sampling fields the resolved model did not get as sent: "temperature", "top_p", "stop", "seed". IgnoredParameters []string `protobuf:"bytes,12,rep,name=ignored_parameters,json=ignoredParameters,proto3" json:"ignored_parameters,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/internal/grpcapi/sampling.go b/internal/grpcapi/sampling.go index 5c245c8..cbd35f1 100644 --- a/internal/grpcapi/sampling.go +++ b/internal/grpcapi/sampling.go @@ -39,6 +39,9 @@ func samplingOf(req *pb.GenerateTextRequest) llm.Sampling { // ignoredParameters names the caller's sampling the model did not get, by // the same table its adapter filters with. func ignoredParameters(m *proxydb.Model, s llm.Sampling) []string { + if s.Temperature == nil && s.TopP == nil && len(s.Stop) == 0 && s.Seed == nil { + return nil + } _, ignored := registry.SamplingSupport(m).Apply(s) return ignored } diff --git a/internal/llm/sampling.go b/internal/llm/sampling.go index 9a0b69e..afda2dc 100644 --- a/internal/llm/sampling.go +++ b/internal/llm/sampling.go @@ -38,9 +38,8 @@ type SamplingSupport struct { var AllSampling = SamplingSupport{Temperature: true, TopP: true, Stop: true, Seed: true} // Apply returns the settings the model gets and the names of those it does -// not. A temperature above the cap is lowered to it rather than dropped; a -// stop list cut to the limit, or of blank sequences the API rejects, is -// reported. +// not get as sent: a temperature lowered to the cap, a stop list cut to the +// limit or rid of blank sequences the API rejects, and every field it lacks. func (s SamplingSupport) Apply(in Sampling) (Sampling, []string) { var out Sampling var ignored []string @@ -49,6 +48,7 @@ func (s SamplingSupport) Apply(in Sampling) (Sampling, []string) { t := *in.Temperature if s.MaxTemperature > 0 && t > s.MaxTemperature { t = s.MaxTemperature + ignored = append(ignored, ParamTemperature) } out.Temperature = &t } else { diff --git a/internal/llm/sampling_test.go b/internal/llm/sampling_test.go index 5a8ec0a..01ce336 100644 --- a/internal/llm/sampling_test.go +++ b/internal/llm/sampling_test.go @@ -19,8 +19,10 @@ func TestSamplingSupportApply(t *testing.T) { {"takes all", AllSampling, all, all, nil}, {"takes nothing", SamplingSupport{}, all, Sampling{}, []string{ParamTemperature, ParamTopP, ParamStop, ParamSeed}}, {"unset stays unset", SamplingSupport{}, Sampling{}, Sampling{}, nil}, - {"temperature capped, not dropped", SamplingSupport{Temperature: true, MaxTemperature: 1}, - Sampling{Temperature: ptr(1.4)}, Sampling{Temperature: ptr(1.0)}, nil}, + {"temperature capped and reported", SamplingSupport{Temperature: true, MaxTemperature: 1}, + Sampling{Temperature: ptr(1.4)}, Sampling{Temperature: ptr(1.0)}, []string{ParamTemperature}}, + {"temperature within the cap", SamplingSupport{Temperature: true, MaxTemperature: 1}, + Sampling{Temperature: ptr(0.4)}, Sampling{Temperature: ptr(0.4)}, nil}, {"stop over the limit keeps the first and is reported", SamplingSupport{Stop: true, MaxStop: 2}, Sampling{Stop: []string{"a", "b", "c"}}, Sampling{Stop: []string{"a", "b"}}, []string{ParamStop}}, {"one of temperature and top_p keeps temperature", SamplingSupport{Temperature: true, TopP: true, OneOfTemperatureTopP: true}, diff --git a/internal/server/anthropic.go b/internal/server/anthropic.go index 7243e47..ceee0e6 100644 --- a/internal/server/anthropic.go +++ b/internal/server/anthropic.go @@ -31,10 +31,11 @@ type messagesRequest struct { Tools []anthropicTool `json:"tools"` ToolChoice *anthropicToolChoice `json:"tool_choice"` Metadata map[string]string `json:"metadata"` - // Sampling, passed on as on chat completions; top_k is not. + // Sampling, passed on as on chat completions; top_k is reported as ignored. Temperature *float64 `json:"temperature"` TopP *float64 `json:"top_p"` StopSequences []string `json:"stop_sequences"` + TopK *int `json:"top_k"` // Proxy extensions, as on chat completions. EnableCache *bool `json:"enable_cache,omitempty"` Placement string `json:"placement,omitempty"` @@ -414,13 +415,16 @@ func (a *OpenAIAPI) messages(ctx context.Context, w http.ResponseWriter, r *http id := "msg_" + randomID() if req.Stream { - return a.streamMessages(ctx, w, preq, id) + return a.streamMessages(ctx, w, preq, id, req.TopK != nil) } resp, err := a.proxy.GenerateText(ctx, preq) if err != nil { return err } setRouteHeaders(w, resp) + if req.TopK != nil { + addIgnoredParam(w, "top_k") + } setCacheHeader(w, preq.EnableCache, resp.Cached) var text, finish string @@ -499,7 +503,7 @@ func anthropicUsage(u *pb.TokensUsage, estimate int64) map[string]any { } } -func (a *OpenAIAPI) streamMessages(ctx context.Context, w http.ResponseWriter, preq *pb.GenerateTextRequest, id string) error { +func (a *OpenAIAPI) streamMessages(ctx context.Context, w http.ResponseWriter, preq *pb.GenerateTextRequest, id string, topK bool) error { flusher, ok := w.(http.Flusher) if !ok { return status.Error(codes.Internal, "streaming unsupported by response writer") @@ -507,6 +511,7 @@ func (a *OpenAIAPI) streamMessages(ctx context.Context, w http.ResponseWriter, p s := &messagesStream{ ctx: ctx, w: w, flusher: flusher, id: id, block: -1, estimate: grpcapi.EstimateTokens(preq), enableCache: preq.EnableCache, toolBlocks: map[int32]bool{}, + topK: topK, } err := a.proxy.GenerateTextStream(&pb.GenerateTextStreamRequest{Request: preq}, s) if err != nil && !s.started { @@ -541,6 +546,7 @@ type messagesStream struct { estimate int64 enableCache bool cacheHit bool + topK bool // the request set top_k, which no model gets started bool model string block int // index of the last block opened @@ -578,6 +584,9 @@ func (s *messagesStream) start(chunk *pb.GenerateTextChunk) error { setRouteHeaders(s.w, chunk) s.model = chunk.ResolvedModel } + if s.topK { + addIgnoredParam(s.w, "top_k") + } setCacheHeader(s.w, s.enableCache, s.cacheHit) startEventStream(s.w, s.flusher) s.started = true diff --git a/internal/server/openai.go b/internal/server/openai.go index e020a5e..2023553 100644 --- a/internal/server/openai.go +++ b/internal/server/openai.go @@ -987,6 +987,14 @@ func setRouteHeaders(w http.ResponseWriter, r routeInfo) { // headerIgnoredParams lists the sampling fields the serving model did not take. const headerIgnoredParams = "X-KTMS-Ignored-Params" +// addIgnoredParam adds a field the HTTP layer itself drops to the header. +func addIgnoredParam(w http.ResponseWriter, name string) { + if v := w.Header().Get(headerIgnoredParams); v != "" { + name = v + "," + name + } + w.Header().Set(headerIgnoredParams, name) +} + // stopSequences reads OpenAI's stop: one string, an array of them, or null. func stopSequences(raw json.RawMessage) ([]string, error) { if len(raw) == 0 || string(raw) == "null" { diff --git a/internal/server/openai_test.go b/internal/server/openai_test.go index 7b02b5d..2cb79fb 100644 --- a/internal/server/openai_test.go +++ b/internal/server/openai_test.go @@ -1543,11 +1543,17 @@ func TestSamplingOverHTTP(t *testing.T) { t.Fatalf("stop as a number: %d", rec.Code) } - rec = post("/v1/messages", `{"max_tokens":8,"messages":[{"role":"user","content":"x"}],"temperature":0.5,"top_p":0.8,"stop_sequences":["Q:"]}`) + rec = post("/v1/messages", `{"max_tokens":8,"messages":[{"role":"user","content":"x"}],"temperature":0.5,"top_p":0.8,"stop_sequences":["Q:"],"top_k":5}`) if rec.Code != 200 || fp.text.GetTemperature() != 0.5 || fp.text.GetTopP() != 0.8 || strings.Join(fp.text.GetStop(), ",") != "Q:" { t.Fatalf("messages: %d %v", rec.Code, fp.text) } - if got := rec.Header().Get("X-KTMS-Ignored-Params"); got != "seed" { + if got := rec.Header().Get("X-KTMS-Ignored-Params"); got != "seed,top_k" { t.Fatalf("messages ignored header %q", got) } + // No model gets top_k, so a stream reports it too. + fp.respIgnored = nil + rec = post("/v1/messages", `{"max_tokens":8,"stream":true,"messages":[{"role":"user","content":"x"}],"top_k":5}`) + if got := rec.Header().Get("X-KTMS-Ignored-Params"); rec.Code != 200 || got != "top_k" { + t.Fatalf("messages stream: %d ignored header %q", rec.Code, got) + } } diff --git a/proto/ktms/v1/gateway.proto b/proto/ktms/v1/gateway.proto index 6aa22df..72b9532 100644 --- a/proto/ktms/v1/gateway.proto +++ b/proto/ktms/v1/gateway.proto @@ -167,7 +167,7 @@ message GenerateTextRequest { // Sampling, passed to the model that serves the call. Unset leaves the // model's default. A model that does not take one gets the call without it, // and the response names it in ignored_parameters. - optional double temperature = 18; // 0..2; capped at the vendor's maximum + optional double temperature = 18; // 0..2; capped at the vendor's maximum, then reported optional double top_p = 19; // 0..1 repeated string stop = 20; // up to 16 sequences; vendors with a lower limit get the first ones optional int64 seed = 21; // best-effort determinism @@ -200,7 +200,7 @@ message GenerateTextResponse { string placement = 9; // "nodes_first" | "nodes_only" | "cloud_first" | "cloud_only" double judge_score = 10; // Experimental: score or confidence from judge evaluation string judge_model = 11; // Experimental: judge model id if judge was invoked - // Sampling fields the resolved model did not take: "temperature", "top_p", "stop", "seed". + // Sampling fields the resolved model did not get as sent: "temperature", "top_p", "stop", "seed". repeated string ignored_parameters = 12; } From df99fd0077961b1749dedc74d988f1af0b147fbc Mon Sep 17 00:00:00 2001 From: Vasil Date: Sat, 31 Oct 2026 14:42:37 +0100 Subject: [PATCH 4/4] fix(sampling): third review pass codex-mini and computer-use count as reasoning models. A Bedrock application inference profile gets stop alone, or with sampling declared what every family takes. A cached replay reports ignored_parameters on its first chunk only, as a live stream does. --- README.md | 3 ++- internal/grpcapi/handler.go | 30 +++++++++++++++++------------- internal/grpcapi/sampling_test.go | 19 +++++++++++++++++++ internal/llm/adapters/google.go | 16 ++++++++-------- internal/registry/sampling.go | 15 ++++++++++++--- internal/registry/sampling_test.go | 4 ++++ 6 files changed, 62 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c2b3464..cd6f36c 100644 --- a/README.md +++ b/README.md @@ -1147,7 +1147,7 @@ describes the model that answered. | Vendor | Takes | | --- | --- | -| `openai`, `azure` | All four, up to 4 `stop` sequences. Reasoning models (an id with `o` or `gpt-5`, as a whole or after a `/`, `-` or `_`, except `gpt-5-chat`) take none. | +| `openai`, `azure` | All four, up to 4 `stop` sequences. Reasoning models (an id starting with `o`, `gpt-5`, `codex-mini` or `computer-use`, as a whole or after a `/`, `-` or `_`, except `gpt-5-chat`) take none: they reject `temperature`, `top_p` and `stop`, and `seed` is not confirmed for them. | | `groq` | All four, up to 4 `stop` sequences. | | `google` | All four, up to 5 `stop` sequences. | | `ollama`, `local` | All four. | @@ -1155,6 +1155,7 @@ describes the model that answered. | `mistral` | `temperature` (capped at 1.5), `top_p`, `stop`. | | `anthropic`, Claude on `bedrock` | `stop`, without sequences of whitespace alone. Claude 4.6 and earlier also take `temperature` (capped at 1) and `top_p`; from Claude 4 on, not both: with both set, only `temperature`. Claude 4.7 and later accept only the default sampling, so they get neither. | | Other `bedrock` models | `temperature` (capped at 1), `top_p`, up to 4 `stop` sequences. | +| `bedrock` application inference profiles (ARNs) | Up to 4 `stop` sequences: the ARN does not say which model is behind it. With `sampling` declared, also `temperature` (capped at 1) or `top_p`, not both, which every family takes. | A temperature above a vendor's cap is lowered to it and reported as `temperature`. When a vendor takes fewer stop sequences than the call has, it gets the first ones and `stop` is reported. The model is told apart diff --git a/internal/grpcapi/handler.go b/internal/grpcapi/handler.go index 84b4419..60fe7a4 100644 --- a/internal/grpcapi/handler.go +++ b/internal/grpcapi/handler.go @@ -461,12 +461,14 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached for _, choice := range cached.Details { if choice.Text != "" { chunk := &pb.GenerateTextChunk{ - Delta: choice.Text, - ResolvedModel: cached.ResolvedModel, - ResolvedVendor: cached.ResolvedVendor, - MatchedRule: cached.MatchedRule, - Cached: true, - IgnoredParameters: cached.IgnoredParameters, + Delta: choice.Text, + ResolvedModel: cached.ResolvedModel, + ResolvedVendor: cached.ResolvedVendor, + MatchedRule: cached.MatchedRule, + Cached: true, + } + if !sentFirst { + chunk.IgnoredParameters = cached.IgnoredParameters } sentFirst = true if err := stream.Send(chunk); err != nil { @@ -520,14 +522,16 @@ func sendCachedStream(stream pb.GatewayService_GenerateTextStreamServer, cached } } if len(cached.Details) == 0 && len(cached.Choices) > 0 { - for _, text := range cached.Choices { + for i, text := range cached.Choices { chunk := &pb.GenerateTextChunk{ - Delta: text, - ResolvedModel: cached.ResolvedModel, - ResolvedVendor: cached.ResolvedVendor, - MatchedRule: cached.MatchedRule, - Cached: true, - IgnoredParameters: cached.IgnoredParameters, + Delta: text, + ResolvedModel: cached.ResolvedModel, + ResolvedVendor: cached.ResolvedVendor, + MatchedRule: cached.MatchedRule, + Cached: true, + } + if i == 0 { + chunk.IgnoredParameters = cached.IgnoredParameters } if err := stream.Send(chunk); err != nil { return err diff --git a/internal/grpcapi/sampling_test.go b/internal/grpcapi/sampling_test.go index f9f215d..c2d7e6e 100644 --- a/internal/grpcapi/sampling_test.go +++ b/internal/grpcapi/sampling_test.go @@ -90,3 +90,22 @@ func TestSamplingValidation(t *testing.T) { } } } + +func TestCachedReplayReportsIgnoredOnce(t *testing.T) { + for _, cached := range []*pb.GenerateTextResponse{ + {Details: []*pb.Choice{{Text: "a"}, {Text: "b"}}, IgnoredParameters: []string{"seed"}}, + {Choices: []string{"a", "b"}, IgnoredParameters: []string{"seed"}}, + } { + stream := &mockStreamServer{ctx: testIdentity()} + if err := sendCachedStream(stream, cached); err != nil { + t.Fatal(err) + } + var n int + for _, c := range stream.chunks { + n += len(c.GetIgnoredParameters()) + } + if n != 1 || len(stream.chunks[0].GetIgnoredParameters()) != 1 { + t.Errorf("ignored_parameters on %d chunks, want the first only", n) + } + } +} diff --git a/internal/llm/adapters/google.go b/internal/llm/adapters/google.go index 29da232..ef502c6 100644 --- a/internal/llm/adapters/google.go +++ b/internal/llm/adapters/google.go @@ -784,11 +784,11 @@ func (a *GoogleAdapter) SubmitBatch(ctx context.Context, batchID string, items [ } if c := inlineRequests[i].Config; c != nil { cfgMap := map[string]any{} - if inlineRequests[i].Config.SystemInstruction != nil { - lineReq["system_instruction"] = inlineRequests[i].Config.SystemInstruction + if c.SystemInstruction != nil { + lineReq["system_instruction"] = c.SystemInstruction } - if inlineRequests[i].Config.MaxOutputTokens > 0 { - cfgMap["max_output_tokens"] = inlineRequests[i].Config.MaxOutputTokens + if c.MaxOutputTokens > 0 { + cfgMap["max_output_tokens"] = c.MaxOutputTokens } if c.Temperature != nil { cfgMap["temperature"] = *c.Temperature @@ -802,11 +802,11 @@ func (a *GoogleAdapter) SubmitBatch(ctx context.Context, batchID string, items [ if c.Seed != nil { cfgMap["seed"] = *c.Seed } - if inlineRequests[i].Config.ResponseMIMEType != "" { - cfgMap["response_mime_type"] = inlineRequests[i].Config.ResponseMIMEType + if c.ResponseMIMEType != "" { + cfgMap["response_mime_type"] = c.ResponseMIMEType } - if inlineRequests[i].Config.ResponseSchema != nil { - cfgMap["response_schema"] = inlineRequests[i].Config.ResponseSchema + if c.ResponseSchema != nil { + cfgMap["response_schema"] = c.ResponseSchema } if len(cfgMap) > 0 { lineReq["generation_config"] = cfgMap diff --git a/internal/registry/sampling.go b/internal/registry/sampling.go index 5d917e5..222e36e 100644 --- a/internal/registry/sampling.go +++ b/internal/registry/sampling.go @@ -18,9 +18,9 @@ const CapabilityNoSampling = "no_sampling" const CapabilitySampling = "sampling" var ( - // OpenAI reasoning models reject temperature, top_p and stop; - // gpt-5-chat is the chat variant that takes them. Ids may carry a prefix. - openAIReasoning = regexp.MustCompile(`(^|[/_-])(o\d|gpt-5)`) + // OpenAI reasoning models reject temperature, top_p and stop; whether they + // take seed is unconfirmed, so it is dropped too. gpt-5-chat takes all four. + openAIReasoning = regexp.MustCompile(`(^|[/_-])(o\d|gpt-5|codex-mini|computer-use)`) // Claude up to 4.6 takes temperature and top_p on the 2023-06-01 API; 4.7 // and later accept only the defaults, so unknown ids get none. claudeWithSampling = regexp.MustCompile(`claude-(3([-.@:]|$)|(opus|sonnet|haiku)-4([-.][0-6])?(-\d{8}|-v\d|-latest|[@:]|$))`) @@ -64,6 +64,15 @@ func SamplingSupport(m *proxydb.Model) llm.SamplingSupport { if strings.Contains(id, "claude") { return claudeSampling(id, declared) } + // An application inference profile hides its model, Claude perhaps: + // stop alone, or with sampling declared what every family takes. + if strings.Contains(id, ":application-inference-profile/") { + s := llm.SamplingSupport{Stop: true, NoBlankStop: true, MaxStop: 4} + if declared { + s.Temperature, s.TopP, s.MaxTemperature, s.OneOfTemperatureTopP = true, true, 1, true + } + return s + } // Converse takes temperature 0..1 and no seed for every model family. return llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4} } diff --git a/internal/registry/sampling_test.go b/internal/registry/sampling_test.go index 0dc03bc..e27b4c5 100644 --- a/internal/registry/sampling_test.go +++ b/internal/registry/sampling_test.go @@ -42,6 +42,10 @@ func TestSamplingSupport(t *testing.T) { {VendorAnthropic, "claude-sonnet-4-60", nil, claudeNew}, {VendorBedrock, "us.anthropic.claude-sonnet-4-5-20250929-v1:0", nil, claudeOld}, {VendorBedrock, "anthropic.claude-opus-4-8", nil, claudeNew}, + {VendorOpenAI, "codex-mini-latest", nil, llm.SamplingSupport{}}, + {VendorBedrock, "arn:aws:bedrock:us-east-1:123:application-inference-profile/abc", nil, llm.SamplingSupport{Stop: true, NoBlankStop: true, MaxStop: 4}}, + {VendorBedrock, "arn:aws:bedrock:us-east-1:123:application-inference-profile/abc", []string{CapabilitySampling}, + llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4, OneOfTemperatureTopP: true, NoBlankStop: true}}, {VendorBedrock, "amazon.nova-pro-v1:0", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1, MaxStop: 4}}, {VendorMistral, "mistral-small-latest", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxTemperature: 1.5}}, {VendorDeepSeek, "deepseek-chat", nil, llm.SamplingSupport{Temperature: true, TopP: true, Stop: true, MaxStop: 16}},