From 3dc5619515ad6ada3fe44d13949555c3e6b65040 Mon Sep 17 00:00:00 2001 From: Vasil Date: Mon, 26 Oct 2026 02:11:58 +0100 Subject: [PATCH] fix(seed): default rules try the model made for the effort first --- CHANGELOG.md | 6 ++++++ internal/seed/seed.go | 24 ++++++++++++++++++++++++ internal/seed/seed_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24a6dd..6f2d5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ All notable changes to KTMS are listed here. The format follows ## [Unreleased] +### Fixed +- The default rules of the auto-seed and `ktms init` try the model made for the effort first: on + high, `gemini-2.5-pro` and `claude-sonnet-5-5` before `gpt-4o`, and `gpt-4o` before + `gpt-4o-mini`; on medium, `gpt-4o` before the low models. They tried the models in list order, + so high went to `gpt-4o-mini` first. A Keystone seeded already keeps its rules. + ## [1.3.3] - 2026-10-26 ### Added diff --git a/internal/seed/seed.go b/internal/seed/seed.go index 06e26ae..c477f55 100644 --- a/internal/seed/seed.go +++ b/internal/seed/seed.go @@ -407,6 +407,13 @@ func DefaultRules(models []*proxydb.Model, activeVendors []string) []*proxydb.Ru fallback = append(fallback, m.ID) } } + // The model made for this effort comes first: on high a model that + // serves only high is tried before one that also serves low. + byFit := func(ids []string) { + slices.SortStableFunc(ids, func(a, b string) int { return lowestEffort(models, b) - lowestEffort(models, a) }) + } + byFit(active) + byFit(fallback) if len(active) > 0 { return append(active, fallback...) } @@ -466,6 +473,23 @@ func DefaultRules(models []*proxydb.Model, activeVendors []string) []*proxydb.Ru } } +// lowestEffort ranks a model by the lowest effort it serves: 0 low, 1 medium, 2 high. +func lowestEffort(models []*proxydb.Model, id string) int { + rank := map[string]int{registry.EffortLow: 0, registry.EffortMedium: 1, registry.EffortHigh: 2} + lowest := 2 + for _, m := range models { + if m.ID != id { + continue + } + for _, e := range m.Efforts { + if r, ok := rank[e]; ok && r < lowest { + lowest = r + } + } + } + return lowest +} + func serves(m *proxydb.Model, effort string) bool { if registry.HasCapability(m, registry.CapabilityEmbedding) { return false diff --git a/internal/seed/seed_test.go b/internal/seed/seed_test.go index b363be2..6729368 100644 --- a/internal/seed/seed_test.go +++ b/internal/seed/seed_test.go @@ -1,6 +1,7 @@ package seed import ( + "slices" "testing" "github.com/ai-process/ktms/internal/registry" @@ -78,3 +79,27 @@ func TestDefaultRulesChainOrder(t *testing.T) { } } } + +func TestDefaultRulesPutTheModelMadeForTheEffortFirst(t *testing.T) { + chains := map[string][]string{} + for _, r := range DefaultRules(DefaultModels(), []string{registry.VendorOpenAI}) { + chains[r.Name] = r.Use + } + want := map[string][]string{ + "default-low": {"gpt-4o-mini"}, + "default-medium": {"gpt-4o", "gpt-4o-mini"}, + "default-high": {"gpt-4o", "gpt-4o-mini"}, + } + for name, w := range want { + if got := chains[name]; !slices.Equal(got[:len(w)], w) { + t.Errorf("%s starts with %v, want %v", name, got, w) + } + } + + // With every vendor, high tries the high-only models before the rest. + for _, r := range DefaultRules(DefaultModels(), []string{registry.VendorOpenAI, registry.VendorGoogle, registry.VendorAnthropic}) { + if r.Name == "default-high" && !slices.Equal(r.Use[:3], []string{"gemini-2.5-pro", "claude-sonnet-5-5", "gpt-4o"}) { + t.Errorf("default-high = %v", r.Use) + } + } +}