diff --git a/CHANGELOG.md b/CHANGELOG.md index b825ec1..0fdbf6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ Notable changes to ktms-admin. The format follows ## [Unreleased] +### Added +- `KTMS_READ_KEY`: an optional Keystone key with the `admin:read` scope (Keystone 1.2+). Admin API + reads and every call of a viewer go with it; changes keep `KTMS_ADMIN_KEY`. The sidebar's + health shows whether each key is accepted for its job: the admin key may change Keystone, the + read key may read and nothing more. + ## [1.0.0] - 2026-10-23 The first stable release, paired with Keystone. The Model Switchboard. (KTMS) 1.0: ktms-admin diff --git a/README.md b/README.md index ec633e7..ba2b476 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ All settings are environment variables; see [`env.example`](env.example). | `SESSION_TTL` | `12h` | Session lifetime. | | `KTMS_ADDR` | empty | `host:port` of the KTMS gateway's gRPC endpoint. (`LLM_PROXY_ADDR` still works, deprecated.) | | `KTMS_ADMIN_KEY` | – | Key with the `admin` scope. Required with `KTMS_ADDR`. (`LLM_PROXY_ADMIN_KEY` still works, deprecated.) | +| `KTMS_READ_KEY` | empty | Optional key with the `admin:read` scope (Keystone 1.2+). Admin API reads and every call of a viewer go with it, so they cannot change Keystone; changes keep `KTMS_ADMIN_KEY`. The sidebar shows whether each key is accepted for its job. | | `KTMS_TLS` | `false` | Use TLS to reach the gateway. (`LLM_PROXY_TLS` still works, deprecated.) | | `TOKENMETER_URL` | empty | Base URL of light-tokenmeter's HTTP port. | | `TOKENMETER_TOKEN` | empty | Bearer token for light-tokenmeter with the `read` scope, when its authentication is on. A missing or refused token marks the usage dot "refused" and names the setting. | @@ -335,7 +336,8 @@ shows its spend this month against the budget and its keys. ## Security model - The console holds the gateway's admin key **on the server**; it never reaches the browser, and vendor keys are - write-only. + write-only. With `KTMS_READ_KEY` (scope `admin:read`) every read and everything a viewer does goes with a key + that cannot change Keystone, so a viewer stays read-only even if a route's role check were wrong. - Session cookies are signed (HMAC-SHA256), `HttpOnly`, `SameSite=Lax` and `Secure` over HTTPS. They carry only name, e-mail, sign-in method and expiry; the role is recomputed from the allowlists on every request, and turning a sign-in method off ends its sessions. diff --git a/env.example b/env.example index ad8ccca..c8984ee 100644 --- a/env.example +++ b/env.example @@ -44,6 +44,8 @@ SESSION_TTL=12h # KTMS gateway (gRPC). The key needs the admin scope. KTMS_ADDR=ktms:9090 KTMS_ADMIN_KEY= +# Optional, Keystone 1.2+: a key with the admin:read scope for reads and viewers. +KTMS_READ_KEY= KTMS_TLS=false # light-tokenmeter (HTTP). With its auth on, a token with the read scope. TOKENMETER_URL=http://light-tokenmeter:8080 diff --git a/internal/api/api.go b/internal/api/api.go index 4276cfc..e7a085e 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -39,6 +39,10 @@ type Server struct { data pb.GatewayServiceClient usage *usage.Client + keyMu sync.Mutex + keyStatus []keyStatus + keyAt time.Time + priceMu sync.Mutex prices map[string]usage.Price pricesAt time.Time @@ -55,8 +59,8 @@ func New(cfg *config.Config, a *auth.Handler) (*Server, func(), error) { creds = nil } opts := []grpc.DialOption{ - grpc.WithUnaryInterceptor(attachKey(cfg.ProxyAdminKey)), - grpc.WithStreamInterceptor(attachStreamKey(cfg.ProxyAdminKey)), + grpc.WithUnaryInterceptor(attachKey(s.keyFor)), + grpc.WithStreamInterceptor(attachStreamKey(s.keyFor)), } if creds == nil { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) @@ -81,20 +85,52 @@ func New(cfg *config.Config, a *auth.Handler) (*Server, func(), error) { return s, closer, nil } -func attachKey(key string) grpc.UnaryClientInterceptor { +func attachKey(key func(context.Context, string) string) grpc.UnaryClientInterceptor { return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, inv grpc.UnaryInvoker, opts ...grpc.CallOption) error { - ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+key) + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+key(ctx, method)) return inv(ctx, method, req, reply, cc, opts...) } } -func attachStreamKey(key string) grpc.StreamClientInterceptor { +func attachStreamKey(key func(context.Context, string) string) grpc.StreamClientInterceptor { return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { - ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+key) + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+key(ctx, method)) return streamer(ctx, desc, cc, method, opts...) } } +type viewerCtx struct{} +type keyCtx struct{} + +// keyFor picks the Keystone key for a call. With KTMS_READ_KEY set, Admin API +// reads and anything a viewer does go with it, so a read or a viewer can +// change nothing even if a route's guard were wrong. +func (s *Server) keyFor(ctx context.Context, method string) string { + if k, ok := ctx.Value(keyCtx{}).(string); ok { + return k + } + if s.cfg.ProxyReadKey == "" { + return s.cfg.ProxyAdminKey + } + if ctx.Value(viewerCtx{}) != nil || readMethod(method) { + return s.cfg.ProxyReadKey + } + return s.cfg.ProxyAdminKey +} + +// readMethods are the Admin API calls Keystone lets admin:read make. Listed, +// not guessed from names, so a new RPC goes with the admin key until added. +var readMethods = map[string]bool{ + pb.AdminService_GetConfig_FullMethodName: true, pb.AdminService_ListAPIKeys_FullMethodName: true, + pb.AdminService_ListNodeKeys_FullMethodName: true, pb.AdminService_ListNodes_FullMethodName: true, + pb.AdminService_ListUserRateLimits_FullMethodName: true, pb.AdminService_ListSpendLimits_FullMethodName: true, + pb.AdminService_ListOrganizations_FullMethodName: true, pb.AdminService_GetTeam_FullMethodName: true, + pb.AdminService_ListTeams_FullMethodName: true, pb.AdminService_GetTeamSpend_FullMethodName: true, + pb.AdminService_GetSpend_FullMethodName: true, "/ktms.v1.AdminService/ExplainRoute": true, +} + +func readMethod(method string) bool { return readMethods[method] } + func (s *Server) Handler() http.Handler { mux := http.NewServeMux() s.auth.Mount(mux) @@ -233,6 +269,9 @@ func (s *Server) guard(pol policy, h handler) http.HandlerFunc { return } } + if u.IsViewer() { + r = r.WithContext(context.WithValue(r.Context(), viewerCtx{}, true)) + } defer func() { if rec := recover(); rec != nil { log.Error().Interface("panic", rec).Str("path", r.URL.Path).Msg("handler panic") @@ -313,7 +352,7 @@ func backendErr(w http.ResponseWriter, err error) { } msg := st.Message() if st.Code() == codes.Unauthenticated || st.Code() == codes.PermissionDenied { - msg = "the KTMS gateway rejected the console's admin key: " + msg + msg = "the KTMS gateway rejected the console's key (KTMS_ADMIN_KEY, or KTMS_READ_KEY for reads when it is set): " + msg } if st.Code() == codes.Unavailable { msg = "the KTMS gateway is unreachable: " + msg @@ -340,29 +379,124 @@ type backendStatus struct { OK bool `json:"ok"` Error string `json:"error,omitempty"` // Auth marks a refused key or token, so the console does not call a running backend down. - Auth bool `json:"auth,omitempty"` + Auth bool `json:"auth,omitempty"` + Keys []keyStatus `json:"keys,omitempty"` +} + +// keyStatus says whether Keystone takes one of the console's keys for what +// the console uses it for. +type keyStatus struct { + Name string `json:"name"` // KTMS_ADMIN_KEY or KTMS_READ_KEY + OK bool `json:"ok"` + Refused bool `json:"refused,omitempty"` // Keystone answered; the key is wrong for its job + Error string `json:"error,omitempty"` +} + +// keyCheckTTL spaces the key probes: each one is a refused call in +// Keystone's log, and a key's scopes rarely change. +const keyCheckTTL = time.Minute + +// checkKeys probes each key without changing anything: MintAPIKey with no +// name and an unknown scope fails validation after the scope check, so its +// code tells whether the key may write. The probes run on their own context. +func (s *Server) checkKeys() []keyStatus { + s.keyMu.Lock() + defer s.keyMu.Unlock() + if s.keyStatus != nil && time.Since(s.keyAt) < keyCheckTTL { + return s.keyStatus + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + probe := func(name, key string, wantWrite bool) keyStatus { + st := keyStatus{Name: name, OK: true} + kctx := context.WithValue(ctx, keyCtx{}, key) + if _, err := s.admin.ListAPIKeys(kctx, &pb.ListAPIKeysRequest{}); err != nil { + return failed(st, err) + } + _, err := s.admin.MintAPIKey(kctx, &pb.MintAPIKeyRequest{Scopes: []string{"-"}}) + switch status.Code(err) { + case codes.InvalidArgument: + if !wantWrite { + st.OK, st.Refused, st.Error = false, true, "the key can change Keystone: mint one with only the admin:read scope" + } + case codes.PermissionDenied: + if wantWrite { + st.OK, st.Refused, st.Error = false, true, "the key cannot change Keystone: it needs the admin scope" + } + case codes.OK: + log.Error().Str("key", name).Msg("Keystone accepted an invalid MintAPIKey probe") + st.OK, st.Error = false, "Keystone accepted an invalid key request" + default: + return failed(st, err) + } + return st + } + var wg sync.WaitGroup + out := []keyStatus{{}} + if s.cfg.ProxyReadKey != "" { + out = append(out, keyStatus{}) + wg.Add(1) + go func() { defer wg.Done(); out[1] = probe("KTMS_READ_KEY", s.cfg.ProxyReadKey, false) }() + } + out[0] = probe("KTMS_ADMIN_KEY", s.cfg.ProxyAdminKey, true) + wg.Wait() + s.keyStatus, s.keyAt = out, time.Now() + return out +} + +func failed(st keyStatus, err error) keyStatus { + c := status.Code(err) + st.OK, st.Error = false, status.Convert(err).Message() + st.Refused = c == codes.Unauthenticated || c == codes.PermissionDenied + return st } func (s *Server) health(w http.ResponseWriter, r *http.Request, _ *auth.User) { ctx, cancel := context.WithTimeout(r.Context(), 4*time.Second) defer cancel() out := map[string]backendStatus{"proxy": {}, "usage": {}} + var wg sync.WaitGroup + var proxy, use backendStatus if s.admin != nil { - st := backendStatus{Enabled: true, OK: true} - if _, err := s.admin.ListAPIKeys(ctx, &pb.ListAPIKeysRequest{}); err != nil { - st.OK, st.Error = false, status.Convert(err).Message() - c := status.Code(err) - st.Auth = c == codes.Unauthenticated || c == codes.PermissionDenied - } - out["proxy"] = st + wg.Add(1) + go func() { + defer wg.Done() + proxy = backendStatus{Enabled: true, OK: true} + if _, err := s.admin.ListAPIKeys(ctx, &pb.ListAPIKeysRequest{}); err != nil { + k := failed(keyStatus{}, err) + proxy.OK, proxy.Error, proxy.Auth = false, k.Error, k.Refused + } + if !proxy.OK && !proxy.Auth { + return // down: the keys cannot be judged + } + proxy.Keys = s.checkKeys() + for _, k := range proxy.Keys { + if !k.OK { + proxy.OK, proxy.Auth = false, proxy.Auth || k.Refused + if proxy.Error == "" { + proxy.Error = k.Name + ": " + k.Error + } + } + } + }() } if s.usage != nil { - st := backendStatus{Enabled: true, OK: true} - if err := s.usage.Ready(ctx); err != nil { - var authErr *usage.AuthError - st.OK, st.Error, st.Auth = false, err.Error(), errors.As(err, &authErr) - } - out["usage"] = st + wg.Add(1) + go func() { + defer wg.Done() + use = backendStatus{Enabled: true, OK: true} + if err := s.usage.Ready(ctx); err != nil { + var authErr *usage.AuthError + use.OK, use.Error, use.Auth = false, err.Error(), errors.As(err, &authErr) + } + }() + } + wg.Wait() + if s.admin != nil { + out["proxy"] = proxy + } + if s.usage != nil { + out["usage"] = use } writeJSON(w, http.StatusOK, out) } diff --git a/internal/api/readkey_test.go b/internal/api/readkey_test.go new file mode 100644 index 0000000..cdb9754 --- /dev/null +++ b/internal/api/readkey_test.go @@ -0,0 +1,173 @@ +package api + +import ( + "context" + "encoding/json" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "github.com/ai-process/ktms-admin/internal/gen/ktms/v1" +) + +// scopedAdmin answers as Keystone would for the key a call carries: the admin +// key may do anything, the read key only read. +type scopedAdmin struct { + *fakeAdmin + key func(context.Context, string) string + admin string + read string + mu sync.Mutex // the health probes call concurrently + used map[string]string // method → key it carried + mints int + down bool +} + +func (f *scopedAdmin) check(ctx context.Context, method string, write bool) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.down { + return status.Error(codes.Unavailable, "connection refused") + } + k := f.key(ctx, method) + f.used[method] = k + switch { + case k == f.admin: + return nil + case k == f.read && !write: + return nil + case k == f.read: + return status.Error(codes.PermissionDenied, "apikeys: insufficient scope: need admin") + } + return status.Error(codes.Unauthenticated, "invalid key") +} + +func (f *scopedAdmin) ListAPIKeys(ctx context.Context, r *pb.ListAPIKeysRequest, o ...grpc.CallOption) (*pb.ListAPIKeysResponse, error) { + if err := f.check(ctx, "/ktms.v1.AdminService/ListAPIKeys", false); err != nil { + return nil, err + } + return f.fakeAdmin.ListAPIKeys(ctx, r, o...) +} + +func (f *scopedAdmin) MintAPIKey(ctx context.Context, r *pb.MintAPIKeyRequest, o ...grpc.CallOption) (*pb.MintAPIKeyResponse, error) { + f.mu.Lock() + f.mints++ + f.mu.Unlock() + if err := f.check(ctx, "/ktms.v1.AdminService/MintAPIKey", true); err != nil { + return nil, err + } + if r.Name == "" { + return nil, status.Error(codes.InvalidArgument, "name is required") + } + return f.fakeAdmin.MintAPIKey(ctx, r, o...) +} + +func scoped(t *testing.T, readKey string) (*Server, *scopedAdmin) { + t.Helper() + s, fa, _ := newServer(t, "") + s.cfg.ProxyAdminKey, s.cfg.ProxyReadKey = "ktms_admin", readKey + sa := &scopedAdmin{fakeAdmin: fa, key: s.keyFor, admin: "ktms_admin", read: "ktms_read", used: map[string]string{}} + s.admin = sa + return s, sa +} + +func TestReadKeyCarriesReadsAndViewers(t *testing.T) { + s, sa := scoped(t, "ktms_read") + h := s.Handler() + viewer, boss := sessionCookie(t, "p@view.com"), sessionCookie(t, "boss@x.com") + + if w := call(h, "GET", "/api/proxy/keys", "", boss, nil); w.Code != 200 || sa.used["/ktms.v1.AdminService/ListAPIKeys"] != "ktms_read" { + t.Fatalf("an admin's read went with %q (%d)", sa.used["/ktms.v1.AdminService/ListAPIKeys"], w.Code) + } + if w := call(h, "POST", "/api/proxy/keys", `{"name":"svc","scopes":["generate"]}`, boss, csrf); w.Code != 200 || sa.used["/ktms.v1.AdminService/MintAPIKey"] != "ktms_admin" { + t.Fatalf("an admin's change went with %q (%d %s)", sa.used["/ktms.v1.AdminService/MintAPIKey"], w.Code, w.Body) + } + if w := call(h, "GET", "/api/proxy/keys", "", viewer, nil); w.Code != 200 || sa.used["/ktms.v1.AdminService/ListAPIKeys"] != "ktms_read" { + t.Fatalf("a viewer's read went with %q", sa.used["/ktms.v1.AdminService/ListAPIKeys"]) + } + + // Whatever a viewer's call is, it carries the read key. + ctx := context.WithValue(context.Background(), viewerCtx{}, true) + if k := s.keyFor(ctx, "/ktms.v1.AdminService/RevokeAPIKey"); k != "ktms_read" { + t.Fatalf("a viewer's change would go with %q", k) + } + for m, want := range map[string]string{ + "/ktms.v1.AdminService/GetConfig": "ktms_read", + "/ktms.v1.AdminService/ExplainRoute": "ktms_read", + "/ktms.v1.AdminService/ReplaceRules": "ktms_admin", + "/ktms.v1.GatewayService/GenerateText": "ktms_admin", + } { + if k := s.keyFor(context.Background(), m); k != want { + t.Errorf("%s goes with %q, want %q", m, k, want) + } + } + + // The interceptor puts the chosen key on the wire. + var got string + _ = attachKey(s.keyFor)(context.Background(), "/ktms.v1.AdminService/ListNodes", nil, nil, nil, + func(ctx context.Context, _ string, _, _ any, _ *grpc.ClientConn, _ ...grpc.CallOption) error { + md, _ := metadata.FromOutgoingContext(ctx) + got = md.Get("authorization")[0] + return nil + }) + if got != "Bearer ktms_read" { + t.Fatalf("authorization = %q", got) + } +} + +func TestWithoutReadKeyEverythingUsesTheAdminKey(t *testing.T) { + s, _ := scoped(t, "") + ctx := context.WithValue(context.Background(), viewerCtx{}, true) + for _, m := range []string{"/ktms.v1.AdminService/ListAPIKeys", "/ktms.v1.AdminService/MintAPIKey"} { + if k := s.keyFor(ctx, m); k != "ktms_admin" { + t.Errorf("%s goes with %q", m, k) + } + } +} + +func TestHealthChecksBothKeys(t *testing.T) { + health := func(s *Server) backendStatus { + t.Helper() + w := call(s.Handler(), "GET", "/api/health", "", sessionCookie(t, "boss@x.com"), nil) + var out map[string]backendStatus + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + return out["proxy"] + } + s, _ := scoped(t, "ktms_read") + if st := health(s); !st.OK || len(st.Keys) != 2 || !st.Keys[0].OK || !st.Keys[1].OK { + t.Fatalf("both keys right: %+v", st) + } + + // The read key in KTMS_ADMIN_KEY: reads work, the console cannot change anything. + s, sa := scoped(t, "") + sa.admin = "nobody" + s.cfg.ProxyAdminKey = "ktms_read" + if st := health(s); st.OK || !st.Auth || st.Keys[0].OK { + t.Fatalf("a read key as the admin key: %+v", st) + } + + // An admin key in KTMS_READ_KEY is reported: it is not read-only. + s, sa = scoped(t, "ktms_admin") + if st := health(s); st.OK || st.Keys[1].OK || !st.Keys[1].Refused { + t.Fatalf("an admin key as the read key: %+v", st) + } + // The probes are cached: Keystone sees them once a minute, not per page. + before := sa.mints + health(s) + if sa.mints != before { + t.Fatalf("keys probed again within %s", keyCheckTTL) + } + + // A gateway that is down is down, not a refused key. + s, sa = scoped(t, "ktms_read") + sa.down = true + if st := health(s); st.OK || st.Auth || st.Error == "" { + t.Fatalf("a gateway that is down: %+v", st) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 6f74ac0..21e68a7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,7 +47,10 @@ type Config struct { ProxyAddr string ProxyAdminKey string - ProxyTLS bool + // ProxyReadKey, a key with the admin:read scope, carries every read and + // every viewer's call, so those cannot change Keystone. Optional. + ProxyReadKey string + ProxyTLS bool TokenmeterURL string // TokenmeterToken is a light-tokenmeter bearer token with the read scope. @@ -114,6 +117,7 @@ func Load() (*Config, error) { InsecureNoAuth: env("INSECURE_NO_AUTH", "") == "true", ProxyAddr: env("KTMS_ADDR", env("LLM_PROXY_ADDR", "")), ProxyAdminKey: env("KTMS_ADMIN_KEY", env("LLM_PROXY_ADMIN_KEY", "")), + ProxyReadKey: env("KTMS_READ_KEY", ""), ProxyTLS: env("KTMS_TLS", env("LLM_PROXY_TLS", "")) == "true", TokenmeterURL: strings.TrimRight(env("TOKENMETER_URL", ""), "/"), // Trimmed: a secret written with echo ends in a newline, which no header may carry. diff --git a/internal/web/static/js/main.js b/internal/web/static/js/main.js index dd82fe0..d9a0a7d 100644 --- a/internal/web/static/js/main.js +++ b/internal/web/static/js/main.js @@ -67,8 +67,9 @@ function Health() { const [h, setH] = useState(session.health); useEffect(() => { api('GET', '/api/health').then((x) => { session.health = x; setH(x); }).catch(() => {}); }, []); const row = (label, st) => st?.enabled && html`
${k.name}${!k.ok && html`${k.refused ? 'refused' : 'unchecked'}`}