From 548585fbe64f9949b70cd4deb14c6b9a5dfd4de3 Mon Sep 17 00:00:00 2001 From: hmziqagent Date: Wed, 4 Nov 2026 21:30:28 +0100 Subject: [PATCH 1/2] docs(research): add four-wave competitive gap research, registry and compensation backlog --- .gitignore | 3 + docs/research/README.md | 33 + docs/research/compensation-backlog.json | 548 ++ docs/research/competitive-gap-analysis.md | 450 ++ .../competitive-landscape-compendium.md | 995 +++ docs/research/harvest-manifest.json | 725 ++ docs/research/niche-repos.md | 1193 ++++ docs/research/repo-intelligence.md | 2695 ++++++++ docs/research/repos-registry.json | 5808 +++++++++++++++++ 9 files changed, 12450 insertions(+) create mode 100644 docs/research/compensation-backlog.json create mode 100644 docs/research/competitive-gap-analysis.md create mode 100644 docs/research/competitive-landscape-compendium.md create mode 100644 docs/research/harvest-manifest.json create mode 100644 docs/research/niche-repos.md create mode 100644 docs/research/repo-intelligence.md create mode 100644 docs/research/repos-registry.json diff --git a/.gitignore b/.gitignore index db36c96..ebd85a5 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ .z-proflow/ .z-flashflow/ .zcode/ +research-repos/ +research-issues/ +research-repos-prep.log diff --git a/docs/research/README.md b/docs/research/README.md index ad0284d..2e67f22 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -17,6 +17,39 @@ extension points for native providers and additional endpoint families. request shaping, response repair, and stable OpenCode configuration. - [Extended endpoints](endpoints.md) — embeddings, rerank, realtime, audio, images, video, files, OCR, batch, vector stores, and search. +- [Competitive gap analysis](competitive-gap-analysis.md) — October 2026 + capability comparison against 40+ LLM gateways and routing products across + two research waves, with prioritized gaps and near-term candidates. +- [Competitive landscape compendium](competitive-landscape-compendium.md) — + consolidated record of the three-wave competitive research: market map, + capability areas, complete gap register (cross-referencing the wave-3 + candidate gaps), implementation blueprints, and a coverage appendix for + all 90 research items. +- [Niche repository survey](niche-repos.md) — third-wave (November 2026) + deep dives into 23 niche repositories — routers, caches, queue + dispatchers, DLP proxies, subscription adapters — each mapped to the gap + register, with a coverage appendix of all 121 deduplicated finds. +- [Product registry](repos-registry.json) — machine-readable registry of + the products researched in depth across the three October waves (72 + products, 859 capability rows), each capability with its evidence URL and + gap-register mapping. Theme, sweep, and frontier findings are folded into + the products they cover, and the 98 uninstantiated wave-3 finds stay in + the niche survey's appendix — hence 72 product rows against 90 inventory + items. The November wave 4 is recorded separately (below) and is + deliberately not folded in: its machine-readable output is the + compensation backlog, whose rows are crate-mapped candidates rather than + capability rows. +- [Repository intelligence](repo-intelligence.md) — fourth-wave (November + 2026) harvest of 63 repositories: 23 full four-angle records (code, + issues, pulls, gaps) and 40 issue-tracker sweeps, each with evidence, + plus a pain-point index, a direction index, recurring unmet needs + observed in three or more repositories, and the stated rule mapping + record findings to backlog candidates. +- [Compensation backlog](compensation-backlog.json) — machine-readable, + priority-sorted backlog of 38 features and fixes implied by the + fourth-wave intelligence (12 high, 19 medium, 7 low), every candidate + backed by evidence carried in the repository-intelligence record and + assigned to the workspace crate where it would land. The normative protocol direction remains in [`protocol.md`](../protocol.md). This directory records edge cases that should diff --git a/docs/research/compensation-backlog.json b/docs/research/compensation-backlog.json new file mode 100644 index 0000000..9961198 --- /dev/null +++ b/docs/research/compensation-backlog.json @@ -0,0 +1,548 @@ +{ + "generated": "2026-11-04", + "source": "docs/research/repo-intelligence.md (fourth research wave: 23 full four-angle harvests + 40 issue-tracker sweeps)", + "candidates": [ + { + "candidate": "Request-path retry and fallback ladder: classified transient-vs-terminal errors, Retry-After honored, per-provider cooldowns, and one whole-request deadline spanning all attempts", + "rationale": "llm-proxy forwards each request exactly once (only discovery has transient retry, crates/llm-proxy-provider/src/discovery.rs:145, verified 2026-11-04). Every gateway in the wave treats classified retry/fallback as core, and the #1 cross-repo pain is misclassification that bans healthy capacity (sub2api #8442, CLIProxyAPI #7197, metapi #582).", + "area": "routing", + "evidence": [ + "https://github.com/ThinkWatchProject/ThinkWatch/pull/122", + "https://github.com/0xzr/freellmpool/pull/97", + "https://github.com/EinStack/glide/pull/68", + "https://github.com/MartialBE/one-hub/issues/852", + "https://github.com/Wei-Shaw/sub2api/issues/8442", + "research-repos/NVIDIA-NeMo__Switchyard/crates/libsy-llm-client/src/client.rs", + "research-repos/EeroEternal__xlb/src/config.rs" + ], + "repos": ["ThinkWatchProject/ThinkWatch", "NVIDIA-NeMo/Switchyard", "EeroEternal/xlb", "0xzr/freellmpool", "EinStack/glide", "MartialBE/one-hub", "Wei-Shaw/sub2api", "router-for-me/CLIProxyAPI", "deanxv/done-hub", "lidge-jun/opencodex", "smg-project/smg", "eliasbiondo/antirion"], + "priority": "high", + "crate": "llm-proxy-provider" + }, + { + "candidate": "Pre-answer in-band failover: stage the upstream stream (bounded buffer) until the first output event so an HTTP 200 followed by an SSE error frame can still be retried or failed over before any byte is committed", + "rationale": "An OpenAI-compatible upstream can return 200 and then reject the stream in its first frame; llm-proxy's documented in-band error handling surfaces that to the client instead of recovering. CLIProxyAPI holds handshake frames in a 48-frame/1MiB budget; sub2api stages 64KiB-to-temp-file; Switchyard holds a one-role prelude.", + "area": "routing", + "evidence": [ + "https://github.com/NVIDIA-NeMo/Switchyard/pull/1133", + "https://github.com/NVIDIA-NeMo/Switchyard/issues/1132", + "research-repos/router-for-me__CLIProxyAPI/internal/runtime/executor/codex_executor_stream.go", + "research-repos/Wei-Shaw__sub2api/backend/internal/service/openai_first_output_timeout.go", + "https://github.com/lidge-jun/opencodex/issues/8107" + ], + "repos": ["router-for-me/CLIProxyAPI", "NVIDIA-NeMo/Switchyard", "Wei-Shaw/sub2api", "lidge-jun/opencodex", "ThinkWatchProject/ThinkWatch"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Cache-token usage fidelity: parse OpenAI prompt_tokens_details.cached_tokens (chat and Responses shapes) and Anthropic 5m/1h cache splits into the existing core cache fields; price cache reads/writes at their own rates", + "rationale": "llm-proxy's core usage carries cache_creation/cache_read (crates/llm-proxy-protocol/src/core.rs:639-641) fed only from Anthropic-dialect fields; a grep for prompt_tokens_details/cached_tokens over crates/ and apps/ returns nothing (verified 2026-11-04), so OpenAI-dialect cached tokens never reach the ledger or cost estimates. Twelve repos show this exact gap breaking cost dashboards, budgets, and client auto-compaction.", + "area": "cost-tracking", + "evidence": [ + "https://github.com/deanxv/done-hub/pull/52", + "https://github.com/deanxv/done-hub/pull/39", + "https://github.com/MartialBE/one-hub/issues/884", + "https://github.com/lidge-jun/opencodex/issues/8101", + "https://github.com/AgentOps-AI/agentops/issues/1445", + "https://github.com/mudler/LocalAI/issues/13006", + "research-repos/deanxv__done-hub/types/common.go" + ], + "repos": ["deanxv/done-hub", "MartialBE/one-hub", "lidge-jun/opencodex", "AgentOps-AI/agentops", "mudler/LocalAI", "mostlygeek/llama-swap", "Inebrio/Routerly", "mozilla-ai/otari", "NVIDIA-NeMo/Switchyard", "api7/aisix", "katanemo/plano", "router-for-me/CLIProxyAPI"], + "priority": "high", + "crate": "llm-proxy-protocol" + }, + { + "candidate": "Context-window preflight: estimate input tokens with the existing offline counter, reject over-limit requests before egress with dialect-exact typed errors (OpenAI context_length_exceeded, Anthropic 'prompt is too long', Responses response.failed)", + "rationale": "llm-proxy already surfaces context_length on /v1/models (crates/llm-proxy-server/src/routes/models.rs:35,142) and ships an offline token counter, but never preflight-checks, so agents get raw upstream 400s and waste spend on doomed calls. mistral.rs shipped exactly this 'for coding agents'; opencodex shows clients branch on the mapped code to decide compact-vs-retry.", + "area": "protocol", + "evidence": [ + "https://github.com/EricLBuehler/mistral.rs/pull/2538", + "https://github.com/lidge-jun/opencodex/pull/8128", + "https://github.com/ericc-ch/copilot-api/issues/210", + "https://github.com/mostlygeek/llama-swap/issues/1281", + "research-repos/os-tack__ostk-cache/src/usage_truth.rs" + ], + "repos": ["EricLBuehler/mistral.rs", "lidge-jun/opencodex", "ericc-ch/copilot-api", "mostlygeek/llama-swap", "os-tack/ostk-cache", "LostRuins/koboldcpp", "yolorouter/yolorouter"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Client-disconnect cancellation and settlement: propagate disconnects upstream, write a cancellation usage row with estimated usage instead of dropping accounting", + "rationale": "llm-proxy's ClientCancelled event (crates/llm-proxy-server/src/routes/core_pipeline.rs:982) covers streams that already started and carries no usage payload; a client leaving while the upstream is awaited leaves no row at all. done-hub decouples the upstream call from the client context so billing always completes; otari writes an estimated row for abandoned streams.", + "area": "observability", + "evidence": [ + "https://github.com/ThinkWatchProject/ThinkWatch/pull/54", + "https://github.com/ThinkWatchProject/ThinkWatch/pull/62", + "https://github.com/mozilla-ai/otari/pull/2869", + "https://github.com/ericc-ch/copilot-api/pull/278", + "research-repos/deanxv__done-hub/providers/base/common.go" + ], + "repos": ["ThinkWatchProject/ThinkWatch", "deanxv/done-hub", "mozilla-ai/otari", "ericc-ch/copilot-api", "EeroEternal/xlb", "EricLBuehler/mistral.rs", "0xzr/freellmpool"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Per-key budgets and multi-window rate limits (RPM/TPM/RPD) wired to the SQLite usage ledger, with in-flight USD reservation for max-output estimates", + "rationale": "llm-proxy rate-limits per identity - caller key id when authenticated, client IP otherwise (crates/llm-proxy-server/src/routes/core_pipeline.rs:173-176, so per-key RPM already exists per the compendium's narrowed gap #3) - but has no token (TPM) windows, per-key spend caps, or in-flight reservations, and client keys grant all-or-nothing model access. Eight repos ship per-key spend caps or multi-window limits; mcp-gateway's enforcer proves the hot-path cost (<0.1ms) in micro-USD atomics; sub2api reserves worst-case USD before forwarding.", + "area": "rate-limiting", + "evidence": [ + "https://github.com/ENTERPILOT/GoModel/pull/1293", + "https://github.com/Nya-Foundation/NyaProxy/pull/133", + "https://github.com/MartialBE/one-hub/issues/835", + "research-repos/yolorouter__yolorouter/internal/model/api_key.go", + "research-repos/mozilla-ai__otari/src/gateway/models/budgets.py", + "research-repos/MikkoParkkola__mcp-gateway/src/cost_accounting/enforcer.rs" + ], + "repos": ["ENTERPILOT/GoModel", "mozilla-ai/otari", "yolorouter/yolorouter", "MikkoParkkola/mcp-gateway", "Nya-Foundation/NyaProxy", "MartialBE/one-hub", "ThinkWatchProject/ThinkWatch", "api7/aisix", "Wei-Shaw/sub2api"], + "priority": "high", + "crate": "llm-proxy-core" + }, + { + "candidate": "Usage-estimation fallback: inject stream_options.include_usage on OpenAI streams, force Accept-Encoding: identity upstream, estimate tokens when usage is absent, and flag rows usage_estimated", + "rationale": "When an upstream omits usage (streams without stream_options, callers leaving mid-stream), llm-proxy records nothing instead of an estimated, flagged row. plano injected exactly these knobs after finding gzipped bodies landing in usage_missing; aisix estimates via bundled tiktoken and tells the client so a caller never reads zero against an unreconcilable bill.", + "area": "cost-tracking", + "evidence": [ + "https://github.com/katanemo/plano/pull/1043", + "research-repos/ThinkWatchProject__ThinkWatch/crates/gateway/src/usage_estimate.rs", + "research-repos/api7__aisix/crates/aisix-proxy/src/token_estimate.rs", + "https://github.com/mozilla-ai/otari/pull/2869" + ], + "repos": ["katanemo/plano", "ThinkWatchProject/ThinkWatch", "api7/aisix", "mozilla-ai/otari"], + "priority": "high", + "crate": "llm-proxy-provider" + }, + { + "candidate": "Phase-shaped timeouts: connect / first-byte (TTFB) / in-stream idle / total, cross-validated at config load, replacing the single flat per-route timeout", + "rationale": "llm-proxy applies one flat TimeoutLayer (crates/llm-proxy-server/src/routes/mod.rs:64-67) and one idle watchdog used identically for prefill wait and reasoning pauses (crates/llm-proxy-provider/src/transport.rs:71-81). Seven repos shape timeouts around LLM stream phases; reasoning models that think for minutes before the first token are killed by flat budgets.", + "area": "ops", + "evidence": [ + "https://github.com/EeroEternal/xlb/pull/2", + "research-repos/EeroEternal__xlb/src/config.rs", + "research-repos/yolorouter__yolorouter/internal/config/config.go", + "https://github.com/ericc-ch/copilot-api/issues/223", + "https://github.com/1mcp-app/agent/issues/709", + "https://github.com/NVIDIA-NeMo/Switchyard/issues/1168" + ], + "repos": ["EeroEternal/xlb", "yolorouter/yolorouter", "ericc-ch/copilot-api", "1mcp-app/agent", "NVIDIA-NeMo/Switchyard", "ThinkWatchProject/ThinkWatch", "mostlygeek/llama-swap"], + "priority": "high", + "crate": "llm-proxy-provider" + }, + { + "candidate": "Client-facing /v1/responses endpoint: streaming event synthesis, wire-exact usage fields, and Codex-proven tool-call grouping/ordering semantics", + "rationale": "llm-proxy's client surface is OpenAI Chat + Anthropic Messages only. Eleven repos serve Responses to clients; one-hub users demand it because Codex-class clients cannot be pointed elsewhere, and its merged PR shows Responses SSE clients hard-require wire-exact usage fields (a zero-value cached_tokens omitted via omitempty broke Codex). completion-to-response is the minimal implementation shape.", + "area": "protocol", + "evidence": [ + "https://github.com/MartialBE/one-hub/issues/868", + "https://github.com/MartialBE/one-hub/pull/910", + "research-repos/NoahStepheno__completion-to-response/internal/transform/transform.go", + "https://github.com/EricLBuehler/mistral.rs/pull/2538", + "research-repos/eliasbiondo__antirion/crates/antirion-pipeline/src/accept.rs" + ], + "repos": ["MartialBE/one-hub", "NoahStepheno/completion-to-response", "EricLBuehler/mistral.rs", "eliasbiondo/antirion", "yolorouter/yolorouter", "api7/aisix", "router-for-me/CLIProxyAPI", "smg-project/smg", "ThinkWatchProject/ThinkWatch", "musistudio/claude-code-router", "deanxv/done-hub"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Egress header hygiene: strip client auth/Origin/browser headers on the upstream hop, never re-attach credentials across redirects, and sanitize client-supplied request ids in logs", + "rationale": "llm-proxy forwards only the inbound auth token in passthrough mode and has no header policy for the server-to-server hop. The cloudflare-workers fix shows caller auth headers overriding provider credentials; llama-swap shows browser Origin headers making Anthropic 401; freellmpool shows redirects leaking credentials to internal targets.", + "area": "auth", + "evidence": [ + "https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/issues/350", + "https://github.com/mostlygeek/llama-swap/issues/1214", + "https://github.com/0xzr/freellmpool/issues/87", + "https://github.com/aws-samples/bedrock-access-gateway/pull/262" + ], + "repos": ["blue-pen5805/llm-proxy-on-cloudflare-workers", "mostlygeek/llama-swap", "0xzr/freellmpool", "aws-samples/bedrock-access-gateway"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Loopback hardening: validate Host/Origin (DNS-rebinding defense) in mode=none, and refuse non-loopback binds without client auth at config validation", + "rationale": "llm-proxy's mode=none safety rests on bind address alone; its CORS layer exists only when allowed_origins is configured and Host is never validated. mcp-gateway shipped an always-on Origin/Host guard as a CWE-346 fix after an external report that a web page could reach the loopback port; jan refused non-loopback binds without a key and stopped '*' trusted-hosts from accepting every Host.", + "area": "auth", + "evidence": [ + "research-repos/MikkoParkkola__mcp-gateway/src/gateway/router/origin_guard.rs", + "https://github.com/janhq/jan/pull/9505", + "https://github.com/janhq/jan/pull/9507", + "https://github.com/cita-777/metapi/issues/631", + "https://github.com/redis/redis-vl-python/pull/746" + ], + "repos": ["MikkoParkkola/mcp-gateway", "janhq/jan", "cita-777/metapi", "redis/redis-vl-python", "AmoyLab/Unla"], + "priority": "high", + "crate": "llm-proxy-server" + }, + { + "candidate": "Subscription OAuth upstream credentials: PKCE browser-login and device-code flows, single-flight proactive refresh, and quota-window telemetry surfaced to the client", + "rationale": "llm-proxy accepts only static bearer keys upstream (SecretString in TOML). Seven repos make consumer-subscription upstreams (Codex/ChatGPT, Claude, Gemini, Grok, Kimi) first-class; claude-code-proxy proves the full flow fits a small Rust binary, and its PR #149 shows quota/reset clocks mapped into the Anthropic usage block.", + "area": "auth", + "evidence": [ + "https://github.com/Inebrio/Routerly/pull/107", + "research-repos/raine__claude-code-proxy/src/providers/codex/auth/manager.rs", + "https://github.com/raine/claude-code-proxy/pull/149", + "https://github.com/cita-777/metapi/pull/430", + "research-repos/deanxv__done-hub/providers/claudecode/base.go", + "https://github.com/ericc-ch/copilot-api/issues/255" + ], + "repos": ["Inebrio/Routerly", "raine/claude-code-proxy", "router-for-me/CLIProxyAPI", "Wei-Shaw/sub2api", "deanxv/done-hub", "cita-777/metapi", "ericc-ch/copilot-api", "musistudio/claude-code-router"], + "priority": "high", + "crate": "llm-proxy-provider" + }, + { + "candidate": "Reasoning-parameter translation: reasoning_effort <-> thinking.budget_tokens <-> Gemini thinkingBudget with per-model clamps, and enforcement of declared budgets", + "rationale": "llm-proxy passes Thinking content blocks through but converts no reasoning parameters between dialects. CLIProxyAPI ships a dedicated conversion engine; Switchyard derives effort from budget ratios with loss diagnostics; bedrock-gateway shows per-generation thinking-API churn breaking static translation; mistral.rs shows accepted-but-unenforced budgets silently spending the whole window.", + "area": "protocol", + "evidence": [ + "research-repos/router-for-me__CLIProxyAPI/internal/thinking/convert.go", + "https://github.com/NVIDIA-NeMo/Switchyard/pull/1216", + "https://github.com/aws-samples/bedrock-access-gateway/issues/248", + "https://github.com/EricLBuehler/mistral.rs/issues/2522", + "https://github.com/NVIDIA-NeMo/Guardrails/issues/2539" + ], + "repos": ["router-for-me/CLIProxyAPI", "NVIDIA-NeMo/Switchyard", "aws-samples/bedrock-access-gateway", "EricLBuehler/mistral.rs", "NVIDIA-NeMo/Guardrails", "oobabooga/textgen"], + "priority": "medium", + "crate": "llm-proxy-protocol" + }, + { + "candidate": "SSE keep-alive heartbeats during silent generation, plus a guaranteed terminal usage event on every stream including tool-call turns", + "rationale": "Silent reasoning windows trip client and intermediary timeouts that heartbeat comment frames prevent (done-hub ::PING; Switchyard asked for event: ping), and koboldcpp shows agent auto-compaction never firing when the terminal usage chunk is dropped on streamed tool requests.", + "area": "protocol", + "evidence": [ + "research-repos/deanxv__done-hub/relay/relay_util/heartbeat.go", + "https://github.com/NVIDIA-NeMo/Switchyard/issues/1168", + "https://github.com/LostRuins/koboldcpp/issues/2480", + "https://github.com/ericc-ch/copilot-api/issues/223" + ], + "repos": ["deanxv/done-hub", "NVIDIA-NeMo/Switchyard", "LostRuins/koboldcpp", "ericc-ch/copilot-api", "ThinkWatchProject/ThinkWatch"], + "priority": "medium", + "crate": "llm-proxy-server" + }, + { + "candidate": "Prompt-cache hygiene: normalize cache-busting volatile bytes (client correlation tokens, per-turn reminder blocks) before egress, keep any request mutation byte-stable past the last cache breakpoint, and optionally auto-inject cache markers", + "rationale": "llm-proxy performs no request-body normalization, no cache_control/prompt_cache_key injection, and has no mutation-safety boundary. ostk-cache measured cache_read going 0 -> 963K tokens/turn just by stripping one volatile token; sub2api engineered fingerprint convergence for the same effect; plano injects markers per provider family.", + "area": "caching", + "evidence": [ + "research-repos/os-tack__ostk-cache/src/billing_strip.rs", + "https://github.com/Wei-Shaw/sub2api/pull/8464", + "https://github.com/musistudio/claude-code-router/pull/1866", + "research-repos/katanemo__plano/crates/brightstaff/src/handlers/llm/prompt_caching.rs", + "research-repos/agiwhitelist__tokdiet/src/providers.ts", + "https://github.com/janhq/jan/pull/9497" + ], + "repos": ["os-tack/ostk-cache", "Wei-Shaw/sub2api", "musistudio/claude-code-router", "katanemo/plano", "agiwhitelist/tokdiet", "janhq/jan", "api7/aisix"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "Cross-protocol reasoning/tool-state replay: preserve Gemini thought signatures on replayed function calls, round-trip OpenAI reasoning_content (never omit when empty), and carry encrypted reasoning through signatures", + "rationale": "metapi shows Gemini upstreams 400-ing replayed tool history missing thought_signature; done-hub ships the sentinel-retry recovery; completion-to-response shows omitting empty reasoning_content breaking DeepSeek multi-turn; claude-code-proxy packs encrypted reasoning into a synthetic signature the client echoes back.", + "area": "protocol", + "evidence": [ + "https://github.com/cita-777/metapi/issues/580", + "research-repos/deanxv__done-hub/relay/common.go", + "research-repos/NoahStepheno__completion-to-response/internal/types/completion.go", + "research-repos/raine__claude-code-proxy/src/providers/codex/translate/reasoning_signature.rs" + ], + "repos": ["cita-777/metapi", "deanxv/done-hub", "NoahStepheno/completion-to-response", "raine/claude-code-proxy", "router-for-me/CLIProxyAPI"], + "priority": "medium", + "crate": "llm-proxy-protocol" + }, + { + "candidate": "Session attribution and optional sticky affinity: record session/conversation ids (from client headers like x-claude-code-session-id and metadata.user_id) on usage rows; optionally pin a session to an upstream for cache locality", + "rationale": "llm-proxy's ledger records only the key id; nine repos attribute per-session and several pin sessions to upstreams explicitly for prompt-cache warmth. otari users cannot see their own spend without session grouping; CCR's fallback misattribution shows why the served model must be recorded per row.", + "area": "observability", + "evidence": [ + "research-repos/Wei-Shaw__sub2api/backend/internal/service/gateway_service.go", + "research-repos/router-for-me__CLIProxyAPI/sdk/cliproxy/auth/session_cache.go", + "research-repos/NVIDIA-NeMo__Switchyard/crates/switchyard-server/src/routing_log.rs", + "research-repos/agiwhitelist__tokdiet/src/session.ts", + "https://github.com/mozilla-ai/otari/issues/2792" + ], + "repos": ["Wei-Shaw/sub2api", "router-for-me/CLIProxyAPI", "NVIDIA-NeMo/Switchyard", "agiwhitelist/tokdiet", "mozilla-ai/otari", "katanemo/plano", "ThinkWatchProject/ThinkWatch", "musistudio/claude-code-router", "lidge-jun/opencodex"], + "priority": "medium", + "crate": "llm-proxy-storage" + }, + { + "candidate": "Per-key model/provider allowlists (glob patterns) enforced after authentication", + "rationale": "llm-proxy's client keys reach everything the catalog exposes; its model allow/deny exists only at catalog level. Four repos ship per-key scoping; CLIProxyAPI's PR shows restricted keys must fail closed for requests without an identifiable model.", + "area": "auth", + "evidence": [ + "https://github.com/MartialBE/one-hub/pull/824", + "https://github.com/router-for-me/CLIProxyAPI/pull/7117", + "research-repos/MikkoParkkola__mcp-gateway/examples/per-client-tool-scopes.yaml", + "https://github.com/Wei-Shaw/sub2api/issues/8480", + "research-repos/deanxv__done-hub/relay/common.go" + ], + "repos": ["MartialBE/one-hub", "router-for-me/CLIProxyAPI", "MikkoParkkola/mcp-gateway", "Wei-Shaw/sub2api", "deanxv/done-hub", "samanhappy/mcphub"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "Gemini-native client ingress: POST /v1beta/models/{model}:generateContent and :streamGenerateContent translated through the existing hub (Gemini is currently upstream-only)", + "rationale": "llm-proxy speaks only OpenAI Chat and Anthropic Messages on the client side. Seven harvested repos serve native Gemini clients - ThinkWatch routes them through the identical pipeline (limits, routing, guards, billing), done-hub mounts a sibling /v1beta group sharing the /v1 middleware chain, and antirion accepts Gemini and Bedrock client dialects inbound. This clears the record's 3+ recurring-needs threshold, unlike the two-repo Ollama dialect candidate that follows it.", + "area": "protocol", + "evidence": [ + "https://github.com/ThinkWatchProject/ThinkWatch/pull/43", + "research-repos/deanxv__done-hub/router/relay-router.go", + "research-repos/yolorouter__yolorouter/internal/router/router.go", + "research-repos/musistudio__claude-code-router/packages/core/src/routing/protocol-endpoints.ts", + "research-repos/agiwhitelist__tokdiet/src/providers.ts", + "research-repos/eliasbiondo__antirion/crates/antirion-pipeline/src/accept.rs" + ], + "repos": ["ThinkWatchProject/ThinkWatch", "deanxv/done-hub", "yolorouter/yolorouter", "router-for-me/CLIProxyAPI", "musistudio/claude-code-router", "agiwhitelist/tokdiet", "eliasbiondo/antirion"], + "priority": "medium", + "crate": "llm-proxy-server" + }, + { + "candidate": "Upstream health and quota-state tracking: passive parsing of 429/rate-limit headers into per-provider cooldowns keyed to real reset times, plus a synthetic probe that auto-disables unreachable models", + "rationale": "llm-proxy's only upstream health notion is discovery-time transient retry. yolorouter parses exactly three dialects and derives bench duration from learned rows with a corroboration cap; sub2api pauses accounts by real quota windows; metapi auto-disables models past probe thresholds; xlb pairs active probes with passive ejection.", + "area": "ops", + "evidence": [ + "https://github.com/0xzr/freellmpool/pull/97", + "research-repos/yolorouter__yolorouter/internal/gateway/rate_limit_evidence.go", + "research-repos/Wei-Shaw__sub2api/backend/internal/service/account_scheduling_threshold_eval.go", + "https://github.com/cita-777/metapi/pull/510" + ], + "repos": ["0xzr/freellmpool", "yolorouter/yolorouter", "Wei-Shaw/sub2api", "cita-777/metapi", "EeroEternal/xlb", "MikkoParkkola/mcp-gateway", "router-for-me/CLIProxyAPI"], + "priority": "medium", + "crate": "llm-proxy-provider" + }, + { + "candidate": "Prometheus exposition and OTel export of the existing lock-free metrics", + "rationale": "llm-proxy's metrics are in-process counters with no scrape or export path. Five repos treat OTel/Prometheus as a standard subsystem; redis-vl's contributor note says 'every deployment writes its own counters' when a library exports none.", + "area": "observability", + "evidence": [ + "https://github.com/EinStack/glide/pull/237", + "https://github.com/AmoyLab/Unla/pull/289", + "research-repos/NVIDIA-NeMo__Switchyard/crates/switchyard-server/src/metrics.rs", + "research-repos/smg-project__smg/model_gateway/src/observability/otel_trace.rs", + "https://github.com/redis/redis-vl-python/pull/814" + ], + "repos": ["EinStack/glide", "AmoyLab/Unla", "NVIDIA-NeMo/Switchyard", "smg-project/smg", "redis/redis-vl-python", "samanhappy/mcphub", "messkan/prompt-cache"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "Pricing catalog sync from models.dev (with refresh and aliasing) plus an explicit unpriced state that is never silently $0", + "rationale": "llm-proxy prices are operator-typed fixed-point TOML with no refresh story. Six repos converge on models.dev as the canonical feed; RouterArena charges unpriced models the table maximum instead of $0 after silent skips corrupted evaluations; tokdiet shows an unpriced month looking like a free month.", + "area": "cost-tracking", + "evidence": [ + "research-repos/mozilla-ai__otari/src/gateway/services/pricing_refresh_service.py", + "https://github.com/katanemo/plano/pull/971", + "research-repos/deanxv__done-hub/model/pricing_modelsdev.go", + "research-repos/musistudio__claude-code-router/scripts/generate-models-json.mjs", + "https://github.com/RouteWorks/RouterArena/pull/218", + "https://github.com/agiwhitelist/tokdiet/issues/1" + ], + "repos": ["mozilla-ai/otari", "katanemo/plano", "deanxv/done-hub", "musistudio/claude-code-router", "RouteWorks/RouterArena", "agiwhitelist/tokdiet", "AgentOps-AI/agentops", "inference-gateway/inference-gateway"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "Hot config reload with refusal semantics: surgical per-provider swaps that never tear down unrelated connections, with restart-required refusals instead of half-applied changes", + "rationale": "llm-proxy requires a restart for config changes. Five repos ship hot reload; mcphub's worst production pain was config edits re-spawning every upstream; mcp-gateway refuses auth-weakening reloads with stable error prefixes and a restart_required flag; preflight keeps the previous runtime on a failed reload.", + "area": "ops", + "evidence": [ + "research-repos/MikkoParkkola__mcp-gateway/src/config_reload/mod.rs", + "research-repos/router-for-me__CLIProxyAPI/internal/watcher/config_reload.go", + "research-repos/ghuntley__preflight/src/main.rs", + "https://github.com/samanhappy/mcphub/issues/1448", + "https://github.com/ENTERPILOT/GoModel/issues/1308" + ], + "repos": ["MikkoParkkola/mcp-gateway", "router-for-me/CLIProxyAPI", "ghuntley/preflight", "samanhappy/mcphub", "ENTERPILOT/GoModel", "ThinkWatchProject/ThinkWatch", "messkan/prompt-cache"], + "priority": "medium", + "crate": "apps/llm-proxy" + }, + { + "candidate": "Idempotency-key replay for retried POSTs: claim the key before budget checks, replay the stored response for matching retries, 409 while in flight", + "rationale": "llm-proxy's dedup window only 409-rejects concurrent identical requests; a client retry after a timeout re-executes and re-bills. mcp-gateway's admission holds active leases that never expire by design; otari's fingerprint covers request-shaping headers so old claims still match; xlb gates replay of non-idempotent requests on the key's presence.", + "area": "protocol", + "evidence": [ + "research-repos/MikkoParkkola__mcp-gateway/src/idempotency/admission.rs", + "research-repos/mozilla-ai__otari/src/gateway/api/routes/_idempotency.py", + "research-repos/EeroEternal__xlb/src/config.rs" + ], + "repos": ["MikkoParkkola/mcp-gateway", "mozilla-ai/otari", "EeroEternal/xlb"], + "priority": "medium", + "crate": "llm-proxy-server" + }, + { + "candidate": "Config-validation hardening: refuse cleartext credentials on http:// URLs to off-machine hosts; warn/refuse on non-loopback binds without auth", + "rationale": "llm-proxy only validates that an endpoint scheme is http or https, so an api_key over plain http to a remote host is accepted silently. mcp-gateway refuses credential-bearing cleartext configs before startup, loopback exempt with an explicit opt-in.", + "area": "auth", + "evidence": [ + "research-repos/MikkoParkkola__mcp-gateway/src/config/mod.rs", + "https://github.com/janhq/jan/pull/9505" + ], + "repos": ["MikkoParkkola/mcp-gateway", "janhq/jan"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "`doctor` command: live provider reachability checks, per-model one-token verification probes, and named remedies for failures", + "rationale": "llm-proxy's `validate` is static config validation and `status` is liveness. mcp-gateway's doctor runs live checks into a pass/fail/warn table where every failure carries fix/verify/rollback commands; the awesome-ai-gateway ecosystem ships standalone key-liveness monitors because balance pages stay green while keys are dead.", + "area": "ops", + "evidence": [ + "research-repos/MikkoParkkola__mcp-gateway/src/commands/doctor.rs", + "https://github.com/cuihuan/awesome-ai-gateway/pull/101", + "https://github.com/Wei-Shaw/sub2api/issues/8454" + ], + "repos": ["MikkoParkkola/mcp-gateway", "cuihuan/awesome-ai-gateway", "Wei-Shaw/sub2api", "musistudio/claude-code-router", "yolorouter/yolorouter"], + "priority": "medium", + "crate": "apps/llm-proxy" + }, + { + "candidate": "In-flight request visibility: expose live requests (provider, model, state, elapsed) in the admin API alongside completed usage rows", + "rationale": "llm-proxy's admin API reports only completed usage; operators debugging slow providers have a blind spot. Routerly's users asked explicitly; claude-code-proxy and ollamaMQ ship live views as first-class operator surfaces.", + "area": "observability", + "evidence": [ + "https://github.com/Inebrio/Routerly/issues/148", + "research-repos/raine__claude-code-proxy/src/monitor.rs", + "research-repos/Chleba__ollamaMQ/src/tui.rs" + ], + "repos": ["Inebrio/Routerly", "raine/claude-code-proxy", "Chleba/ollamaMQ"], + "priority": "medium", + "crate": "llm-proxy-api" + }, + { + "candidate": "Streaming truncation semantics: never synthesize success on an abnormal end; emit typed in-band error events carrying finish-reason context", + "rationale": "llm-proxy already defers in-band errors until after visible output; the recurring ecosystem failure is masking truncation as completion (sub2api #8505, koboldcpp #2481, jan #9515) or emitting a bare socket destroy (CCR #1917). Typed terminal events let clients decide retry-safety.", + "area": "protocol", + "evidence": [ + "https://github.com/Wei-Shaw/sub2api/issues/8505", + "https://github.com/LostRuins/koboldcpp/issues/2481", + "https://github.com/musistudio/claude-code-router/issues/1917", + "https://github.com/0xzr/freellmpool/pull/17", + "https://github.com/EinStack/glide/issues/272" + ], + "repos": ["Wei-Shaw/sub2api", "LostRuins/koboldcpp", "musistudio/claude-code-router", "0xzr/freellmpool", "EinStack/glide", "janhq/jan"], + "priority": "medium", + "crate": "llm-proxy-protocol" + }, + { + "candidate": "Streaming tool-call fragment robustness: longest-match tool-name resolution and tolerance for quotes/newlines inside JSON arguments during fragment reassembly", + "rationale": "llm-proxy's SSE decoders keep tool-arg fragment state; the recurring upstream failure mode in the wild is ambiguous tool-name prefixes (search vs search_web silently executing the wrong tool) and quotes/newlines inside streamed JSON arguments.", + "area": "protocol", + "evidence": [ + "https://github.com/oobabooga/textgen/issues/7666", + "https://github.com/oobabooga/textgen/pull/7477", + "https://github.com/EricLBuehler/mistral.rs/issues/2466" + ], + "repos": ["oobabooga/textgen", "EricLBuehler/mistral.rs", "smg-project/smg"], + "priority": "medium", + "crate": "llm-proxy-protocol" + }, + { + "candidate": "Per-provider concurrency caps with bounded admission: semaphore-limited in-flight requests per provider/model, returning 503 (not 429) with a short Retry-After when saturated", + "rationale": "llm-proxy has no concurrency cap anywhere in the egress path. xlb's RAII lease and queue_full 503, metapi's per-site max_concurrency, and ollamaMQ's per-model caps all exist because capacity shedding misread as rate-limiting arms hour-long client backoff ladders (llama-swap #1256).", + "area": "rate-limiting", + "evidence": [ + "research-repos/EeroEternal__xlb/src/config.rs", + "https://github.com/cita-777/metapi/pull/609", + "https://github.com/mostlygeek/llama-swap/issues/1256", + "research-repos/Chleba__ollamaMQ/src/dispatcher.rs" + ], + "repos": ["EeroEternal/xlb", "cita-777/metapi", "mostlygeek/llama-swap", "Chleba/ollamaMQ", "Nya-Foundation/NyaProxy", "smg-project/smg"], + "priority": "medium", + "crate": "llm-proxy-server" + }, + { + "candidate": "Provider API-key pools: multiple credentials per provider with rotation, cooldown on 401/429/5xx, and utilization-aware ordering", + "rationale": "llm-proxy supports one static key per provider. Users of one-hub forks call the one-key-per-channel shape 'very inconvenient' (done-hub #57); CCR's users file it as the top missing feature (#1923); the cloudflare-workers discovery path already rotates on 429 within a request.", + "area": "auth", + "evidence": [ + "https://github.com/deanxv/done-hub/issues/57", + "https://github.com/musistudio/claude-code-router/issues/1923", + "https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/pull/349", + "https://github.com/ericc-ch/copilot-api/issues/255" + ], + "repos": ["deanxv/done-hub", "musistudio/claude-code-router", "blue-pen5805/llm-proxy-on-cloudflare-workers", "ericc-ch/copilot-api", "router-for-me/CLIProxyAPI", "MartialBE/one-hub"], + "priority": "medium", + "crate": "llm-proxy-core" + }, + { + "candidate": "Self-loop prevention: refuse configurations whose upstream resolves to the proxy's own listener, and stamp/strip a hop header on egress", + "rationale": "These proxies are configured via base-URL swaps, where a wrong URL silently creates infinite forwarding. ostk-cache refuses self-referential upstreams and 508s on hop-header ingress; inference-gateway short-circuits calls that loop back through its own listener.", + "area": "ops", + "evidence": [ + "research-repos/os-tack__ostk-cache/crates/ostk-cache-core/src/capture.rs", + "https://github.com/inference-gateway/inference-gateway/issues/822" + ], + "repos": ["os-tack/ostk-cache", "inference-gateway/inference-gateway"], + "priority": "low", + "crate": "llm-proxy-server" + }, + { + "candidate": "Retention hygiene: prove ledger retention knobs drive eviction, and never orphan sidecar data when pruning", + "rationale": "llm-proxy's SQLite ledger has max_age/max_rows/prune_interval; the ecosystem shows retention bugs where body files outlive rows (13 GB of orphans in nine days on CCR) and capacity knobs that never reach the backend (llama-cpp-python disk cache).", + "area": "ops", + "evidence": [ + "https://github.com/musistudio/claude-code-router/pull/1931", + "https://github.com/mostlygeek/llama-swap/pull/1294", + "https://github.com/abetlen/llama-cpp-python/pull/2400" + ], + "repos": ["musistudio/claude-code-router", "mostlygeek/llama-swap", "abetlen/llama-cpp-python"], + "priority": "low", + "crate": "llm-proxy-storage" + }, + { + "candidate": "Config provenance: print resolved settings with the source of each (CLI > env > config > default)", + "rationale": "llm-proxy supports three override layers (config, env interpolation, CLI) but `validate` prints resolved values without origins, leaving 'why is my flag ignored?' unanswerable. ostk-cache's Source-tagged --print-config answers it from output.", + "area": "ops", + "evidence": [ + "research-repos/os-tack__ostk-cache/src/config.rs" + ], + "repos": ["os-tack/ostk-cache"], + "priority": "low", + "crate": "apps/llm-proxy" + }, + { + "candidate": "Admin/ops MCP surface: expose the admin usage API as MCP tools so an agent can operate the proxy itself", + "rationale": "Three gateways expose their management plane over MCP (inference-gateway stateless 2026-07-28; done-hub's per-user dashboard tool; sub2api's four pre-configured read/write servers reusing REST ownership checks). llm-proxy's read-only admin API is the natural substrate.", + "area": "mcp", + "evidence": [ + "https://github.com/inference-gateway/inference-gateway/pull/709", + "research-repos/deanxv__done-hub/mcp/server.go", + "https://github.com/Wei-Shaw/sub2api/pull/8389" + ], + "repos": ["inference-gateway/inference-gateway", "deanxv/done-hub", "Wei-Shaw/sub2api"], + "priority": "low", + "crate": "llm-proxy-api" + }, + { + "candidate": "Ollama-native client dialect (/api/chat, /api/generate, /api/tags) reusing the existing admission chain", + "rationale": "llm-proxy's inbound surface is OpenAI Chat + Anthropic Messages only. ollamaMQ serves the full Ollama surface on both sides; opencodex added an Ollama-compatible inbound API by reusing one admission path rather than new code - the low-cost pattern for dialect breadth. Kept at low priority despite sitting below the record's 3+ repo threshold (unlike the Gemini ingress candidate above): local-inference alignment is strategic for llm-proxy but the harvested demand is two repos.", + "area": "protocol", + "evidence": [ + "https://github.com/lidge-jun/opencodex/pull/8099", + "research-repos/Chleba__ollamaMQ/README.md" + ], + "repos": ["lidge-jun/opencodex", "Chleba/ollamaMQ"], + "priority": "low", + "crate": "llm-proxy-server" + }, + { + "candidate": "Cache-Control: no-store on all admin API responses", + "rationale": "done-hub shipped this after a CDN cached authenticated API responses into an authorization bypass; llm-proxy emits no Cache-Control/Pragma headers anywhere, including its constant-time-token-gated admin API.", + "area": "auth", + "evidence": [ + "research-repos/deanxv__done-hub/middleware/cache.go" + ], + "repos": ["deanxv/done-hub"], + "priority": "low", + "crate": "llm-proxy-api" + }, + { + "candidate": "Retry terminality signals: when llm-proxy exhausts its own retries, mark the final error non-retryable (x-should-retry: false plus a computed Retry-After) so client retry loops don't stack", + "rationale": "claude-code-proxy's exhausted 503 lacked x-should-retry:false, so Claude Code stacked its own retry loop on the proxy's - one deterministic turn hung ~30 minutes over ~100 upstream calls. ThinkWatch returns computed Retry-After plus x-should-retry: false on every refusal.", + "area": "protocol", + "evidence": [ + "https://github.com/raine/claude-code-proxy/pull/175", + "https://github.com/ThinkWatchProject/ThinkWatch/pull/81", + "https://github.com/router-for-me/CLIProxyAPI/pull/7239" + ], + "repos": ["raine/claude-code-proxy", "ThinkWatchProject/ThinkWatch", "router-for-me/CLIProxyAPI", "inference-gateway/inference-gateway"], + "priority": "low", + "crate": "llm-proxy-server" + } + ] +} diff --git a/docs/research/competitive-gap-analysis.md b/docs/research/competitive-gap-analysis.md new file mode 100644 index 0000000..b2cbc8a --- /dev/null +++ b/docs/research/competitive-gap-analysis.md @@ -0,0 +1,450 @@ +# Competitive gap analysis — llm-proxy vs. LLM gateways (October 2026) + +Products surveyed in October 2026 across two research waves: LiteLLM, Portkey +(rebranded Prisma AIRS AI Gateway in its docs, https://portkey.ai/docs/product/ai-gateway/fallbacks), +Kong AI Gateway, Cloudflare AI Gateway, Agent Router (renamed from Envoy AI +Gateway, https://theagentrouter.ai/), Higress, agentgateway (split out of +kgateway, https://kgateway.dev/blog/kgateway-v2.2-release-blog), Helicone, +Langfuse/LangSmith, OpenRouter, Vercel AI Gateway, Glama, one-api/new-api, +gproxy, openproxy, LLMG, rust-llm-proxy, BricksLLM, Bifrost, vLLM Production +Stack, IBM ContextForge, Azure APIM AI gateway, AWS Bedrock AgentCore +Gateway, and — added in the second wave — Tyk AI Gateway/AI Studio, F5 AI +Gateway, uni-api, GPT-Load, Respan (formerly Keywords AI), Martian, Not +Diamond, Apache APISIX, and Traefik Hub. + +Status notes: TensorZero's repo is archived (2026-06-12, +https://github.com/tensorzero/tensorzero); BricksLLM's last commit and +release are 2025-01-05 (https://api.github.com/repos/bricks-cloud/BricksLLM); +one-api is effectively dormant (last release v0.6.10, Feb 2025, +https://github.com/songquanpeng/one-api/releases); OpenPipe is sunset — +platform migrated to Weights & Biases/CoreWeave and docs.openpipe.ai no +longer resolves (https://openpipe.ai/security); the Portkey OSS gateway has +been dormant since 2026-05-25 after Palo Alto Networks completed its +acquisition while the enterprise line continues (https://github.com/Portkey-AI/gateway/releases); +F5's original gateway is in transition — docs redirect, Python SDK archived +2026-03-18, successor in Early Access (https://www.f5.com/company/news/press-releases/f5-unleashes-next-generation-agentic-ready-ai-gateway-to-optimize-the-economics-and-governance-of-enterprise-ai-costs); +Martian's gateway was live as of 2026-10-27 (https://api.withmartian.com/v1/models); +an "Upstash AI Gateway" named in one brief was verified not to exist — 11 +live checks incl. sitemap, llms.txt, GitHub, npm, and DNS (https://upstash.com/docs/llms.txt). + +Method: llm-proxy claims are verified against this working tree (file:line +cited); wave-1 grep checks ran 2026-10-14, and the wave-2 code claims +(hot_reload rejection, CacheControl shape, discovery field capture, +ModelCard, SSE keep-alive, cancellation handling, body limit) were +re-verified against the tree on 2026-10-27. Competitor claims come from +official docs and release notes current to October 2026 and from vendored +source pins read directly (`ref/litellm` at b84f7f82, `ref/oc-go-cc` at +99d756c, `ref/llm-api-key-proxy` at af40b91). No competitor code was built +or run. Entries first added in the second wave are tagged `new:`. + +## Baseline: llm-proxy today + +- Client surface: `POST /providers/{p}/v1/chat/completions`, + `POST /providers/{p}/v1/messages`, + `POST /providers/{p}/v1/messages/count_tokens`, + `GET /providers/{p}/v1/models` — all four API routes carry the mandatory + `/providers/{provider}` prefix — plus `/health`, `/ready`, `/version` + (`crates/llm-proxy-server/src/routes/mod.rs:50-52,69-79`; + `docs/protocol.md:25-30`) (corrected 2026-10-28 after code audit); + 32 MiB body limit (`crates/llm-proxy-server/src/routes/mod.rs:37,62`); + provider API routes share a configurable `TimeoutLayer` returning 408 + while health/version routes are exempt + (`crates/llm-proxy-server/src/routes/mod.rs:44,54-67`). +- Translation: hub-and-spoke `CoreRequest`/`CoreResponse`/`CoreEvent` + (`docs/protocol.md:7-9`), with 10 content kinds Text/Image/Document/Audio/ + Video/ToolUse/ToolResult/Thinking/RedactedThinking/Refusal + (`docs/protocol.md:80-91`). Client adapters: OpenAI Chat + Anthropic + Messages. Upstream adapters: OpenAI Chat, Anthropic Messages, OpenAI + Responses, Gemini GenerateContent (`crates/llm-proxy-provider/src/adapter/`). +- Streaming: normalized event state machine; in-band errors after visible + output (`docs/protocol.md:120-133,139-151`); 3-second SSE keep-alive + comments on idle streams + (`crates/llm-proxy-server/src/routes/core_pipeline.rs:701,883`). +- Auth: `mode=none` (loopback-only, enforced) or `client-keys` with argon2id + PHC hashes (`config.toml.example:42-54`); admin API behind a constant-time + bearer token (`crates/llm-proxy-api/src/auth.rs`). The upstream `api_key` + is a `SecretString` — redacted in `Debug`, skipped by serialization, + zeroized on drop (`crates/llm-proxy-core/src/provider_config.rs:357-375`) — + but stored as plaintext in the operator's TOML on disk. +- Routing: provider named in URL; static model-alias map + (`crates/llm-proxy-core/src/provider_config.rs:392-394`). No request-path + retry/fallback; only catalog discovery has bounded transient retry + (`crates/llm-proxy-provider/src/discovery.rs:160-170`). +- Catalogs: static, live (OpenAI-compatible `/v1/models`, Anthropic, Gemini), + or hybrid, with single-flight refresh and atomic disk cache + (`crates/llm-proxy-server/src/catalog_service.rs`). Discovery captures only + id/display_name/context_length — no price or lifecycle fields + (`crates/llm-proxy-provider/src/discovery.rs:272-288`, verified 2026-10-27), + and the `/v1/models` `ModelCard` hardcodes `created: 0` with a null + `created_at` (`crates/llm-proxy-server/src/routes/models.rs:24-36`). +- Persistence: SQLite usage ledger, background writer, retention controls; + the only request-body-derived column is `body_hash` + (`crates/llm-proxy-storage/src/sqlite.rs:105-131`; + `config.toml.example:56-68`) — full token/cost columns, no payloads. +- Admin API: read-only `/usage/summary|requests|keys` with window/group_by + filters and cursor paging (`crates/llm-proxy-api/src/routes.rs:53-55, 116-117`). +- Metrics: lock-free atomic counters incl. per-provider-model counts + (`crates/llm-proxy-core/src/metrics.rs`); request-id injection; graceful + shutdown (`crates/llm-proxy-server/src/shutdown.rs`). +- Middleware: token-bucket RPM rate limit, burst-tolerant with continuous + refill (`crates/llm-proxy-server/src/middleware.rs:117-129`), keyed by + the caller's key id when authenticated and by client IP otherwise, with + loopback exemption (`crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`; + per-key RPM since 2026-10-13, commit 499b22a; + `config.toml.example:14-21`) (corrected 2026-10-28 after code audit), a + dedup window that rejects identical in-flight path+body requests with 409 + Conflict (`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`) + — rejection, not coalescing — plus opt-in forwarded-header trust, CORS, + and panic catching. +- Extras: offline tiktoken counting with chars/4 fallback + (`crates/llm-proxy-core/src/token/counter.rs:282`); per-provider fixed-point + USD per-token prices, estimation only + (`crates/llm-proxy-core/src/provider_config.rs:472,500`); per-provider + egress forward proxy socks5h/socks5/http/https + (`crates/llm-proxy-provider/src/transport.rs:124-125,293`); env-var + interpolation; musl single-binary releases. Config changes require a + restart — `server.hot_reload=true` is rejected at load + (`crates/llm-proxy-core/src/provider_config.rs:1531-1538`, verified + 2026-10-27). + +## Auth and keys + +llm-proxy ships two modes: loopback-only no-auth, or static client keys stored +as argon2id hashes in config (`config.toml.example:42-54`). Keys carry no +budgets, model/provider scoping, expiry, or issuance/rotation workflow; +changes require a config edit and restart. + +Competitors treat caller-facing keys as managed records: LiteLLM issues virtual keys via `POST /key/generate` carrying max_budget, budget_duration, tpm/rpm limits, max_parallel_requests, model allowlists and aliases, layered under org→team→user→key budgets (https://docs.litellm.ai/docs/proxy/virtual_keys, https://docs.litellm.ai/docs/proxy/access_control); Bifrost `sk-bf-*` keys carry expiry, an instant `is_active` kill switch, and deny-by-default provider/model scoping (https://docs.getbifrost.ai/features/governance/virtual-keys); OpenRouter provisions credit-capped inference keys via a separate management-key API (https://openrouter.ai/docs/guides/overview/auth/management-api-keys.md); Vercel adds 12-hour OIDC project tokens and an unauthenticated compromised-secret revocation endpoint (https://vercel.com/docs/ai-gateway/authentication-and-byok/oidc); gproxy scopes keys by ordered allow/deny rules over provider, model glob, and operation (https://gproxy.leenhawk.com/guides/permissions/); one-api tokens support expiry, quota, IP ranges, and model allowlists (https://github.com/songquanpeng/one-api/blob/main/README.md); even small rust-llm-proxy ships issue/rotate/revoke endpoints (https://raw.githubusercontent.com/habibi-dev/rust-llm-proxy/master/README.md). +new: agentgateway swaps client virtual keys (`vk-...` Bearer, strict/optional/permissive modes) for provider keys, stores only key hashes, enforces per-key model allowlists (403 `model_not_allowed`), and its v1.7 UI auto-generates rotated values (https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/virtual-keys.md). new: Portkey ships dual-secret rotation — `POST /v2/api-keys/:id/rotate` keeps the old secret valid for `key_transition_period_ms` (min/default 30 min, key id unchanged) plus scheduled rotation policies (https://portkey.ai/docs/product/enterprise-offering/org-management/api-key-rotation). new: Respan (ex-Keywords AI) keys carry Read/Write/Gateway/Admin scopes, Production/Test environments, spend caps, expiry, and an admin-gated `/api/temporary-keys/` endpoint for short-lived minting (https://www.respan.ai/docs/documentation/admin/respan-api-keys). new: GPT-Load AccessKeys are scoped to groups/models/protocols with IP-CIDR restrictions and rotation that preserves id, permissions, and consumed quota (https://www.gpt-load.com/docs/access-keys). new: Tyk App API keys are created per App and must be activated by an administrator before they work — an approve/reject issuance workflow (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/apps.md). new: uni-api exposes `/v1/generate-api-key` behind admin-role keys (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). new: IBM ContextForge hashes gateway tokens (SHA-256) with jti-based revocation/replay detection, encrypts tool credentials AES-GCM, hashes passwords with Argon2id, and optionally resolves secrets from HashiCorp Vault (https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/). new: Tyk stores vendor keys as `$SECRET/Name` references encrypted under `TYK_AI_SECRET_KEY` with env-reference allowlists (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/secrets.md). +Weaker comparators: the Portkey OSS gateway's only auth middleware is the admin-UI token (its middleware directory contains just adminAuth, cache, hooks, log, requestValidator — https://github.com/Portkey-AI/gateway/tree/main/src/middlewares), Higress key-auth stores plaintext credentials in YAML (https://higress.ai/en/docs/latest/user/plugins/authentication/key-auth), and Cloudflare gateway tokens are account-scoped, not restrictable to one gateway (https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/). Upstream-credential encryption at rest ships at gproxy (AES-256-GCM under `GPROXY_MASTER_KEY`, https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md), LiteLLM (NaCl SecretBox keyed from `LITELLM_SALT_KEY` — spend logs and cached prompts stay plaintext — https://docs.litellm.ai/docs/proxy/security_encryption_faq.md, plus enterprise secret-manager backends https://docs.litellm.ai/docs/secret_managers/overview.md), Kong (Vault entity over Konnect Config Store/AWS/GCP/Azure/HashiCorp, https://developer.konghq.com/gateway/entities/vault/), new: Portkey (enterprise secret references to AWS Secrets Manager/Azure Key Vault/HashiCorp Vault resolved at runtime, plus AWS KMS envelope encryption over configs and integration keys — https://portkey.ai/docs/product/enterprise-offering/secret-references.md, https://portkey.ai/docs/product/enterprise-offering/kms.md), and new-api/GPT-Load (`CRYPTO_SECRET`/`ENCRYPTION_KEY`, https://github.com/QuantumNous/new-api, https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md); llm-proxy has no equivalent (plaintext TOML, per the baseline above). + +## Routing and resilience + +llm-proxy routes by URL-path provider plus static alias resolution +(`crates/llm-proxy-core/src/provider_config.rs:392-394`) and performs no +retries, fallbacks, or cooldowns on the request path +(`crates/llm-proxy-provider/src/discovery.rs:160-170` is discovery-only). + +Mature gateways ship retry + fallback + cooldown as core machinery. LiteLLM: six routing strategies (simple-shuffle weighted, least-busy, usage-, latency-, and cost-based; https://docs.litellm.ai/docs/routing), per-exception RetryPolicy, TTL cooldown cache tripped on 429 or >50% failure rate, typed fallback chains (general / context-window / content-policy, max_fallbacks default 5 and cooldown default 5s, `ref/litellm/litellm/constants.py:13,37-43`; https://docs.litellm.ai/docs/proxy/reliability); new: opt-in mid-stream fallback continuation on prefill-capable deployments via assistant prefill, constrained to plain-text partial output (https://docs.litellm.ai/docs/proxy/config_settings). Portkey: JSON config objects compose fallback/loadbalance/conditional strategies recursively; retries cap cumulative wait at 60s and honor Retry-After (https://portkey.ai/docs/product/ai-gateway/fallbacks, https://portkey.ai/docs/product/ai-gateway/automatic-retries); new: a `cb_config` circuit breaker per strategy with cooldown interval ≥30s (https://portkey.ai/docs/product/ai-gateway/circuit-breaker). Bifrost: a FailureClass taxonomy (credential/quota/model_access/rate_limit/caller_fault/transient/timeout…) drives retry-same-key vs rotate-key vs stop; per-request fallback arrays each run as a fresh request with their own retry budget; TTFT timeouts fall back, post-output errors do not (https://docs.getbifrost.ai/features/fallbacks, https://github.com/maximhq/bifrost/blob/dev/core/schemas/failureclass.go); new: plus CEL routing rules with scope precedence, a session-affinity provider pin, and an LLM complexity router exposing `complexity_tier` to rules (https://docs.getbifrost.ai/providers/routing-rules.md, https://docs.getbifrost.ai/providers/session-affinity.md, https://docs.getbifrost.ai/features/complexity-router.md). Cloudflare: up to 5 attempts with per-request backoff control plus ordered fallbacks and versioned dynamic routing (https://developers.cloudflare.com/ai-gateway/configuration/request-handling/, https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/). Agent Router: priority-ordered backends where retries walk the priority chain (https://theagentrouter.ai/docs/capabilities/traffic/provider-fallback). Kong: seven load-balancing algorithms and a circuit breaker (https://developer.konghq.com/plugins/ai-proxy-advanced/). Higress: API-key pool failover with health checks and cooldown restore (https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-proxy/README_EN.md). Vendored oc-go-cc ships three-state per-model circuit breakers and fallback chains (`ref/oc-go-cc/internal/router/fallback.go:15-115`). gproxy routes gateway-defined model names over member providers with operator-set weights and fallback tiers (https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md). +Also shipping retry/fallback: OpenRouter model-array fallbacks and price-weighted provider balancing (https://openrouter.ai/docs/guides/routing/model-fallbacks.md), Vercel health-aware ordering plus model fallback chains with per-provider BYOK timeouts (https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks, https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts), TensorZero three-level retry/fallback hierarchy (https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/guides/retries-fallbacks.mdx), one-api automatic retry and new-api configurable cross-channel retry, off by default (https://github.com/songquanpeng/one-api/blob/main/README.md, https://github.com/QuantumNous/new-api/blob/main/model/request_policy.go), gproxy retries within one provider's credentials before moving on (https://gproxy.leenhawk.com/introduction/architecture/), and llm-api-key-proxy same-key server-error retries under a global deadline (`ref/llm-api-key-proxy/src/rotator_library/README.md`). +new: Second-wave additions — AWS Bedrock AgentCore Gateway: fallback blocks declaring 1–2 ordered targets, triggers on status-code classes, CONNECTION_ERROR/FIRST_BYTE_TIMEOUT/UNROUTABLE, a default-on circuit breaker (open state skips and does not bill), no fallback after a stream starts (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules-fallback.html). It also offers weighted (2-way) route/config-bundle traffic splits in its rule engine (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules.html). Tyk: per-provider failover waterfalls (max 10 rungs; triggers 408/429/5xx/timeout/connect, never 401/403; `X-Tyk-Served-LLM`/`X-Tyk-Failover` headers; pre-first-token only) (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/llm-management.md); its Model Router load-balances across vendor pools by glob model patterns, weighted or round-robin (Enterprise, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/model-router.md). F5: per-service retries (default codes 429/500/502/503/504, Retry-After priority over backoff, 1.5x growth capped 60s) plus a single named `fallbackServiceName` (https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html). agentgateway: per-route retries that always stay on the same backend — cross-model failover is delegated to virtual-model eviction groups (https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/retries.md, https://agentgateway.dev/docs/standalone/latest/documentation/llm/virtual-models.md). uni-api: `AUTO_RETRY` up to 100 attempts with tiered channel/key cooldowns plus six per-key channel scheduling algorithms incl. success-rate-aware `smart_round_robin` and key rotation/cooldown knobs (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). GPT-Load: tier-priority weighted scheduling (group weight × credential weight) under a shared global retry budget that only moves down tiers, no switch after committed stream output (https://www.gpt-load.com/docs/internals/scheduling). Respan: retries on by default (≤3 attempts) with ordered `fallback_models`; validation/context/credential failures never retried (https://www.respan.ai/docs/documentation/features/gateway/retries). + +## Rate limiting and quotas + +llm-proxy rate limits authenticated callers per key id and unauthenticated +callers per client IP — a burst-tolerant token bucket with continuous +refill (`crates/llm-proxy-server/src/middleware.rs:117-129`) keyed at +`crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176` (per-key RPM +since 2026-10-13, commit 499b22a) — exempting loopback +(`config.toml.example:14-21`). Token-based (TPM) limits and concurrency +caps are what remain absent (corrected 2026-10-28 after code audit). + +Competitors scope limits to identity, not IP. LiteLLM enforces max_parallel_requests/rpm/tpm per key, user, and team in one batched pre-call check, with a reserve-then-reconcile TPM path (estimate prompt tokens + max_tokens up front, settle actuals after, refund on failure) (https://docs.litellm.ai/docs/proxy/users). Kong limits on token kinds or cost with policy-matched ceilings at consumer, IP, header, path, model, and provider level (https://developer.konghq.com/plugins/ai-rate-limiting-advanced/). Agent Router charges token budgets (typed token kinds or CEL expressions) into Redis-backed windows and never interrupts an admitted stream (https://theagentrouter.ai/docs/capabilities/traffic/usage-based-ratelimiting). Higress counts tokens per consumer/header/IP over Redis (https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-token-ratelimit). Bifrost applies request and token limits at provider-config, key, team, and customer level with typed 401/402/403/429 errors (https://docs.getbifrost.ai/features/governance/budget-and-limits). gproxy rate limits per user/key including concurrency (https://gproxy.leenhawk.com/guides/permissions/). new-api adds per-IP, per-group, and model-level limits (https://docs.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced). +new: Second-wave additions — agentgateway: request- and token-type limits keyed by CEL expressions (`jwt.sub`, `apiKey.user`, `llm.requestModel`), local in-memory LRU buckets (≤65,536/rule) or a remote Envoy Rate Limit gRPC service; token limits enforce in two phases when the backend sets `tokenize: true` (upfront estimate blocks, otherwise post-hoc) (https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/rate-limits.md). AWS AgentCore: dimension-keyed limits (targetName, toolName, qualifiedModelId, JWT claims, IAM principal; immutable dimension keys, most-specific-wins) over requests/sec, tokens/min with pre-estimate plus provider-reported actuals, and concurrent connections — fail-open by default, explicitly "not a security boundary" (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rate-limits.html). Traefik Hub: `ai-rate-limit` token bucket (burst-tolerant) vs `ai-quota` sliding window (no bursts), input/output/total-token or USD-cost limits, per-consumer `sourceCriterion`, pre-request body-length/4 estimate rejection, Redis-backed fail-open (https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit). Azure APIM: one `llm-token-limit` policy combining tokens-per-minute (429) and long-period token quotas over hourly-to-yearly fixed UTC windows (403), keyed on any counter-key expression, with optional `estimate-prompt-tokens` pre-call rejection (https://learn.microsoft.com/en-us/azure/api-management/llm-token-limit-policy). GPT-Load: per-AccessKey RPM plus a concurrency cap (https://www.gpt-load.com/docs/access-keys). uni-api: stacked windows like `15/min,10/day` per key, per-model substring limits, and per-upstream-key knobs (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). Respan: layered per-key cost/request/token limits plus customer (402) and org-plan (429) tiers (https://www.respan.ai/docs/documentation/features/gateway/limits). TensorZero shipped scoped rate-limit rules over inferences/tokens/cost with priority resolution and Valkey/Postgres-persisted state before archival (https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/enforce-custom-rate-limits.mdx). +Weaker comparators: Cloudflare limits per gateway only (https://developers.cloudflare.com/ai-gateway/features/rate-limiting/), one-api's global IP limit excludes the relay data path (https://github.com/songquanpeng/one-api/blob/main/README.en.md), Vercel does not rate-limit paid-tier requests (https://vercel.com/docs/ai-gateway/rate-limits), and Tyk ships no core RPM/TPM limiter on the LLM path (rate limiting arrives as plugins, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/plugins-overview.md). + +## Cost tracking + +llm-proxy computes estimated USD cost from operator-configured per-token +prices (`crates/llm-proxy-core/src/provider_config.rs:472`) and records usage +in the SQLite ledger with body digests +(`crates/llm-proxy-storage/src/sqlite.rs:105-131`). Prices are neither +shipped nor synced — a grep for pricing code across `crates/` and `apps/` +(run 2026-10-14) finds per-token prices only in +`crates/llm-proxy-core/src/provider_config.rs` and their consumer +`crates/llm-proxy-server/src/routes/core_pipeline.rs`, and discovery fetches +model lists, not prices (`crates/llm-proxy-provider/src/discovery.rs`) — and +no code path enforces a spend limit (those same two files are the only price +consumers). + +Competitors maintain pricing databases wired into budgets: LiteLLM computes per-request cost against a 2,752-model price map, writes full spend logs, exports spend to Prometheus, and supports pre-call budget reservation (https://docs.litellm.ai/docs/proxy/cost_tracking); Portkey attributes cost from the open Portkey-AI/models database, 2,300+ models pushed 2026-10 (https://github.com/Portkey-AI/models); Helicone prices from a 300+ model repository (https://docs.helicone.ai/references/how-we-calculate-cost.md); agentgateway prices by token kind (input/output/cacheRead/cacheWrite/reasoning/audio) with context-length tiers and models.dev catalog import (https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/costs.md); Kong models provider billing including TTL-tiered cache-write rates (https://developer.konghq.com/ai-gateway/model-cost-management/). Budget enforcement wired to prices ships at OpenRouter, Vercel, Cloudflare, and LangSmith (https://openrouter.ai/docs/guides/best-practices/spend-controls.md, https://vercel.com/docs/ai-gateway/observability-and-spend/budgets, https://developers.cloudflare.com/ai-gateway/features/spend-limits/, https://docs.langchain.com/langsmith/llm-gateway-spend-policies), and BricksLLM pioneered per-key USD caps in 2023 (https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md). gproxy enforces USD budgets on users, keys, orgs, and teams, settled after the upstream call (https://gproxy.leenhawk.com/guides/permissions/). +new: Budget shape and semantics (second wave) — Bifrost: budget arrays `{max_limit USD, reset_duration 1m–1Y}` with optional UTC calendar alignment, fiscal-quarter starts, temporary overrides, and reset-usage actions; enforced cumulatively at customer→team→key→provider-config scope (https://docs.getbifrost.ai/features/governance/budget-and-limits.md). LiteLLM: stackable multi-window `budget_limits` (e.g. $10/24h + $100/30d), calendar-aligned resets with IANA timezone and `budget_reset_time`, soft budgets, temporary raises, and tag-based budgets attached per request header or W3C baggage (https://docs.litellm.ai/docs/proxy/users, https://docs.litellm.ai/docs/proxy/budget_reset_and_tz, https://docs.litellm.ai/docs/proxy/tag_budgets). agentgateway: per-key USD-or-token budgets with rolling windows, Block (429 `budget_exceeded`) or Audit modes, multiple independent windows per key, requiring its SQLite/Postgres config DB (https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/budget-limits/per-key.md). Portkey enterprise: usage policies with AND-conditions and `group_by` counters (per-user, per-model, per-key, even per-MCP-server) with weekly/monthly resets — breach returns 412 (https://portkey.ai/docs/product/enterprise-offering/budget-policies). Cloudflare: spend-limit rules scoped by provider/model/metadata with split-by-value buckets (per-user budgets via `cf.user_id`) and a budget-exceeded fallback to a cheaper dynamic-route model — the only budget→fallback action found (https://developers.cloudflare.com/ai-gateway/features/spend-limits/). OpenRouter: layered guardrail/workspace/key budgets with in-flight holds that pre-reject oversized requests (`weight_exceeds_budget` vs retryable `in_flight_budget_exhausted`, both 402) (https://openrouter.ai/docs/api-reference/limits). new-api: subscription quota packages plus a prepaid wallet — a monetization loop, not just a cap (https://github.com/QuantumNous/new-api). Tyk: monthly budgets at LLM/App/team level with a ~30s budget pulse pushing block-lists to edge gateways; enforcement and team pools are Enterprise-only — Community tracks but never blocks (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/budgeting.md). Respan: per-key spend caps with alert/Block thresholds and customer-level budgets returning 402 (https://www.respan.ai/docs/documentation/features/gateway/limits). GPT-Load: cost quotas as never-resetting totals or periodic windows from one minute to one year — explicitly soft enforcement, charged after completion (https://www.gpt-load.com/docs/access-keys). uni-api: per-key dollar-credit quotas with configurable 30-day default windows, 0 blocks use (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). Traefik Hub: `costLimit` in USD alongside its token limits (https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit). new: Status-code taxonomy has not converged — Bifrost returns 402, Portkey 412, Respan 402/429, LiteLLM a documented 400/401/429 mix — so the choice must be deliberate (https://github.com/maximhq/bifrost/blob/dev/plugins/governance/main.go, https://portkey.ai/docs/api-reference/inference-api/error-codes, https://docs.litellm.ai/docs/proxy/users). new: agentgateway also attributes invoice-grade cost into provider bills — Bedrock per-request STS role assumption so session names land in CloudTrail/CUR, Vertex billing labels into Cloud Billing export (https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/attribution.md). + +## Caching + +llm-proxy has no response cache. Its dedup window rejects identical in-flight +requests with 409 +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`): the second +caller gets an error, not the first response — pressure control, neither a +cache nor coalescing. + +Exact-match caching is near-universal: LiteLLM (8+ backends; SHA-256 over recognized params; streams reassembled then replayed synthetically; https://docs.litellm.ai/docs/proxy/caching), Portkey (simple + semantic, TTL bounds 60s–90d, namespacing; https://portkey.ai/docs/product/ai-gateway/cache-simple-and-semantic), Cloudflare (SHA-256 over provider+endpoint+model+auth+body, TTL 60s–1mo; https://developers.cloudflare.com/ai-gateway/features/caching/), Helicone (edge cache, plus an OSS Rust proxy hashing seed+path+body with RFC 7234 freshness over Moka or Redis, buffered bodies, 2xx-only; https://docs.helicone.ai/features/advanced-usage/caching.md, https://github.com/helicone/ai-gateway), and OpenRouter opt-in with free hits (https://openrouter.ai/docs/guides/features/response-caching.md). Higress ai-cache's base mode caches chat completions in Redis keyed by GJSON paths over the request body (exact string match, streaming replay included); new: a semantic mode adds a DashVector collection plus an embedding provider through the console (https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-cache, https://higress.ai/en/docs/ai/scene-guide/semantic-cache). new: Respan caches exact "messages only" matches with per-customer/per-model scoping and a hit/savings dashboard (https://www.respan.ai/docs/documentation/features/gateway/caching). new: Tyk delivers caching as plugins — a community exact-match plugin and an enterprise semantic plugin (cosine ≥0.95 over Redis/memory vector stores) (https://tyk.io/docs/ai-management/ai-studio/semantic-caching). new: the vLLM Production Stack ships an alpha, feature-gated semantic cache (sentence-transformer, threshold 0.95) (https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/experimental/README.md). Semantic caching also ships at Bifrost (hash + cosine similarity over vector stores; https://docs.getbifrost.ai/features/semantic-caching), Kong (https://developer.konghq.com/plugins/ai-semantic-cache/), and Azure APIM via RediSearch (https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities). +Single-flight is rarer than it looks: Cloudflare documents the duplicate-origin race and OpenRouter documents that simultaneous identical requests are not coalesced (URLs above); the Portkey OSS, Helicone OSS, and LiteLLM cache implementations examined at source level contain no in-flight dedup; coalescing was not verified either way for the remaining products. llm-proxy's dedup window rejects duplicates outright rather than serving them, so origin protection exists but the second caller still fails. new: Verified wave-2 absences — agentgateway ships no LLM response cache (homepage advertises "semantic caching" but the Kubernetes API reference contains no such resource; https://agentgateway.dev/), and AWS AgentCore, Tyk core, GPT-Load, and uni-api document none. + +## Guardrails + +llm-proxy ships no content inspection, redaction, or moderation. + +LiteLLM integrates ~50 guardrail services across seven hook phases including pre-call, post-call, and MCP phases (https://docs.litellm.ai/docs/proxy/guardrails/quick_start, https://docs.litellm.ai/release_notes); new: it also ships reversible Presidio PII masking with `output_parse_pii` re-insertion (https://docs.litellm.ai/docs/tutorials/presidio_pii_masking.md). Portkey runs 20+ deterministic checks plus LLM checks (PII, moderation, language, gibberish) with deny=HTTP 446 / flag=246 semantics that can drive fallback orchestration (https://portkey.ai/docs/product/guardrails); new: its pre-LLM PII redaction is irreversible by design (https://portkey.ai/docs/product/guardrails/pii-redaction). Cloudflare ships a fixed taxonomy (S1–S13 hazards via Llama Guard 3 8B plus P1 prompt injection via Prompt Guard 2) with per-category flag/block (https://developers.cloudflare.com/ai-gateway/features/guardrails/) plus DLP scans shared with Cloudflare One (https://developers.cloudflare.com/ai-gateway/features/dlp/). Kong's ai-sanitizer redacts 20+ PII entity types bidirectionally (https://developer.konghq.com/plugins/ai-sanitizer/). Bifrost (enterprise) enforces CEL rules, Gitleaks secret detection, and LLM-as-judge policies (https://docs.getbifrost.ai/enterprise/guardrails). LangSmith's gateway redacts PII/secrets in-path with placeholder restoration, including streams (https://docs.langchain.com/langsmith/llm-gateway-data-policy). Helicone's LLM Security screens prompt injection via Meta Prompt Guard (advanced: Llama Guard), currently OpenAI models only (https://docs.helicone.ai/features/advanced-usage/llm-security.md). new-api screens sensitive words on prompts and/or responses (https://github.com/QuantumNous/new-api/blob/main/model/request_policy.go); new: that screening became off-by-default in rc.45 (https://github.com/QuantumNous/new-api/releases). Higress delegates moderation to Aliyun's paid service (https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-security-guard). Cloud vendors: Azure AI Content Safety (https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities), AWS Bedrock Guardrails (https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html). +new: Second-wave additions — Tyk: Tengo-script filters with vendor-agnostic redaction/quarantine helpers over normalized messages (OpenAI/Anthropic/Google/Bedrock shapes) calling Lakera, Azure Content Safety, Presidio, Bedrock Guardrails, or HTTP classifiers, with streaming re-evaluation and tool-result injection screening from 2.3 (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/filters.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/guardrails.md). F5: a processor chain (prompt-injection ML detector, deterministic Data Security PII matcher incl. an `mcp` matcher, language-id, system-prompt injection) with modify/reject/annotate outcomes (https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/processors-overview.html). agentgateway: ordered prompt guards — regex with built-in PII detectors (mask/reject/audit), OpenAI moderation, Bedrock/Azure/Google vendor guards, generic webhooks, failClosed default; masking is reject-only on streams (https://agentgateway.dev/docs/standalone/latest/documentation/llm/prompt-guards/overview). GPT-Load: experimental JEV guardrails via an external decision service plus global RE2 redaction with reversible per-AccessKey encryption restored in responses and streams (https://www.gpt-load.com/docs/advanced/guardrails, https://www.gpt-load.com/docs/advanced/redaction). uni-api: optional omni-moderation screening of user messages (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). Respan: free 8-category request blocking via its span-01-security model, org-wide only (https://www.respan.ai/docs/documentation/features/gateway/guardrails). AWS AgentCore: deny-by-default Cedar policies on MCP `tools/call` with Bedrock Guardrails confidence-score conditions (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/use-gateway-with-policy.html). +Agent Router documents no guardrails as of October 2026 — its security surface is transport/auth (https://theagentrouter.ai/docs/capabilities/security/); that absence is per its docs, not a code audit. + +## Observability + +llm-proxy exposes atomic counters in-process, injects request ids, logs +plain/JSON, and shuts down gracefully; it collects up to 1000 latency +samples (`crates/llm-proxy-core/src/metrics.rs:34-35`) and computes +percentiles in-process (`:210-239` — `Snapshot.latencies` plus +`calculate_percentile`/`calculate_p95`/`calculate_p99`) — but has no +`/metrics` scrape endpoint, no TTFT measurement, and no trace export, and +the percentiles are never exposed (corrected 2026-10-28 after code audit). + +Competitors standardize on Prometheus + OTel GenAI conventions. LiteLLM exports request/latency/TTFT/queue/overhead histograms, token and spend counters with identity labels, budget-remaining gauges (https://docs.litellm.ai/docs/proxy/prometheus), and OTel trace export with `traceparent` propagation via its callback pipeline (https://docs.litellm.ai/docs/proxy/logging). agentgateway — also Rust/tokio — implements the OTel GenAI semconv metric names directly (`gen_ai.server.request.duration`, `time_to_first_token`, `time_per_output_token`, token-usage histograms) on :15020 plus OTLP tracing with per-request custom labels and metric removal via CEL (https://agentgateway.dev/docs/standalone/latest/documentation/observability/metrics/reference/, https://agentgateway.dev/docs/standalone/latest/documentation/observability/metrics/overview). Bifrost exposes per-provider/model/virtual-key cost, token, latency-histogram, TTFT, and key-health metrics with SQLite/Postgres-backed logs and a web UI (https://docs.getbifrost.ai/features/observability/prometheus, https://docs.getbifrost.ai/features/observability/default), and an `llm.call` OTel span per request with per-attempt spans exported over OTLP HTTP or gRPC (https://docs.getbifrost.ai/features/otel.md). Kong's Prometheus `ai_metrics` exposes request/cost/token counters (token_type labels incl. cache-write TTL splits), LLM latency histograms, and TTFT under the gen_ai conventions (https://developer.konghq.com/ai-gateway/monitor-ai-llm-metrics/). Cloudflare adds OTLP export under GenAI semconv and per-user spend anomaly detection (https://developers.cloudflare.com/ai-gateway/observability/otel-integration/). Vercel ships per-attempt routing logs and OTel trace drains (https://vercel.com/docs/ai-gateway/observability-and-spend/logs, https://vercel.com/docs/ai-gateway/observability-and-spend/trace-drains). OpenRouter broadcasts OTLP traces to ~20 destinations (https://openrouter.ai/docs/guides/features/broadcast/webhook.md). TensorZero shipped three Prometheus metrics plus OTLP traces before archival (https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/export-prometheus-metrics.mdx); the vLLM Production Stack ships Prometheus/Grafana/OTel in-chart (https://github.com/vllm-project/production-stack); Higress ai-statistics emits per-route/cluster/model/consumer token and latency counters (https://higress.ai/en/docs/latest/user/plugins/ai/api-o11y/ai-statistics); vendored oc-go-cc computes p95/p99 at /health (`ref/oc-go-cc/internal/metrics/metrics.go:132-154`). The OTel client-inference span shape is standardized at https://raw.githubusercontent.com/open-telemetry/semantic-conventions-genai/main/docs/gen-ai/client-inference.md. +new: Second-wave additions — Tyk serves a secure-by-default Prometheus `/metrics` exporting OTel GenAI semconv metrics (`gen_ai_server_request_duration_seconds`, time-to-first-token, time-per-output-token) plus `aistudio_*` governance counters, a Server-Timing header splitting gateway overhead from upstream latency, and opt-in OTel span export with always-on W3C traceparent propagation (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/observability.md). IBM ContextForge ships OTLP tracing to eight backends plus a JWT-protected Prometheus endpoint (https://ibm.github.io/mcp-context-forge/latest/manage/observability/). AWS AgentCore publishes CloudWatch metrics (Latency measured to first response token, p50/p90/p99, throttles, per-target execution time) and vended logs including request/response bodies with OTel-compatible spans (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-gateway-metrics.html). Helicone computes TTFT and p50/p95/p99 latency in its analytics dashboards rather than exporting histograms (docs mirror: https://helicone-helicone-7.mintlify.app/observability/metrics). +new: Streaming SLO mechanics (second wave) — LiteLLM sends SSE `: ping` keep-alives on silent streams (`keepalive_seconds` 1–300), cancels upstream on client disconnect (`cancel_on_disconnect`, logged 499), and can embed usage in every chunk (`always_include_stream_usage`) for billing when the final chunk never arrives (https://docs.litellm.ai/docs/proxy/config_settings). OpenRouter documents per-provider stream-cancel billing semantics (abort propagates at ~24 providers; Bedrock/Groq/Google/Mistral keep processing and bill) and a final usage-chunk accounting frame with in-band mid-stream error events (https://openrouter.ai/docs/api_reference/streaming). oc-go-cc emits a `:keepalive` comment every 3s during streaming (`ref/oc-go-cc/internal/handlers/messages.go:239-266`); uni-api ships an SSE heartbeat interval (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). Keep-alive itself is parity: llm-proxy already emits 3s SSE keep-alive comments (`crates/llm-proxy-server/src/routes/core_pipeline.rs:701,883`). None of the products whose pages are cited above documents SLO machinery (burn-rate alerts, error budgets) — percentiles are left to PromQL, and TTFT measurement points differ across products. +Payload capture is a separate axis llm-proxy declines: its ledger keeps digests only (`crates/llm-proxy-storage/src/sqlite.rs:130-131`), while Helicone records full bodies with per-request omit headers (https://docs.helicone.ai/features/advanced-usage/omit-logs.md), LiteLLM stores messages with redaction controls (https://docs.litellm.ai/docs/proxy/logging), Cloudflare logs payloads with per-request opt-out (https://developers.cloudflare.com/ai-gateway/observability/logging/). new: Cloudflare Logpush additionally exports bodies customer-key-encrypted — a fresh AES-256-GCM key per log wrapped with the customer's RSA-4096 public key (https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/). OpenRouter retains prompt/completion content only behind explicit opt-in (https://openrouter.ai/docs/guides/features/logs.md). new: Per-request metadata/tag attribution — Helicone `Helicone-Property-*` headers, LiteLLM body `metadata.tags`/`x-litellm-tags` plus User-Agent auto-tagging, Portkey `x-portkey-metadata`, Cloudflare `cf-aig-metadata` (deliberately excluded from the cache key), Vercel `providerOptions.gateway.user`/`tags`, Glama `x-glama-metadata` — is the dashboard grouping key everywhere; llm-proxy records only the key id in usage events. + +## MCP + +llm-proxy has no MCP client, server, or gateway functionality. + +The MCP wire protocol standardized Streamable HTTP + OAuth 2.1 resource-server auth in the 2025-11-25 spec (https://modelcontextprotocol.io/specification/2025-11-25/basic/transports, https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization); a 2026-07-28 release candidate removes sessions and adds gateway-friendly per-operation routing headers (https://blog.modelcontextprotocol.io/posts/2026-07-28-release-candidate). +Shipped gateways: LiteLLM routes all MCP tool traffic through one endpoint with six-level permissions and per-tool allowlists (https://docs.litellm.ai/docs/mcp, https://docs.litellm.ai/docs/mcp_control); Cloudflare MCP portals aggregate up to 80 HTTP MCP servers behind Access policies with managed upstream OAuth (https://developers.cloudflare.com/cloudflare-one/access-controls/ai-controls/mcp-portals/); agentgateway federates MCP servers behind one endpoint with CEL authorization per MCP method (https://agentgateway.dev/docs/standalone/latest/documentation/mcp/mcp-authz.md); new: v1.7 adds an ExtMCP guardrail processor chain and an inline CEL processor (https://agentgateway.dev/docs/standalone/latest/documentation/mcp/guardrails/about.md); Bifrost re-exposes federated tools at /mcp with virtual-key-scoped filtering (https://docs.getbifrost.ai/mcp/overview); new: tool execution there is explicit by default (a separate `/v1/mcp/tool/execute` call) with opt-in agent auto-approval and a sandboxed code mode across 3+ servers (https://docs.getbifrost.ai/mcp/overview.md). Kong bridges REST↔MCP in four modes (https://developer.konghq.com/plugins/ai-mcp-proxy/); new: Kong 2.1.0 added the stateless 2026-07-28 revision with `ttlMs`/`cacheScope` cache hints and W3C trace-context to tool calls (https://developer.konghq.com/ai-gateway/changelog/). Azure APIM exports managed REST APIs as MCP servers (https://learn.microsoft.com/en-us/azure/api-management/mcp-server-overview); AWS Bedrock AgentCore Gateway converts APIs/Lambda into MCP tools with per-tool credential injection (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html); new: it also serves stateful MCP sessions with elicitation/sampling and semantic tool search over the catalog (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-features.html). IBM ContextForge federates MCP/A2A/REST/gRPC (https://github.com/IBM/mcp-context-forge); Glama runs a hosted MCP gateway with per-tool ACLs (https://glama.ai/mcp/gateway); Higress hosts MCP servers in-gateway (https://higress.cn/en/docs/ai/mcp-quick-start/). +new: Second-wave additions — Tyk ships MCP twice: Tyk Gateway v5.13+ proxies remote MCP servers with OAuth 2.1, per-key tool allowlists, per-tool rate limits and circuit breakers (https://tyk.io/docs/ai-management/mcp-gateway/overview.md), and AI Studio exposes its own tools as MCP servers at `/tools/{slug}/mcp` (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/tools.md). Respan runs a remote MCP server exposing logs, traces, prompts and monitors to AI tools (https://www.respan.ai/docs/documentation/mcp). OpenRouter hosts an MCP server at mcp.openrouter.ai whose OAuth flow issues spend-capped 7-day keys (https://openrouter.ai/docs/guides/overview/mcp-server.md). + +## Protocol breadth + +llm-proxy speaks two client formats (OpenAI Chat, Anthropic Messages) and +four upstream formats, plus count_tokens; embeddings/audio/images/batch/ +realtime are scoped future work (docs/research/endpoints.md). + +Breadth leaders: LiteLLM proxies chat, Responses, batches, files, assistants, images, audio, rerank, RAG, realtime, OCR, containers, fine-tuning, A2A, and search families (https://docs.litellm.ai/release_notes); Portkey OSS ships handlers for completions, embeddings, batches, files, fine-tunes, images, speech, transcriptions, translations, realtime WebSocket, Responses, and Anthropic messages + count_tokens (https://github.com/Portkey-AI/gateway/tree/main/src/handlers); OpenRouter serves chat, Responses, and Anthropic Messages plus embeddings, rerank, image/video generation, TTS/STT, batch, files, containers (https://openrouter.ai/docs/_llms/api-reference.txt); Vercel serves the same three client formats plus embeddings, rerank, and Decisions (https://vercel.com/docs/ai-gateway/sdks-and-apis); new: Vercel organizes the surface as nine modalities including video generation and a typed Decision API (https://vercel.com/docs/ai-gateway/modalities). new: Cloudflare's hosted REST API exposes `/ai/v1/chat/completions`, `/ai/v1/responses` (OpenAI Responses schema), and `/ai/v1/messages` (https://developers.cloudflare.com/ai-gateway/usage/rest-api/). new: the LangSmith LLM Gateway (beta) serves `/v1/chat/completions`, `/v1/messages`, and `/v1/responses` through one API key (https://docs.langchain.com/langsmith/llm-gateway). new: Helicone's gateway accepts the OpenAI Responses format at `/v1/responses` for both OpenAI and Anthropic models (https://docs.helicone.ai/gateway/concepts/responses-api.md). new: agentgateway's client-facing route types include OpenAI Chat Completions, Anthropic Messages, OpenAI Responses, and Gemini generateContent, translated between format families in both directions (https://agentgateway.dev/docs/standalone/latest/documentation/llm/about.md). new: AWS AgentCore inference accepts chat/completions, messages, responses, and Bedrock converse/InvokeModel wire formats with cross-dialect translation (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-targets-inference.html). new-api accepts four client protocols (OpenAI, Responses, Anthropic, Gemini) with cross-conversion and realtime WebSocket (https://github.com/QuantumNous/new-api); new: it also ships gateway-provided `/v1/tools/search` and `/v1/tools/fetch` billed per call (https://github.com/QuantumNous/new-api/releases). Agent Router accepts four client formats under configurable prefixes and translates to ~20 providers (https://theagentrouter.ai/docs/capabilities/llm-integrations/supported-endpoints); Higress ai-proxy auto-detects OpenAI vs Claude clients and targets ~30 provider backends (https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy); LLMG auto-registers 70+ providers from environment variables (https://modpotatodotdev.github.io/LLMG/providers/all/); Kong passes requests upstream in native provider format across 17 providers (https://developer.konghq.com/plugins/ai-proxy/). new: Tyk fronts four surfaces (vendor-native passthrough, OpenAI-format translation, an Anthropic→Bedrock bridge, legacy), with governed batches/Files/Responses arriving only in the unreleased 2.3.0 line (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/proxy.md, https://github.com/TykTechnologies/ai-studio/blob/main/CHANGELOG-v2.3.0.md). new: uni-api proxies chat, responses (+compact), messages, images, embeddings, audio speech/transcriptions, moderations, and search across 16+ upstream engines (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). new: GPT-Load accepts OpenAI Chat/Responses/Images/Embeddings/Rerank, Mistral-native OCR/audio, Anthropic Messages, and Gemini incl. `:embedContent` across 35 channels (https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md). new: Respan serves OpenAI chat, OpenAI Responses, and Anthropic-native surfaces over 1000+ models (https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart). new: F5's client schema is OpenAI-only (a chat-completions subset plus /v1/models) with Anthropic upstream-executor only (https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html). new: Martian's hosted gateway (live as of 2026-10-27) fronts 312 models from 49 providers at `/v1/chat/completions`, `/v1/responses`, and `/v1/messages` (https://api.withmartian.com/v1/models, https://docs.withmartian.com/api-reference/endpoints); Not Diamond remains a routing-decision API, not a proxy (https://docs.notdiamond.ai/reference/token_model_select_v2_modelrouter_modelselect_post.md). Deliberately narrow: Langfuse's Rust ai-gateway relays only OpenAI + Anthropic, byte-preserving (https://github.com/langfuse/langfuse/tree/main/ai-gateway). + +Translation fidelity is only partially comparable from public docs. llm-proxy normalizes 10 content kinds (`docs/protocol.md:80-91`) and records normalized usage including cache/reasoning token splits (`crates/llm-proxy-storage/src/sqlite.rs:116-121`). agentgateway normalizes cached-prompt-token accounting across providers (Anthropic/Bedrock exclude, OpenAI/Azure/Gemini include them; https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/rate-limits.md) and reports both normalized and provider-raw token counts for billing parity (https://agentgateway.dev/docs/standalone/latest/documentation/llm/observability.md). TensorZero normalizes structured outputs to native constrained decoding or forced tool calls (https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/generate-structured-outputs.mdx). Per-field coverage of provider-specific options (reasoning controls, tool-choice variants, response formats) was not audited for any product here. + +## Client BYOK, data residency, saturation, and SDKs + +new: Axes added in this revision that a gateway comparison expects. + +- Client-supplied upstream keys (BYOK passthrough): Cloudflare accepts + provider keys per request with aliased Secrets-Store selection + (https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/); + Vercel takes ordered per-provider credentials via + `providerOptions.gateway.byok`, falling back to system credentials + (https://vercel.com/docs/ai-gateway/authentication-and-byok/byok); + OpenRouter attaches prioritized or fallback BYOK credentials with + per-key model/user filters (https://openrouter.ai/docs/guides/overview/auth/byok.md); + Agent Router injects per-request credentials from ext-authz metadata or + the `x-aigw-api-key` header (https://theagentrouter.ai/docs/capabilities/security/upstream-auth); + Bifrost accepts direct keys via `x-bf-direct-key` behind an opt-in + (https://docs.getbifrost.ai/features/keys-management.md); the Portkey OSS + gateway takes provider keys per request by design + (https://github.com/Portkey-AI/gateway). llm-proxy: opt-in BYOK + passthrough exists — `passthrough_auth: bool` per provider + (`crates/llm-proxy-core/src/provider_config.rs:378-385`), the inbound + client's bearer/x-api-key token used as the upstream credential at + `crates/llm-proxy-server/src/routes/core_pipeline.rs:727-730` (passed to + `resolve_target` at `:753-759`); a missing client token fails 401 + "missing client auth token for passthrough provider" + (`crates/llm-proxy-server/src/routes/error_response.rs:54`); shipped + 2026-06-20 (commit 201e29e) (corrected 2026-10-28 after code audit). +- Data residency/compliance posture: Vercel routes only to providers with + signed ZDR agreements on request and pins inference regions + (https://vercel.com/docs/ai-gateway/security-and-compliance); OpenRouter + offers eu/us regional endpoints and per-key `allowed_data_regions` + guardrails (https://openrouter.ai/docs/guides/features/in-region-routing.md); + LiteLLM filters fallbacks by `region_name` + (https://docs.litellm.ai/docs/proxy/reliability). llm-proxy has no + data-posture-aware routing. +- Saturation behavior: LiteLLM fails over-capacity requests fast with 429 + instead of queueing (https://docs.litellm.ai/docs/routing); uni-api + admits or queues requests using cgroup memory awareness + (`RESOURCE_ADMISSION_MODE`, https://raw.githubusercontent.com/yym68686/uni-api/main/README.md); + the vLLM Production Stack routes by request priority header + (https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/routers/routing_logic.py); + Helicone documents no queueing on rate limits + (https://docs.helicone.ai/features/advanced-usage/custom-rate-limits.md). + llm-proxy over-limit requests get 429 + (`crates/llm-proxy-server/src/middleware.rs:136`); no queueing or + prioritization exists. +- Client SDK coverage: Vercel ships a native AI SDK integration + (https://vercel.com/docs/ai-gateway/sdks-and-apis); OpenRouter generates + clients in three languages (https://openrouter.ai/docs/_llms/api-reference.txt); + the Portkey SDK is a superset of the OpenAI SDK + (https://portkey.ai/docs/product/ai-gateway/universal-api); Bifrost + documents zero-code SDK drop-ins for six SDKs + (https://raw.githubusercontent.com/maximhq/bifrost/main/README.md). + llm-proxy targets stock OpenAI/Anthropic SDKs via base-URL swap; no + dedicated SDK. + +## Upstream cloud authentication + +new: llm-proxy authenticates upstreams with a single static bearer `api_key` +(`SecretString`, `crates/llm-proxy-core/src/provider_config.rs:374-385`); a +wave-2 workspace grep for `sigv4|bedrock|vertex|azure|entra|workload-identity` +matched only docs/research design notes and zero Rust/TOML files, so AWS +SigV4, Azure Entra, and GCP ADC upstreams are unreachable today. + +The converged competitor pattern is a per-provider auth object separating credential acquisition (ambient cloud identity chains vs explicit secrets) from request attachment (bearer header vs SigV4 signature), plus an expiry-driven credential cache. agentgateway — Rust, like llm-proxy — builds all of it in-process on official SDK crates (aws-config/aws-sigv4, azure_identity, google-cloud-auth; verified in its Cargo.toml): per-backend mutually-exclusive auth methods (static/aws/azure/gcp/copilot/jwtSign/oauthTokenExchange), SigV4 computed last after every other request mutation, AssumeRole with per-request CEL session tags cached and refreshed before expiry, Azure Entra via the full DefaultAzureCredential chain with per-policy credential-source caches, and GCP ADC access/id tokens incl. impersonated and GDCH service accounts (https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/backend-authn/, https://raw.githubusercontent.com/agentgateway/agentgateway/main/crates/agentgateway/Cargo.toml). LiteLLM: boto3 credential chains, SigV4 over a filtered header set so forwarded client headers cannot break the canonical request, a DualCache IAM policy (static ~59 min, ambient 600s, STS TTL minus 60s; AssumeRole deliberately uncached), and a Vertex token manager with single-flight proactive refresh (FRESH/STALE/INVALID) plus one re-auth retry on 401 — source-verified at `ref/litellm/litellm/llms/bedrock/base_aws_llm.py:74-148,1400-1439` and `ref/litellm/litellm/llms/vertex_ai/vertex_llm_base.py:480-603` (https://docs.litellm.ai/docs/providers/bedrock, https://docs.litellm.ai/docs/providers/azure, https://docs.litellm.ai/docs/providers/vertex). Kong declares per-type provider auth (aws assume-role incl. batch_role_arn, azure managed identity/client secret, gcp service-account JSON/Workload Identity Federation) on its AI Model Provider entity (https://developer.konghq.com/ai-gateway/entities/ai-model-provider/). Agent Router's BackendSecurityPolicy CRD covers APIKey/AWSCredentials/AzureCredentials/GCPCredentials with controller-side OIDC token exchange writing rotated secrets at expiry minus a 5-minute window, and its extproc SigV4 self-corrects the signing region from the resolved Bedrock host (https://theagentrouter.ai/docs/capabilities/security/upstream-auth, source-verified https://raw.githubusercontent.com/theagentrouter/agent-router/main/internal/backendauth/aws.go). Azure APIM authenticates to Azure AI via managed identities (https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities); AWS AgentCore vaults outbound credentials per target (token-vault ARNs, OAuth flows, SigV4 via the gateway role; https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-outbound-auth.html). +new: Upstream mTLS client certificates exist only at agentgateway (file cert/key/root in its backendTLS policy; SPIFFE SVID rotation is Enterprise) — LiteLLM, Kong, Portkey, and Agent Router document none (https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/backend-tls/). + +## Prompt caching and cache locality + +new: llm-proxy's protocol carries a cache marker — `CoreContent::Text` +holds `cache: Option` — but `CacheControl` has only a +`type` field (`Ephemeral`/`Other`) with no TTL, so Anthropic's 1-hour +lifetime is unrepresentable, and no adapter translates markers, injects +them, or routes for cache locality +(`crates/llm-proxy-protocol/src/core.rs:142,261-286`, verified 2026-10-27). +Cache read/write usage columns already land in the ledger +(`crates/llm-proxy-storage/src/sqlite.rs:116-121`). + +Competitors treat prompt caching as a routing and translation concern. OpenRouter translates block markers bidirectionally (Anthropic `cache_control` ↔ OpenAI `prompt_cache_breakpoint`, TTLs dropped in translation), auto-applies a top-level `cache_control` to the last cacheable block, sticky-routes follow-ups to the cache-holding provider for 10 minutes only when cache-read price beats prompt price, and derives sticky keys from `session_id`/`x-session-id`/`prompt_cache_key`/a message hash (https://openrouter.ai/docs/features/prompt-caching). LiteLLM injects markers at configured points (role/index/`tool_config`, mapped to Bedrock `cachePoint` and OpenAI `prompt_cache_breakpoint`), caps injection at Anthropic's 4 blocks (v1.84.9), ships a one-flag Anthropic auto-caching mode (v1.94.0+), translates Anthropic markers to Gemini `cachedContents`, and — closest to a routing feature — pins deployments by SHA-256 of the cacheable prefix plus tools for 300s in its `prompt_caching` pre-call check (https://docs.litellm.ai/docs/tutorials/prompt_caching, https://docs.litellm.ai/docs/completion/prompt_caching, source-verified `ref/litellm/litellm/router_utils/prompt_caching_cache.py:56-197`). Vercel's `caching: 'auto'` classifies providers into explicit-marker vs implicit and places two markers (last message; message before the last user message, system fallback) plus advisory `cache_anchor_items`/`cache_ttl` that degrade with warnings instead of errors; its session affinity pins provider and credential with a pin lifetime matched per provider (5m Anthropic/Bedrock/Google, 10m OpenAI/Azure, 1h DeepSeek, minus 15s, capped 1h), a session-ID hierarchy ending in a conversation fingerprint, minimum-prompt-size gating, and a reported affinity outcome per request (https://vercel.com/docs/ai-gateway/models-and-providers/automatic-caching, https://vercel.com/docs/ai-gateway/models-and-providers/session-affinity). Bifrost pins session IDs (incl. auto-adopted Claude Code/Codex/OpenCode headers) to the last-serving provider+key for 1h (https://docs.getbifrost.ai/providers/session-affinity.md). GPT-Load derives soft affinity from AccessKey/protocol/first-input prefix with LRU capacity 10,000 (https://www.gpt-load.com/docs/internals/scheduling); new-api tracks per-model channel affinity (https://github.com/QuantumNous/new-api). Helicone passes Anthropic `cache_control` through and documents the pricing multipliers (https://docs.helicone.ai/gateway/concepts/prompt-caching). + +## Model lifecycle metadata + +new: llm-proxy discovery discards lifecycle data: `parse_models` keeps only +id/display_name/context_length (`crates/llm-proxy-provider/src/discovery.rs:272-288`, +verified 2026-10-27), and `/v1/models` hardcodes `created: 0` with null +`created_at` (`crates/llm-proxy-server/src/routes/models.rs:24-36`). Nothing +warns or blocks on a deprecated model. + +Providers publish machine-readable lifecycle fields: OpenAI's Model object carries optional `shutdown_date` (notice: ≥6 months GA, 3 months specialized, ~2 weeks preview; https://developers.openai.com/api/docs/api-reference/models, https://developers.openai.com/api/docs/deprecations); Anthropic ships a tri-state `lifecycle` (active/deprecated/retired) plus `deprecated_at`/`retires_at`, keeping retired models listed while inference fails (https://platform.claude.com/docs/en/api/models-list); Bedrock's `modelLifecycle` (ACTIVE/LEGACY + timestamps) drops EOL models from listings, with a 6-month or 45-day Legacy window (https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html, https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html); the Gemini Model resource has no lifecycle fields (https://ai.google.dev/api/models). +Gateways consuming it: OpenRouter's catalog carries `expiration_date` (live check: 14 of 464 entries dated, expired entries delisted) plus `canonical_slug` permanent aliases and a provider-feed deprecation pipeline (https://openrouter.ai/docs/guides/overview/models, https://openrouter.ai/docs/guides/providers/integration); LiteLLM's cost map carries `deprecation_date` on 874 entries, `/model/deprecations` buckets configured models by urgency, and Slack alerts fire on a daily lock — but LiteLLM never blocks or rewrites (https://github.com/BerriAI/litellm/blob/main/litellm/proxy/common_utils/model_deprecation.py, https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json). new: Of the gateways cited in this section, none emits a request-time signal (sunset-style header, response field, or staged rejection) from these dates — LiteLLM stops at dashboards and alerts, and OpenRouter delists; no successor-model mapping ships anywhere cited here. + +## Admin and tooling + +llm-proxy ships a CLI (serve/stop/status/validate/init/models/autostart, +`apps/llm-proxy/src/cli.rs:23`), a read-only usage admin API +(`crates/llm-proxy-api/src/routes.rs:53-55, 116-117`), musl single-binary releases, +and per-provider egress forward proxying +(`crates/llm-proxy-provider/src/transport.rs:293`). + +Competitor admin surfaces: LiteLLM mutates gateway behavior at runtime via a DB-backed control plane and ships a Next.js admin UI (https://docs.litellm.ai/docs/proxy/virtual_keys); gproxy embeds a web console for providers, routes, users, and usage (https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md); Bifrost serves a web UI with live log streaming (https://docs.getbifrost.ai/features/observability/default); agentgateway has a UI with key management and a CEL playground plus the experimental agctl CLI (https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/costs.md); new-api/one-api ship multi-user consoles with payments (https://github.com/QuantumNous/new-api); Vercel exposes CLI + REST + MCP control-plane tools (https://vercel.com/docs/ai-gateway/sdks-and-apis). new: Second wave adds the Higress console (https://higress.cn/en/docs/latest/user/quickstart/), Tyk's AI Studio console with edge microgateways (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/edge-gateways.md), IBM ContextForge's admin UI with LLM provider CRUD and model testing (https://ibm.github.io/mcp-context-forge/latest/api/llm-proxy/), GPT-Load's embedded health/log/route-check/usage UI (https://www.gpt-load.com/docs/monitor), and Respan's dashboard (https://www.respan.ai/docs/). +Prompt management is a control-plane feature llm-proxy lacks entirely: TensorZero binds prompts as config variants (https://github.com/tensorzero/tensorzero/blob/main/docs/experimentation/run-adaptive-ab-tests.mdx), Langfuse serves versioned prompts with stale-while-revalidate client caching (https://langfuse.com/docs/prompt-management/features/caching.md), Portkey versions prompts immutably and A/B-tests them through loadbalance configs (https://portkey.ai/docs/product/prompt-engineering-studio/prompt-versioning). new: Admin audit trails ship at LiteLLM (entity lifecycle audit with before/after values and S3 export, https://docs.litellm.ai/docs/proxy/audit_logs), Kong (RSA-signed admin audit logs with TTL purge, https://developer.konghq.com/gateway/audit-logs), Cloudflare (gateway config-change audit on all plans, https://developers.cloudflare.com/ai-gateway/reference/audit-logs/), Portkey (audit logs API, https://portkey.ai/docs/product/enterprise-offering/audit-logs), and new-api (security audit log with step-up verification, https://github.com/QuantumNous/new-api/releases). +new: Extensibility/hook systems (second wave): LiteLLM exposes in-process Python CustomLogger callbacks plus full request-lifecycle hooks (modify/reject/stream/header) and custom auth/key-policy hooks (https://docs.litellm.ai/docs/proxy/call_hooks, https://docs.litellm.ai/docs/observability/custom_callback); Bifrost loads Go `.so` plugins at runtime with hot reload across the full request lifecycle incl. MCP hooks (https://docs.getbifrost.ai/plugins); Kong ships Lua/PDK plugins, serverless Lua snippets, and external runners (https://developer.konghq.com/custom-plugins/reference, https://developer.konghq.com/plugins/pre-function/); Higress and APISIX run proxy-wasm plugins in Go/Rust/C++ (https://github.com/alibaba/higress/tree/main/plugins, https://apisix.apache.org/docs/apisix/plugins/ai-proxy/); Portkey compiles TypeScript plugin manifests into the gateway (https://github.com/Portkey-AI/gateway/blob/main/plugins/README.md); Tyk unifies gateway/UI/analytics plugins behind one gRPC-isolated SDK (https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/plugins-overview.md). Hook names, registration, and lifecycle are bespoke in each system cited here — no common API emerged among them — and Agent Router ships no custom extension point at all (its ext_proc request was closed stale; https://theagentrouter.ai/). +Config lifecycle: llm-proxy rejects `server.hot_reload=true` at startup (`crates/llm-proxy-core/src/provider_config.rs:1531-1538`). new: agentgateway hot-reloads everything except an explicit startup-only list (adminAddr/storage/database/logging/tracing) via a debounced file watch (re-registering on atomic replace) with validate-then-swap, last-known-good fallback, a `config_synchronized` metric, `--validate-only`, and file/hybrid/readOnly storage modes (https://agentgateway.dev/docs/standalone/latest/documentation/setup/update, https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/state_manager.rs). new: uni-api (now Rust) compiles validated snapshots atomically and swaps an all-`Arc` immutable snapshot under one write, with revision-CAS admin overlays (https://github.com/yym68686/uni-api/blob/main/src/runtime/state.rs). LiteLLM converges pods by 30s DB polling with YAML-bootstrap-then-DB-overlay precedence and incremental router updates whose errors are caught and swallowed — source-verified at `ref/litellm/litellm/proxy/proxy_server.py:5027-5099` (https://docs.litellm.ai/docs/proxy/configs, https://docs.litellm.ai/docs/proxy/prod); Higress pushes runtime config over xDS in milliseconds (https://github.com/higress-group/higress/blob/main/docs/architecture.md); new: Kong data planes receive configuration dynamically from the Konnect control plane (https://developer.konghq.com/ai-gateway/); new: APISIX applies Admin-API changes to running nodes dynamically from etcd (https://apisix.apache.org/docs/apisix/admin-api/). oc-go-cc reloads via fsnotify+SIGHUP (`ref/oc-go-cc/internal/config/watcher.go:19-101`) and openproxy adds SIGUSR2 zero-downtime upgrade (https://raw.githubusercontent.com/x5iu/openproxy/master/README.md). new: No product cited here ships versioned config with in-gateway history/diff/rollback — Kong's decK (https://raw.githubusercontent.com/Kong/deck/main/README.md) is the closest, keeping history in git. On Rust performance: LiteLLM is migrating translation to a bundled Rust core (beta, off by default; https://docs.litellm.ai/docs/proxy/rust_gateway), Portkey's self-hosted gateway runs a Rust front-end process from v2.30.0 (https://portkey.ai/docs/self-hosting/major-performance-improvements.md), and TensorZero's Rust gateway claimed 0.94 ms p99 overhead at 10k QPS — self-published (https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/benchmarks.mdx) and archived; llm-proxy's own overhead was not measured for this report, so no performance comparison is asserted. + +## Transport, state, and deployment topology + +llm-proxy's listener is plain HTTP: TLS exists only on the upstream client +(reqwest rustls-tls, `crates/llm-proxy-provider/Cargo.toml:23`), and a grep +for server-side TLS (rustls/axum-server/certificate) across workspace +Cargo.toml files, run 2026-10-14, returns no listener TLS dependency; HTTPS +fronting is delegated to a reverse proxy. openproxy shows the in-proxy +alternative (rustls TLS 1.3 listener, +https://raw.githubusercontent.com/x5iu/openproxy/master/README.md). + +All llm-proxy state is single-node: the usage ledger is a local SQLite file +(`config.toml.example:56-68`), and the rate limiter, dedup window, metrics, +and catalog refresh are in-process — nothing shares state across replicas, +so N instances mean N independent limits and N ledgers. LiteLLM keeps +limits, cooldowns, and spend in Redis/Postgres for multi-proxy deployments +(https://docs.litellm.ai/docs/proxy/load_balancing); gproxy shares +rate-limit counters via a cache backend +(https://gproxy.leenhawk.com/guides/permissions/); Bifrost's OSS governance +is explicitly single-node ("running multiple OSS Bifrost nodes with a +Postgres backend is not supported", +https://docs.getbifrost.ai/features/governance/budget-and-limits) — the same +shape llm-proxy has; new-api's per-node Redis limits count independently +(https://github.com/QuantumNous/new-api). new: Second-wave notes — AWS +AgentCore reaches VPC-internal targets and identity providers through VPC +Lattice private endpoints (https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-vpc-egress.html); Cloudflare supports custom gateway domains and a custom-provider registry for arbitrary HTTPS providers (https://developers.cloudflare.com/ai-gateway/configuration/custom-domains/, https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/); Kong 2.2.0 ships FIPS 140-3 and distroless variants (https://developer.konghq.com/ai-gateway/changelog/). + +## What llm-proxy lacks + +Ranked by impact on a self-hosted proxy serving coding-agent workloads — request-path reliability and spend/rate control first, ecosystem interop second, operator convenience last. Prevalence is the count of surveyed products named per item across both waves; every named product is sourced in the section cited for that item or inline in the entry. Items marked `new:` were first identified in the second wave. + +1. **Request-path retries, fallback chains, and provider cooldowns** — 22 products (LiteLLM, Portkey, Bifrost, Cloudflare, Agent Router, Higress, Kong, OpenRouter, Vercel, TensorZero, one-api, new-api, gproxy, oc-go-cc, agentgateway, AWS AgentCore, Tyk, F5, uni-api, GPT-Load, Respan, llm-api-key-proxy; sources in Routing and resilience). The llm-proxy request path has none; retry/fallback is scoped future work (`docs/research/core-protocol.md:170-186`). +2. **Per-key spend budgets with enforcement and periodic resets** — 15 ship USD budgets (LiteLLM, Bifrost, OpenRouter, Vercel, Cloudflare, LangSmith, gproxy, BricksLLM, agentgateway, Portkey, Tyk, Respan, GPT-Load, uni-api, Traefik Hub; sources in Cost tracking); one-api/new-api/Azure APIM enforce token-quota budgets instead (sources in Cost tracking and Rate limiting). llm-proxy pricing is estimation-only with no enforcement (`crates/llm-proxy-core/src/provider_config.rs:472`). +3. **Identity-scoped rate limits: per-key RPM/TPM and concurrency caps** — 14 products (LiteLLM, Kong, Agent Router, Higress, Bifrost, gproxy, new-api, agentgateway, AWS AgentCore, Traefik Hub, Respan, GPT-Load, uni-api, TensorZero; sources in Rate limiting and quotas). Per-key RPM exists for authenticated callers (token bucket keyed by key id, `crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`, since 2026-10-13); TPM limits and concurrency caps do not (corrected 2026-10-28 after code audit). +4. **new: Cloud-native upstream authentication (AWS SigV4/STS, Azure Entra, GCP ADC/workload identity)** — 6 products (agentgateway, LiteLLM, Kong, Agent Router, Azure APIM, AWS AgentCore; sources in Upstream cloud authentication). llm-proxy supports only static bearer keys (`crates/llm-proxy-core/src/provider_config.rs:374-385`). +5. **Upstream credential pools with rotation, cooldowns, and quota windows** — 8 products (llm-api-key-proxy `ref/llm-api-key-proxy/src/rotator_library/config/defaults.py:139-158`, gproxy, Bifrost, Higress, new-api — sources in Routing and resilience — plus openproxy weighted upstream key pools https://raw.githubusercontent.com/x5iu/openproxy/master/README.md, GPT-Load credential cooldown/blacklist isolation https://www.gpt-load.com/docs/internals/scheduling, and uni-api key rotation with per-key cooldown knobs https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). llm-proxy takes one static key per provider (`crates/llm-proxy-core/src/provider_config.rs:375`). +6. **OpenAI Responses client endpoint (`/v1/responses`)** — 15 products (LiteLLM, Portkey OSS, OpenRouter, Vercel, LangSmith, Helicone, new-api, Agent Router, agentgateway, AWS AgentCore, Cloudflare, Respan, uni-api, GPT-Load, Martian; sources in Protocol breadth; Tyk ships one only in its unreleased 2.3.0 line). The upstream Responses adapter exists; the client surface does not (`crates/llm-proxy-server/src/routes/mod.rs:69-79`) (corrected 2026-10-28 after code audit). +7. **Guardrails: PII redaction, moderation, prompt-injection detection** — 17 products (LiteLLM, Portkey, Cloudflare, Kong, Bifrost, LangSmith, Helicone, new-api, Higress, Azure, AWS, Tyk, F5, agentgateway, GPT-Load, uni-api, Respan; sources in Guardrails). llm-proxy ships none. +8. **MCP gateway (client, server, or federation)** — 11 products (LiteLLM, Cloudflare, agentgateway, Bifrost, Kong, Azure APIM, AWS AgentCore, IBM ContextForge, Higress, Glama, Tyk; sources in MCP). llm-proxy has no MCP functionality. +9. **Key lifecycle API (create/list/update/rotate/revoke) without restart** — 12 products (LiteLLM, Bifrost, OpenRouter, Vercel, rust-llm-proxy, one-api, Portkey, agentgateway, GPT-Load, Respan, Tyk, uni-api; sources in Auth and keys). llm-proxy keys are static config (`config.toml.example:42-54`). +10. **Exact-match response caching** — 8 products ship exact-match (LiteLLM, Portkey, Cloudflare, Helicone, OpenRouter, Higress, Respan, Tyk); semantic variants at Bifrost, Kong, Azure APIM, LiteLLM's Qdrant backend, and the vLLM stack's alpha (sources in Caching). llm-proxy has no cache. +11. **OTel trace export under GenAI semantic conventions** — 9 products (Cloudflare, Vercel, OpenRouter, agentgateway, LiteLLM, Bifrost, Tyk, IBM ContextForge, vLLM Production Stack; sources in Observability). llm-proxy tracing exports no OTLP spans. +12. **Prometheus `/metrics` endpoint** — 9 products (LiteLLM, agentgateway, Bifrost, Kong, TensorZero, vLLM Production Stack, Higress, Tyk, IBM ContextForge; sources in Observability). llm-proxy counters are in-process only (`crates/llm-proxy-core/src/metrics.rs`). +13. **Weighted/multi-strategy load balancing per model group** — 12 products (LiteLLM, Portkey, Kong, gproxy, Bifrost, OpenRouter, Cloudflare Auto Router, Higress, uni-api, GPT-Load, AWS AgentCore, Tyk; sources in Routing and resilience, with Cloudflare and Higress sourced inline in the first edition's gap 11 at https://developers.cloudflare.com/ai-gateway/features/auto-router/ and https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-load-balancer/README_EN.md). An llm-proxy alias resolves to exactly one provider (`crates/llm-proxy-core/src/provider_config.rs:392-394`). +14. **new: Cross-provider prompt-cache plumbing and cache-locality routing** — 7 products (OpenRouter, LiteLLM, Vercel, Helicone, Bifrost, GPT-Load, new-api; sources in Prompt caching and cache locality). llm-proxy's `CacheControl` cannot express TTLs and nothing translates or pins (`crates/llm-proxy-protocol/src/core.rs:261-286`). +15. **new: Upstream credential encryption at rest / secret-manager references** — 8 products (gproxy, LiteLLM, Kong, Tyk, IBM ContextForge, new-api, GPT-Load, Portkey; sources in Auth and keys). llm-proxy stores keys in plaintext TOML. +16. **new: Config hot reload without restart** — 8 products (agentgateway, uni-api, Higress, Kong, APISIX, LiteLLM, oc-go-cc, openproxy; sources in Admin and tooling). llm-proxy rejects `server.hot_reload` (`crates/llm-proxy-core/src/provider_config.rs:1531-1538`). +17. **Latency histograms and TTFT metrics** — 9 products (LiteLLM, agentgateway, Bifrost, oc-go-cc, vLLM Production Stack, Kong, Tyk, Helicone, AWS AgentCore; sources in Observability). llm-proxy computes latency percentiles in-process (`crates/llm-proxy-core/src/metrics.rs:210-239`) but never exports them (corrected 2026-10-28 after code audit). +18. **new: Model lifecycle metadata (deprecation dates, successor aliases) surfaced to clients** — 2 gateways consume it (LiteLLM, OpenRouter) and three providers publish machine-readable fields (OpenAI, Anthropic, Bedrock; sources in Model lifecycle metadata). llm-proxy discovery discards it (`crates/llm-proxy-provider/src/discovery.rs:272-288`). +19. **Web admin UI** — 11 products (LiteLLM, gproxy, Bifrost, agentgateway, new-api, one-api, Higress, Tyk, IBM ContextForge, GPT-Load, Respan; sources in Admin and tooling). llm-proxy admin is read-only JSON (`crates/llm-proxy-api/src/routes.rs:53-55, 116-117`). +20. **new: Stream-cancellation usage settlement** — 2 products document billing/accounting semantics for cancelled streams (LiteLLM `cancel_on_disconnect` + per-chunk usage, https://docs.litellm.ai/docs/proxy/config_settings; OpenRouter per-provider cancel-billing semantics + final usage chunk, https://openrouter.ai/docs/api_reference/streaming). llm-proxy persists a `ClientCancelled` failure event with no usage payload when a client disconnects — tokens delivered before cancellation are never settled into the ledger (`crates/llm-proxy-server/src/routes/core_pipeline.rs:952-985`, `crates/llm-proxy-storage/src/lib.rs:80`). SSE keep-alive is already parity (`crates/llm-proxy-server/src/routes/core_pipeline.rs:701,883`). + +Already-scoped future work (docs/research/endpoints.md) — embeddings, rerank, +realtime, transcription/TTS, image/video generation, files, OCR, batch, +fine-tuning, search bridges — is not re-listed above. Axes surveyed in +"Client BYOK, data residency, saturation, and SDKs" are comparison context, +not ranked gaps: llm-proxy ships opt-in BYOK passthrough (`passthrough_auth`, +since 2026-06-20) but lacks data-posture routing, queueing/prioritization, +and a dedicated SDK there, each sourced in that section (corrected +2026-10-28 after code audit). + +## Near-term candidates + +Each candidate names its code touchpoint and a done-condition. Gaps without a candidate below (OTel traces, MCP, key lifecycle, guardrails, load-balancing strategies, the Responses client endpoint, admin UI, cloud-native upstream auth, hot reload, credential encryption at rest, stream-cancellation usage settlement) are deferred: each needs a dependency, config/schema, or control-plane decision the candidates avoid. + +- Pre-first-event retry/fallback: the streaming path already awaits the + first `CoreEvent` via `FirstByteResult` before SSE headers commit + (`crates/llm-proxy-server/src/routes/core_pipeline.rs:702-705`); after the + first event, keep the in-band error invariant + (`docs/protocol.md:139-151`). Done when a 429/5xx/connect error from the + primary advances to the configured fallback in that window and the ledger + records both attempts — this requires extending the usage-row schema with + an attempt ordinal and provider per upstream attempt (or an attempts-count + column), a storage migration scoped into this candidate + (`crates/llm-proxy-storage/src/sqlite.rs:105-131`). +- Client-key budgets, check-then-accrue: reject when the in-memory counter + is over budget, accrue actual cost after completion against the existing + SQLite ledger (`crates/llm-proxy-storage/src/sqlite.rs:105`). Reset + windows are a rolling duration (optionally calendar-aligned later — the + two resets competitors converge on, Cost tracking above); accrued spend + must be rehydrated from the ledger on startup so a restart does not zero + an in-flight window, and the window start persisted with the key record. + Deliberately not reservation-based: bounded overshoot from in-flight + calls is accepted, and LiteLLM's cost reservation silently skips when + cost is unestimatable (https://github.com/BerriAI/litellm/issues/35524); + Bifrost's lazy window reset needs no scheduler + (https://github.com/maximhq/bifrost/blob/dev/plugins/governance/store.go). + Done when an over-budget key gets a defined status (402) pre-dispatch, + settled cost lands in the ledger, and accrued spend survives a restart. +- TPM (per-key RPM already exists, keyed by CallerId at + `crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`): reuse the + offline token counter (`crates/llm-proxy-core/src/token/counter.rs:282`) + for reserve-then-reconcile TPM — reservation suits tokens because + estimates are cheap and refundable (corrected 2026-10-28 after code + audit). Done when over-limit requests get 429 with retry-after and + rate-limit headers, reconciled to provider-reported usage after + completion. +- Prometheus `/metrics` with latency/TTFT histograms: hand-rendered text + exposition walking the metrics snapshot + (`crates/llm-proxy-core/src/metrics.rs`), served on the admin listener. + Done when the endpoint scrapes counters plus a latency histogram and TTFT + for streams. +- Exact-match response cache keyed on a canonical `CoreRequest` + serialization, in-memory v1 with a bounded TTL (floor/default/max — the + 60s-floor convention Cloudflare and Portkey use, Caching above), an entry + cap, and LRU eviction under the cap. Done when a second identical request + is served from the store with a HIT header and zero recorded usage, + entries expire at the TTL and evict under the cap, and the dedup window + still rejects concurrent identical misses before origin + (`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`) — + coalescing (serving the second caller) remains unbuilt. +- Credential pools: optional `[[provider.credentials]]` tables extending the + single `api_key` field (`crates/llm-proxy-core/src/provider_config.rs:375`) + with per-credential cooldown state and quota windows persisted via the + existing storage layer. Done when a credential that 401s is locked out, + its cooldown survives restart, and the credential id appears in ledger + rows. +- new: Prompt-cache TTL and marker passthrough: add a `ttl` field to + `CacheControl` (`crates/llm-proxy-protocol/src/core.rs:261-266`) and pass + markers through both client adapters so Anthropic 5m/1h semantics survive + a round trip; usage columns for cache read/write already exist + (`crates/llm-proxy-storage/src/sqlite.rs:116-121`). Done when a request + carrying a 1h marker on `/v1/messages` produces the same marker upstream + and the ledger records cache-write tokens. +- new: Model lifecycle capture: extend `StaticModelCatalogEntry` + (`crates/llm-proxy-core/src/provider_config.rs:735-747`, currently + `deny_unknown_fields`, so new fields need `#[serde(default)]`) and + discovery's `parse_models` to capture per-kind lifecycle fields + (OpenAI `shutdown_date`, Anthropic `lifecycle`/`retires_at`, Bedrock-style + `modelLifecycle`), surfaced per client shape in `/v1/models` + (`crates/llm-proxy-server/src/routes/models.rs:24-36`) with retired + entries kept listed. Done when both client protocols list deprecation + dates and a warn-level log fires on deprecated-model use; blocking stays + opt-in and out of scope. diff --git a/docs/research/competitive-landscape-compendium.md b/docs/research/competitive-landscape-compendium.md new file mode 100644 index 0000000..106aadd --- /dev/null +++ b/docs/research/competitive-landscape-compendium.md @@ -0,0 +1,995 @@ +# Competitive landscape compendium — llm-proxy research record (October 2026) + +Consolidated record of the two-wave competitive research summarized in +[`competitive-gap-analysis.md`](competitive-gap-analysis.md), reorganized +for deciding what to build next. Competitor claims and their URLs originate +in the audited analysis (competitive-gap-analysis.md) and the research +runs' recorded findings; task counts and severity tiers come from the runs' +final results; llm-proxy-side claims were re-verified against the working +tree (code audit, 2026-10-28). + +## Scope and method + +- Two research waves, October 2026: 57 research tasks (product scouts, theme + studies, doc sweeps, and 10 deep dives — 6 from wave 1, 4 from wave 2) + plus 10 frontier analyses — 67 inventory items total, all accounted for + in the coverage appendix. Wave-1 grep checks ran 2026-10-14; wave-2 code + claims (hot_reload rejection, `CacheControl` shape, discovery field + capture, `ModelCard`, SSE keep-alive, cancellation handling, body limit) + were re-verified on 2026-10-27. +- Third wave, November 2026: niche-repository hunts over the + Rust/Go/TypeScript/Python long tail produced a deduplicated find list of + 121 candidate repositories, of which 23 were researched in full — 90 + inventory items across the three waves. Wave-3 records live modularly in + [`niche-repos.md`](niche-repos.md) with the machine-readable registry + [`repos-registry.json`](repos-registry.json) covering all three waves; + the most significant wave-3 finds are summarized in the market map below + and all 23 are accounted for in the coverage appendix. +- Competitor claims from official docs, release notes, and changelogs + current to October 2026, plus vendored source pins read directly: + `ref/litellm` at b84f7f82, `ref/oc-go-cc` at 99d756c, `ref/llm-api-key-proxy` + at af40b91. `ref/REFS` also pins `ref/llm-proxy` at fb588f6 (2024-05-24), + uncited by the analysis but researched directly for this compendium on + 2026-10-28 (Reference repositories); a 2026-10-28 follow-up likewise + researched kgateway from its official docs. No competitor code was built + or run. +- llm-proxy claims verified against this working tree, cited `file:line`; + wave-2 additions are tagged `new:` where the distinction matters. + +## Market map + +One factual line per researched product with its source URL; status notes +are from the analysis's survey header. kgateway and the `ref/llm-proxy` pin +were researched directly on 2026-10-28 (Scope and method). + +### Open-source gateways + +| Product | Line | Source | +|---------|------|--------| +| LiteLLM | Python gateway, DB-backed control plane, virtual keys, six routing strategies, one of several breadth leaders | https://docs.litellm.ai/release_notes | +| Portkey OSS gateway | Node gateway, dormant since 2026-05-25 after the Palo Alto Networks acquisition; enterprise line (rebranded Prisma AIRS AI Gateway) continues | https://github.com/Portkey-AI/gateway/releases, https://portkey.ai/docs/product/ai-gateway/fallbacks | +| Kong AI Gateway | `ai-proxy-advanced`: seven load-balancing algorithms, circuit breaker; FIPS 140-3 and distroless variants since 2.2.0 | https://developer.konghq.com/plugins/ai-proxy-advanced/, https://developer.konghq.com/ai-gateway/changelog/ | +| agentgateway | Rust/tokio, split out of kgateway; virtual keys, CEL rate limits, cloud upstream auth, OTel GenAI metrics, MCP federation; 23 native providers incl. Bedrock, Vertex, GitHub Copilot; A2A proxying and REST-as-tools; MCP OAuth2 resource-server auth with CEL-based MCP authorization; promptCaching policy auto-inserting provider prompt-cache markers | https://agentgateway.dev/docs/standalone/latest/documentation/llm/about.md, https://agentgateway.dev/docs/standalone/latest/documentation/about/introduction.md, https://agentgateway.dev/docs/standalone/latest/documentation/mcp/about.md, https://agentgateway.dev/docs/standalone/latest/documentation/configuration/traffic-management/llm.md | +| kgateway | Solo.io's Envoy-based Gateway API control plane (ex-Gloo, CNCF sandbox, very active; v2.5.0-rc.1 on 2026-10-27). Its Envoy-based AI Gateway was deprecated in v2.1.x and removed in v2.2.0 — the AI line became agentgateway (Rust, Linux Foundation 2025, Agentic AI Foundation 2026). What remains: Gateway API edge features — JWT/OAuth2/OIDC + ext-proc auth, local + global rate limiting, retries with per-try timeouts and outlier detection, priority failover, transformations, mirroring. Absent now: any in-tree AI gateway, LLM cost tracking, prompt guardrails, token rate limiting, MCP | https://kgateway.dev/, https://kgateway.dev/blog/kgateway-v2.2-release-blog, https://kgateway.dev/docs/envoy/2.1.x/ai/about.md, https://kgateway.dev/docs/envoy/2.2.x/reference/release-notes.md, https://kgateway.dev/docs/envoy/latest/about/overview.md, https://kgateway.dev/docs/envoy/latest/llms.txt | +| Agent Router (ex-Envoy AI Gateway) | Priority-ordered backend fallback, Redis-backed usage rate limits, BackendSecurityPolicy CRD; no guardrails as of October 2026 | https://theagentrouter.ai/ | +| Higress | wasm-plugin gateway; ai-proxy auto-detects OpenAI vs Claude clients across ~30 backends, xDS config push, hosted MCP servers | https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy | +| Bifrost (Maxim) | Go gateway; `sk-bf-*` virtual keys, FailureClass retry taxonomy, layered USD budgets, MCP federation, Go `.so` plugins | https://docs.getbifrost.ai/features/governance/budget-and-limits, https://github.com/maximhq/bifrost/blob/dev/core/schemas/failureclass.go | +| TensorZero | Rust gateway, archived 2026-06-12; three-level retry/fallback, scoped rate-limit rules, self-published 0.94 ms p99 benchmark at 10k QPS | https://github.com/tensorzero/tensorzero | +| one-api | Token expiry/quota/IP/model controls; dormant (v0.6.10, Feb 2025) | https://github.com/songquanpeng/one-api/blob/main/README.md, https://github.com/songquanpeng/one-api/releases | +| new-api | Four client protocols, subscription quotas plus prepaid wallet, console with payments | https://github.com/QuantumNous/new-api | +| uni-api | (now Rust) `AUTO_RETRY` to 100 attempts, six per-key scheduling algorithms, stacked rate-limit windows, cgroup-memory-aware admission | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| GPT-Load | Tier-priority weighted scheduling (group × credential weight) under a global retry budget; AccessKeys with IP-CIDR restrictions | https://www.gpt-load.com/docs/internals/scheduling | +| gproxy | Weighted fallback tiers over member providers, allow/deny key scoping, AES-256-GCM credential encryption, embedded web console | https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md | +| openproxy | rustls TLS 1.3 listener, SIGUSR2 zero-downtime upgrade, weighted upstream key pools | https://raw.githubusercontent.com/x5iu/openproxy/master/README.md | +| LLMG | Auto-registers 70+ providers from environment variables | https://modpotatodotdev.github.io/LLMG/providers/all/ | +| rust-llm-proxy | Issue/rotate/revoke key endpoints | https://raw.githubusercontent.com/habibi-dev/rust-llm-proxy/master/README.md | +| llm-api-key-proxy | Key-pool rotator; same-key server-error retries under a global deadline | `ref/llm-api-key-proxy/src/rotator_library/README.md`, https://github.com/Mirrowel/LLM-API-Key-Proxy | +| IBM ContextForge (mcp-context-forge) | Federates MCP/A2A/REST/gRPC; SHA-256 tokens with jti revocation, AES-GCM tool credentials, Argon2id, optional Vault; OTLP to eight backends | https://github.com/IBM/mcp-context-forge, https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/ | +| Tyk AI Gateway / AI Studio | Failover waterfalls, Tengo filters, `$SECRET/` references, edge microgateways | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/llm-management.md | +| vLLM Production Stack | Request-priority routing, alpha feature-gated semantic cache, in-chart Prometheus/Grafana/OTel | https://github.com/vllm-project/production-stack | +| Apache APISIX | `new:` ai-proxy plugin; Admin-API changes applied to running nodes from etcd | https://apisix.apache.org/docs/apisix/plugins/ai-proxy/, https://apisix.apache.org/docs/apisix/admin-api/ | +| BricksLLM | Pioneered per-key USD caps in 2023; last commit/release 2025-01-05 | https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md, https://api.github.com/repos/bricks-cloud/BricksLLM | + +### Cloud and enterprise gateways + +| Product | Line | Source | +|---------|------|--------| +| Cloudflare AI Gateway | Caching, guardrails (S1–S13 + P1), spend limits, OTLP under GenAI semconv, BYOK, encrypted-body Logpush, custom domains and providers | https://developers.cloudflare.com/ai-gateway/features/caching/ | +| Vercel AI Gateway | Three client formats plus nine modalities incl. Decisions; BYOK, OIDC project tokens, session affinity, automatic prompt caching, budgets, trace drains | https://vercel.com/docs/ai-gateway/sdks-and-apis | +| Azure API Management (GenAI Gateway) | `llm-token-limit` policy, managed identities, RediSearch semantic caching, REST-as-MCP export | https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities | +| AWS Bedrock AgentCore Gateway | Four inference wire formats, fallback rule blocks, dimension-keyed rate limits, Cedar-policy MCP authorization, per-target credential vaulting, VPC Lattice egress | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html | +| F5 AI Gateway | In transition (docs redirect, Python SDK archived 2026-03-18, successor Early Access); processor-chain guardrails, per-service retries, one named fallback service | https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html, https://www.f5.com/company/news/press-releases/f5-unleashes-next-generation-agentic-ready-ai-gateway-to-optimize-the-economics-and-governance-of-enterprise-ai-costs | +| Traefik Hub | `new:` `ai-rate-limit` token bucket vs `ai-quota` sliding window over tokens or USD, Redis-backed fail-open | https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit | + +### Observability platforms + +| Product | Line | Source | +|---------|------|--------| +| Helicone | TTFT and p50/p95/p99 dashboards, edge cache, OSS Rust proxy (RFC 7234 over Moka/Redis), LLM Security screening, full-body capture with omit headers, `Helicone-Property-*` metadata | https://docs.helicone.ai/features/advanced-usage/caching.md, https://github.com/helicone/ai-gateway | +| Langfuse | Rust ai-gateway relaying OpenAI + Anthropic byte-preserving; versioned prompts with stale-while-revalidate caching | https://github.com/langfuse/langfuse/tree/main/ai-gateway, https://langfuse.com/docs/prompt-management/features/caching.md | +| LangSmith (LLM Gateway, beta) | `/v1/chat/completions`, `/v1/messages`, `/v1/responses` behind one key; spend policies; in-path PII/secret redaction with restoration incl. streams | https://docs.langchain.com/langsmith/llm-gateway, https://docs.langchain.com/langsmith/llm-gateway-data-policy | + +### Routers and SaaS + +| Product | Line | Source | +|---------|------|--------| +| OpenRouter | Chat/Responses/Anthropic plus embeddings, rerank, image/video, TTS/STT, batch, files, containers; BYOK, fallbacks, prompt-cache marker translation, OTLP broadcast, spend-capped MCP server | https://openrouter.ai/docs/_llms/api-reference.txt | +| Martian | Hosted gateway live 2026-10-27; 312 models, 49 providers at `/v1/chat/completions`, `/v1/responses`, `/v1/messages` | https://api.withmartian.com/v1/models, https://docs.withmartian.com/api-reference/endpoints | +| Not Diamond | Routing-decision API, not a proxy | https://docs.notdiamond.ai/reference/token_model_select_v2_modelrouter_modelselect_post.md | +| Respan (formerly Keywords AI) | OpenAI chat, Responses, and Anthropic surfaces over 1000+ models; scoped keys, budgets, retries, caching, guardrails, remote MCP server, dashboard | https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart | +| OpenPipe | Sunset; migrated to Weights & Biases/CoreWeave, docs.openpipe.ai dead | https://openpipe.ai/security | +| Glama | Hosted MCP gateway with per-tool ACLs; `x-glama-metadata` header | https://glama.ai/mcp/gateway | +| Upstash "AI Gateway" | Negative finding: verified not to exist (11 live checks incl. sitemap, llms.txt, GitHub, npm, DNS) | https://upstash.com/docs/llms.txt | + +### Chinese-ecosystem aggregators + +| Product | Line | Source | +|---------|------|--------| +| one-api | Token quotas/expiry/IP/model controls, automatic retry; global IP limit excludes the relay data path; dormant | https://github.com/songquanpeng/one-api/blob/main/README.en.md, https://github.com/songquanpeng/one-api/releases | +| new-api | Cross-channel retry (off by default), per-IP/group/model limits, sensitive-word screening (off by default since rc.45), subscription + prepaid wallet, `/v1/tools/search` and `/v1/tools/fetch` | https://github.com/QuantumNous/new-api, https://github.com/QuantumNous/new-api/releases | +| Higress | Redis token rate limits per consumer/header/IP; ai-cache over Redis with DashVector semantic mode | https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-token-ratelimit | +| uni-api | Per-key dollar-credit quotas, stacked windows, tiered channel/key cooldowns | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| GPT-Load | 35 channels (OpenAI/Mistral/Anthropic/Gemini incl. `:embedContent`), credential cooldown/blacklist isolation | https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md | + +### Niche repositories (wave 3, November 2026) + +One-line rows for the most significant third-wave finds; full modular +records (capabilities, evidence, absences) in +[`niche-repos.md`](niche-repos.md), machine-readable rows in +[`repos-registry.json`](repos-registry.json). + +| Product | Line | Source | +|---------|------|--------| +| Switchyard (NVIDIA-NeMo) | Rust routing toolkit (libsy + Python/C/Node bindings): ordered per-candidate fallback with per-backend cooldowns and Retry-After, 429-aware fallback policy, triple-protocol demo server, judge/stage-router suite, OTel GenAI spans, theoretical cache-hit-rate tracking | https://github.com/NVIDIA-NeMo/Switchyard | +| SMG (Shepherd Model Gateway) | Rust engine-agnostic gateway: KV-cache-aware routing (four index modes), queue-depth/KV-usage overload vetoes, WASM plugin middleware, priority admission with preemption-before-first-byte, OIDC/RBAC, 150+ Prometheus metrics | https://lightseek.org/smg/v1.14.0/concepts/routing/cache-aware | +| CLIProxyAPI (router-for-me) | Go proxy (57.5k stars) turning CLI subscriptions into APIs: OAuth login flows, credential pools with three rotation strategies, retry/cooldown state machines, session-affinity sticky routing, v8 management API (ToS-gray; pattern reference) | https://github.com/router-for-me/CLIProxyAPI | +| sub2api (Wei-Shaw) | Go multi-tenant subscription-resale gateway: five-state account isolation, 401/403/429-keyed failover, cache-affinity sticky hashing, token-level cost ledger with multipliers, hash-verified remote price catalog (ToS-flagged; design reference) | https://github.com/Wei-Shaw/sub2api | +| AISIX (api7) | Rust OSS data-plane AI gateway from the APISIX creators: six routing strategies, Redis-cluster RPM/TPM/concurrency limits, exact+semantic cache, guardrail stack, MCP+A2A gateways, per-attempt usage events, SIGHUP/etcd hot config | https://github.com/api7/aisix | +| Antirion (eliasbiondo) | 18-day-old Rust gateway with embedded console: route trees with hedge races and circuit breakers, reservation-based limits, exact+semantic cache, SAML/SCIM/WebAuthn identity, OTLP+alerting, signed supply-chain releases | https://github.com/eliasbiondo/antirion | +| Plano (katanemo) | Envoy + proxy-wasm Rust data plane for agentic apps: NL-intent agent orchestration, three-mode LLM routing with fallback, MCP/HTTP guardrail filter chains, header-keyed token budgets, Agentic Signals quality detectors | https://docs.planoai.dev/guides/llm_router.html | +| ThinkWatch | Rust dual AI+MCP gateway (BSL 1.1): OIDC SSO with five-role RBAC, per-user MCP OAuth so tool calls run as the user, reversible PII redaction in streams, sliding-window limits + budgets, ClickHouse audit with SIEM export | https://github.com/ThinkWatchProject/ThinkWatch | +| Otari (mozilla-ai) | Python gateway you own and run: pre-dispatch budget holds reconciled after settlement, workspace-scoped virtual keys with model allowlists, budget-conditional routing, managed MCP tool loop, layered guardrails, idempotency replay + first-chunk hold | https://github.com/mozilla-ai/otari | +| Claude Code Router (musistudio) | TypeScript control plane for coding agents (38k stars): ordered conditional routing rules with request rewriting, sandboxed route scripts, credential pools, per-key local quotas, ToolHub MCP aggregation with lazy tool resolution | https://github.com/musistudio/claude-code-router | +| mistral.rs | Rust inference engine serving OpenAI /v1 and Anthropic /v1/messages side by side with full Anthropic SSE fidelity, agent permission gates, MCP client+server modes, Prometheus TTFT/ITL histograms | https://github.com/EricLBuehler/mistral.rs | +| claude-code-proxy (raine) | Rust Anthropic-compatible proxy over subscription upstreams (Codex/Kimi/Grok/Cursor): request-path retries, full OAuth credential lifecycle with keychain storage, model-ID routing with service tiers, monitor TUI | https://github.com/raine/claude-code-proxy | +| yolorouter | Go single-binary gateway for coding CLIs: key pool with Retry-After benching and auto-recovery, generation-guarded circuit breakers, seven-phase timeouts, request-path tool-output compression, community price catalog | https://github.com/yolorouter/yolorouter | +| xLB (EeroEternal) | Rust thin-L7/L4 streaming load balancer ("the HAProxy role"): phase-shaped timeouts (connect/first_byte/idle/total), wall-clock retry budget, non-idempotent retry split, TTFT-delta metric, client-abort propagation (dormant) | https://github.com/EeroEternal/xlb | +| GPTCache (zilliztech) | Seminal Python semantic-cache library (maintenance-mode): five-stage pipeline taxonomy, 11 similarity evaluators, LLM-verification gate, eviction subsystem, SSE hit-replay | https://github.com/zilliztech/GPTCache | + +### Reference repositories (vendored pins and README references) + +- `ref/litellm` at b84f7f82 — routing constants, Bedrock/Vertex auth + internals, prompt-caching cache, proxy config convergence. +- `ref/oc-go-cc` at 99d756c — circuit breakers, p95/p99 health metrics, 3s + SSE keep-alive, fsnotify+SIGHUP reload. +- `ref/llm-api-key-proxy` at af40b91 — same-key retries and rotator + defaults (`src/rotator_library/config/defaults.py:139-158`). +- `ref/llm-proxy` at fb588f6 — the vendored pin is `proxyllm` v0.2.3, an + in-process Python routing library (no HTTP server, no endpoints; + `ref/llm-proxy/README.md`, `ref/llm-proxy/proxyllm/proxyllm.py`): each + call routes to one of 6 provider SDK adapters by three strategies — cost + (pre-flight worst-case estimate), static Elo, or zero-shot category + classification via bart-large-mnli + (`ref/llm-proxy/proxyllm/utils/cost.py`, + `ref/llm-proxy/proxyllm/utils/categorization.py`) — with sequential + fallback down a min-heap, no retries or health tracking, a hard-coded + May-2024 price+Elo catalog + (`ref/llm-proxy/proxyllm/config/internal_config.py`), the user's own keys + via .env, and console logging only. Development stopped 2024-05-24 with + guardrails, multimodal, and context injection on its unfinished roadmap. + As contrast it validates gateway-style scope: it is a library, not a + proxy. +- gproxy, LLMG, openproxy, rust-llm-proxy — README reference projects, + sourced above under open-source gateways. + +## Baseline: llm-proxy today + +Citations verified against the working tree (wave-2 items re-verified +2026-10-27). + +| Area | State | Citations | +|------|-------|-----------| +| Client surface | `POST /providers/{p}/v1/chat/completions`, `POST /providers/{p}/v1/messages`, `POST /providers/{p}/v1/messages/count_tokens`, `GET /providers/{p}/v1/models` — all four API routes carry the mandatory `/providers/{provider}` prefix — plus `/health`, `/ready`, `/version`; 32 MiB body limit; configurable `TimeoutLayer` returning 408 on provider routes, health/version exempt | `crates/llm-proxy-server/src/routes/mod.rs:50-52,69-79`, `docs/protocol.md:25-30`, `crates/llm-proxy-server/src/routes/mod.rs:37,62`, `:44,54-67` | +| Translation | Hub-and-spoke `CoreRequest`/`CoreResponse`/`CoreEvent`, 10 content kinds; client adapters OpenAI Chat + Anthropic Messages; upstream adapters OpenAI Chat, Anthropic Messages, OpenAI Responses, Gemini GenerateContent | `docs/protocol.md:7-9`, `docs/protocol.md:80-91`, `crates/llm-proxy-provider/src/adapter/` | +| Streaming | Normalized event state machine; in-band errors after visible output; 3s SSE keep-alive comments | `docs/protocol.md:120-133,139-151`, `crates/llm-proxy-server/src/routes/core_pipeline.rs:701,883` | +| Auth | `mode=none` (loopback-only, enforced) or `client-keys` with argon2id PHC hashes; admin API behind constant-time bearer token; upstream `api_key` is a `SecretString` (redacted in `Debug`, unserialized, zeroized on drop) but plaintext in the operator's TOML on disk | `config.toml.example:42-54`, `crates/llm-proxy-api/src/auth.rs`, `crates/llm-proxy-core/src/provider_config.rs:357-375` | +| Routing | URL-path provider plus static model-alias map; no request-path retry/fallback, only bounded transient retry in catalog discovery | `crates/llm-proxy-core/src/provider_config.rs:392-394`, `crates/llm-proxy-provider/src/discovery.rs:160-170` | +| Catalogs | Static/live/hybrid with single-flight refresh and atomic disk cache; discovery keeps only id/display_name/context_length (no price or lifecycle fields); `ModelCard` hardcodes `created: 0`, null `created_at` | `crates/llm-proxy-server/src/catalog_service.rs`, `crates/llm-proxy-provider/src/discovery.rs:272-288`, `crates/llm-proxy-server/src/routes/models.rs:24-36` | +| Persistence | SQLite usage ledger, background writer, retention controls; only body-derived column is `body_hash` — full token/cost columns, no payloads | `crates/llm-proxy-storage/src/sqlite.rs:105-131`, `config.toml.example:56-68` | +| Admin API | Read-only `/usage/summary|requests|keys` with window/group_by filters and cursor paging | `crates/llm-proxy-api/src/routes.rs:53-55,116-117` | +| Metrics | Lock-free atomic counters incl. per-provider-model counts; latency ring buffer up to 1000 samples with in-process percentile helpers (never exposed); request ids; graceful shutdown | `crates/llm-proxy-core/src/metrics.rs:34-35,210-239`, `crates/llm-proxy-server/src/shutdown.rs` | +| Middleware | Per-identity token-bucket RPM, burst-tolerant with continuous refill, keyed by key id when authenticated else client IP (since 2026-10-13), loopback exempt; dedup window rejecting identical in-flight path+body requests with 409 (rejection, not coalescing); opt-in forwarded-header trust, CORS, panic catching | `crates/llm-proxy-server/src/middleware.rs:117-129`, `crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`, `config.toml.example:14-21`, `crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185` | +| Extras | Offline tiktoken counting, chars/4 fallback; per-provider fixed-point USD per-token prices, estimation only; per-provider egress forward proxy (socks5h/socks5/http/https); env-var interpolation; musl single-binary. Config changes require restart — `server.hot_reload=true` rejected at load | `crates/llm-proxy-core/src/token/tiktoken.rs` (HeuristicTokenizer; heuristic documented at `crates/llm-proxy-core/src/token/counter.rs:5`), `crates/llm-proxy-core/src/provider_config.rs:472,500`, `crates/llm-proxy-provider/src/transport.rs:124-125,293`, `crates/llm-proxy-core/src/provider_config.rs:1531-1538` | +| Cancellation | `ClientCancelled` failure event persists with no usage payload — tokens delivered before cancellation never settle | `crates/llm-proxy-server/src/routes/core_pipeline.rs:952-985`, `crates/llm-proxy-storage/src/lib.rs:80` | + +## Capability areas + +### Auth and keys + +llm-proxy: loopback-only no-auth or static argon2id client keys +(`config.toml.example:42-54`); no budgets, scoping, expiry, or +issuance/rotation; config edit + restart to change. Upstream: one static +bearer per provider, plaintext on disk. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Virtual keys (`POST /key/generate`: max_budget, budget_duration, tpm/rpm, max_parallel_requests, model allowlists/aliases) under org→team→user→key budgets | https://docs.litellm.ai/docs/proxy/virtual_keys, https://docs.litellm.ai/docs/proxy/access_control | +| Bifrost | `sk-bf-*` keys: expiry, instant `is_active` kill switch, deny-by-default provider/model scoping | https://docs.getbifrost.ai/features/governance/virtual-keys | +| agentgateway `new:` | Client virtual keys (`vk-...`, strict/optional/permissive), hash-only storage, per-key model allowlists (403 `model_not_allowed`), v1.7 UI auto-rotated values | https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/virtual-keys.md | +| Portkey `new:` | Dual-secret rotation (`POST /v2/api-keys/:id/rotate`, old secret valid `key_transition_period_ms`, min/default 30 min, id unchanged) plus scheduled policies | https://portkey.ai/docs/product/enterprise-offering/org-management/api-key-rotation | +| OpenRouter | Credit-capped inference keys via management-key API | https://openrouter.ai/docs/guides/overview/auth/management-api-keys.md | +| Vercel | 12-hour OIDC project tokens; unauthenticated compromised-secret revocation | https://vercel.com/docs/ai-gateway/authentication-and-byok/oidc | +| gproxy | Ordered allow/deny rules over provider, model glob, operation | https://gproxy.leenhawk.com/guides/permissions/ | +| one-api | Token expiry, quota, IP ranges, model allowlists | https://github.com/songquanpeng/one-api/blob/main/README.md | +| rust-llm-proxy | Issue/rotate/revoke endpoints | https://raw.githubusercontent.com/habibi-dev/rust-llm-proxy/master/README.md | +| Respan `new:` | Read/Write/Gateway/Admin scopes, Production/Test environments, spend caps, expiry, admin-gated `/api/temporary-keys/` | https://www.respan.ai/docs/documentation/admin/respan-api-keys | +| GPT-Load `new:` | AccessKeys scoped to groups/models/protocols, IP-CIDR, rotation preserving id/permissions/quota | https://www.gpt-load.com/docs/access-keys | +| Tyk `new:` | App keys activated by an administrator (approve/reject issuance) | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/apps.md | +| uni-api `new:` | `/v1/generate-api-key` behind admin-role keys | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| IBM ContextForge `new:` | SHA-256 token hashes with jti revocation/replay detection, AES-GCM tool credentials, Argon2id, optional Vault | https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/ | +| Weaker | Portkey OSS auth middleware is admin-UI-token only (its middleware directory contains just adminAuth, cache, hooks, log, requestValidator); Higress key-auth stores plaintext in YAML; Cloudflare gateway tokens account-scoped, not gateway-restrictable | https://github.com/Portkey-AI/gateway/tree/main/src/middlewares, https://higress.ai/en/docs/latest/user/plugins/authentication/key-auth, https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/ | +| Encryption at rest | gproxy AES-256-GCM under `GPROXY_MASTER_KEY`; LiteLLM NaCl SecretBox from `LITELLM_SALT_KEY` (spend logs and cached prompts stay plaintext) plus enterprise secret managers; Kong Vault entity over Konnect/AWS/GCP/Azure/HashiCorp; `new:` Portkey secret references + AWS KMS envelope encryption; new-api and GPT-Load `CRYPTO_SECRET`/`ENCRYPTION_KEY`; `new:` Tyk `$SECRET/Name` under `TYK_AI_SECRET_KEY` with env-reference allowlists | https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md, https://docs.litellm.ai/docs/proxy/security_encryption_faq.md, https://docs.litellm.ai/docs/secret_managers/overview.md, https://developer.konghq.com/gateway/entities/vault/, https://portkey.ai/docs/product/enterprise-offering/secret-references.md, https://portkey.ai/docs/product/enterprise-offering/kms.md, https://github.com/QuantumNous/new-api, https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/secrets.md | + +Verdict: gap #9 (medium, key lifecycle), gap #15 (medium, encryption at +rest). + +### Routing and resilience + +llm-proxy: URL-path provider + static alias +(`crates/llm-proxy-core/src/provider_config.rs:392-394`); no request-path +retry/fallback/cooldown +(`crates/llm-proxy-provider/src/discovery.rs:160-170` is discovery-only); +retry/fallback is scoped future work (`docs/research/core-protocol.md:170-186`). + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Six strategies (weighted simple-shuffle, least-busy, usage/latency/cost-based); per-exception RetryPolicy; TTL cooldown cache on 429 or >50% failure; typed fallback chains (general / context-window / content-policy; max_fallbacks default 5, cooldown default 5s, `ref/litellm/litellm/constants.py:13,37-43`); `new:` mid-stream fallback via assistant prefill; `region_name` fallback filtering | https://docs.litellm.ai/docs/routing, https://docs.litellm.ai/docs/proxy/reliability, https://docs.litellm.ai/docs/proxy/config_settings | +| Portkey | Recursive fallback/loadbalance/conditional configs; retries cap cumulative wait at 60s, honor Retry-After; `new:` per-strategy `cb_config` circuit breaker (cooldown ≥30s) | https://portkey.ai/docs/product/ai-gateway/fallbacks, https://portkey.ai/docs/product/ai-gateway/automatic-retries, https://portkey.ai/docs/product/ai-gateway/circuit-breaker | +| Bifrost | FailureClass taxonomy (credential/quota/model_access/rate_limit/caller_fault/transient/timeout) driving retry-same-key vs rotate-key vs stop; per-fallback retry budgets; TTFT timeouts fall back, post-output errors do not; `new:` CEL routing rules, session-affinity pin, complexity router | https://docs.getbifrost.ai/features/fallbacks, https://github.com/maximhq/bifrost/blob/dev/core/schemas/failureclass.go, https://docs.getbifrost.ai/providers/routing-rules.md, https://docs.getbifrost.ai/providers/session-affinity.md, https://docs.getbifrost.ai/features/complexity-router.md | +| Cloudflare | Up to 5 attempts with per-request backoff, ordered fallbacks, versioned dynamic routing; Auto Router weighted balancing | https://developers.cloudflare.com/ai-gateway/configuration/request-handling/, https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/, https://developers.cloudflare.com/ai-gateway/features/auto-router/ | +| Agent Router | Priority-ordered backends; retries walk the priority chain | https://theagentrouter.ai/docs/capabilities/traffic/provider-fallback | +| Kong | Seven load-balancing algorithms, circuit breaker | https://developer.konghq.com/plugins/ai-proxy-advanced/ | +| Higress | API-key pool failover with health checks and cooldown restore; ai-load-balancer extension | https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-proxy/README_EN.md, https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-load-balancer/README_EN.md | +| oc-go-cc | Three-state per-model circuit breakers, fallback chains (`ref/oc-go-cc/internal/router/fallback.go:15-115`) | `ref/oc-go-cc` | +| gproxy | Weights and fallback tiers over member providers; retries within one provider's credentials before moving on | https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md, https://gproxy.leenhawk.com/introduction/architecture/ | +| OpenRouter | Model-array fallbacks, price-weighted provider balancing | https://openrouter.ai/docs/guides/routing/model-fallbacks.md | +| Vercel | Health-aware ordering, model fallback chains, per-provider BYOK timeouts | https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks, https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts | +| TensorZero | Three-level retry/fallback hierarchy | https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/guides/retries-fallbacks.mdx | +| one-api / new-api | Automatic retry; new-api configurable cross-channel retry, off by default | https://github.com/songquanpeng/one-api/blob/main/README.md, https://github.com/QuantumNous/new-api/blob/main/model/request_policy.go | +| llm-api-key-proxy | Same-key server-error retries under a global deadline | `ref/llm-api-key-proxy/src/rotator_library/README.md` | +| AWS AgentCore `new:` | Fallback blocks (1–2 ordered targets; status-code classes, CONNECTION_ERROR/FIRST_BYTE_TIMEOUT/UNROUTABLE; default-on circuit breaker whose open state skips and does not bill; no fallback after stream start); weighted 2-way traffic splits | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules-fallback.html, https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules.html | +| Tyk `new:` | Failover waterfalls (max 10 rungs; 408/429/5xx/timeout/connect triggers, never 401/403; `X-Tyk-Served-LLM`/`X-Tyk-Failover` headers; pre-first-token only); Enterprise glob Model Router weighted/round-robin | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/llm-management.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/model-router.md | +| F5 `new:` | Per-service retries (429/500/502/503/504, Retry-After priority, 1.5x growth capped 60s) plus one `fallbackServiceName` | https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html | +| agentgateway `new:` | Per-route retries stay on the same backend; cross-model failover via virtual-model eviction groups | https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/retries.md, https://agentgateway.dev/docs/standalone/latest/documentation/llm/virtual-models.md | +| uni-api `new:` | `AUTO_RETRY` to 100, tiered channel/key cooldowns, six scheduling algorithms incl. success-rate-aware `smart_round_robin` | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| GPT-Load `new:` | Tier-priority weighted scheduling, down-tier-only global retry budget, no switch after committed stream output | https://www.gpt-load.com/docs/internals/scheduling | +| Respan `new:` | Retries on by default (≤3, ordered `fallback_models`); validation/context/credential failures never retried | https://www.respan.ai/docs/documentation/features/gateway/retries | + +Verdict: gap #1 (high, 22 products), gap #13 (medium, 12 products). + +### Rate limiting and quotas + +llm-proxy: per-identity token-bucket RPM, burst-tolerant with continuous +refill (`crates/llm-proxy-server/src/middleware.rs:117-129`), keyed by key +id when authenticated, client IP otherwise, loopback exempt +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`, since +2026-10-13; `config.toml.example:14-21`); TPM (token-based) limits and +concurrency caps absent; 429 with no queueing or prioritization. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | max_parallel_requests/rpm/tpm per key/user/team in one batched pre-call check; reserve-then-reconcile TPM (estimate prompt + max_tokens, settle actuals, refund on failure) | https://docs.litellm.ai/docs/proxy/users | +| Kong | Token-kind or cost ceilings at consumer/IP/header/path/model/provider level | https://developer.konghq.com/plugins/ai-rate-limiting-advanced/ | +| Agent Router | Token budgets (typed kinds or CEL) into Redis windows; never interrupts an admitted stream | https://theagentrouter.ai/docs/capabilities/traffic/usage-based-ratelimiting | +| Higress | Tokens per consumer/header/IP over Redis | https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-token-ratelimit | +| Bifrost | Request/token limits at provider-config/key/team/customer level, typed 401/402/403/429 errors | https://docs.getbifrost.ai/features/governance/budget-and-limits | +| gproxy | Per user/key incl. concurrency | https://gproxy.leenhawk.com/guides/permissions/ | +| new-api | Per-IP, per-group, model-level limits | https://docs.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced | +| agentgateway `new:` | CEL-keyed limits (`jwt.sub`, `apiKey.user`, `llm.requestModel`); in-memory LRU buckets (≤65,536/rule) or Envoy Rate Limit gRPC; two-phase token enforcement when backend sets `tokenize: true` (upfront estimate blocks, otherwise post-hoc) | https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/rate-limits.md | +| AWS AgentCore `new:` | Dimension-keyed limits (targetName, toolName, qualifiedModelId, JWT claims, IAM principal; immutable keys, most-specific-wins) over req/s, tokens/min (pre-estimate + actuals), concurrency; fail-open, "not a security boundary" | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rate-limits.html | +| Traefik Hub `new:` | Token-bucket `ai-rate-limit` (burst-tolerant) vs sliding-window `ai-quota` (no bursts); input/output/total-token or USD; `sourceCriterion`; body-length/4 pre-rejection; Redis fail-open | https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit | +| Azure APIM `new:` | One `llm-token-limit` policy: tokens/min (429) + long-period quotas over hourly-to-yearly fixed UTC windows (403), any counter-key expression, optional `estimate-prompt-tokens` | https://learn.microsoft.com/en-us/azure/api-management/llm-token-limit-policy | +| GPT-Load `new:` | Per-AccessKey RPM plus concurrency cap | https://www.gpt-load.com/docs/access-keys | +| uni-api `new:` | Stacked windows (`15/min,10/day`) per key, per-model substring, per-upstream-key knobs | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| Respan `new:` | Layered per-key cost/request/token limits plus customer (402) and org-plan (429) tiers | https://www.respan.ai/docs/documentation/features/gateway/limits | +| TensorZero `new:` | Scoped rules over inferences/tokens/cost with priority resolution, Valkey/Postgres state (pre-archival) | https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/enforce-custom-rate-limits.mdx | +| Weaker | Cloudflare per gateway only; one-api global IP limit excludes the relay data path; Vercel does not rate-limit paid tier; Tyk no core RPM/TPM limiter on the LLM path (plugins only) | https://developers.cloudflare.com/ai-gateway/features/rate-limiting/, https://github.com/songquanpeng/one-api/blob/main/README.en.md, https://vercel.com/docs/ai-gateway/rate-limits, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/plugins-overview.md | + +Verdict: gap #3 (medium, 14 products). + +### Cost tracking + +llm-proxy: estimated USD from operator-configured per-token prices into the +SQLite ledger (`crates/llm-proxy-storage/src/sqlite.rs:105-131`). Pricing +code exists in `crates/llm-proxy-core/src/provider_config.rs:472,500` and +`crates/llm-proxy-core/src/provider_registry.rs:387-398` (`pricing_for`), +re-exported at `crates/llm-proxy-core/src/lib.rs:36`, with commented +examples in `apps/llm-proxy/src/defaults.rs:52-60` — no shipped or synced +price table; estimation only, never enforced. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Per-request cost against a 2,752-model price map, spend logs, Prometheus spend export, pre-call reservation | https://docs.litellm.ai/docs/proxy/cost_tracking | +| Portkey | Attribution from Portkey-AI/models (2,300+ models, pushed 2026-10) | https://github.com/Portkey-AI/models | +| Helicone | 300+ model pricing repository | https://docs.helicone.ai/references/how-we-calculate-cost.md | +| agentgateway | Per-token-kind pricing (input/output/cacheRead/cacheWrite/reasoning/audio) with context tiers, models.dev import | https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/costs.md | +| Kong | Model cost management incl. TTL-tiered cache-write rates | https://developer.konghq.com/ai-gateway/model-cost-management/ | +| Enforcement wired to prices | OpenRouter, Vercel, Cloudflare, LangSmith budgets/policies; BricksLLM pioneered per-key USD caps (2023); gproxy USD budgets on users/keys/orgs/teams settled after the call | https://openrouter.ai/docs/guides/best-practices/spend-controls.md, https://vercel.com/docs/ai-gateway/observability-and-spend/budgets, https://developers.cloudflare.com/ai-gateway/features/spend-limits/, https://docs.langchain.com/langsmith/llm-gateway-spend-policies, https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md, https://gproxy.leenhawk.com/guides/permissions/ | +| Bifrost `new:` | Arrays `{max_limit USD, reset_duration 1m–1Y}`, UTC calendar alignment, fiscal quarters, overrides, cumulative customer→team→key→provider-config enforcement | https://docs.getbifrost.ai/features/governance/budget-and-limits.md | +| LiteLLM budgets `new:` | Stackable multi-window `budget_limits` ($10/24h + $100/30d), IANA-tz calendar resets, `budget_reset_time`, soft budgets, temporary raises, tag budgets via header or W3C baggage | https://docs.litellm.ai/docs/proxy/users, https://docs.litellm.ai/docs/proxy/budget_reset_and_tz, https://docs.litellm.ai/docs/proxy/tag_budgets | +| agentgateway budgets `new:` | Per-key USD-or-token rolling windows, Block (429 `budget_exceeded`) / Audit modes, multiple windows per key, needs SQLite/Postgres config DB | https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/budget-limits/per-key.md | +| Portkey enterprise `new:` | AND-condition usage policies with `group_by` counters (per-user/model/key/MCP-server), weekly/monthly resets, breach 412 | https://portkey.ai/docs/product/enterprise-offering/budget-policies | +| Cloudflare `new:` | Spend rules by provider/model/metadata, split-by-value buckets (`cf.user_id`), budget-exceeded fallback to cheaper model — only budget→fallback action found | https://developers.cloudflare.com/ai-gateway/features/spend-limits/ | +| OpenRouter `new:` | Layered guardrail/workspace/key budgets with in-flight holds pre-rejecting oversized requests (`weight_exceeds_budget` vs retryable `in_flight_budget_exhausted`, both 402) | https://openrouter.ai/docs/api-reference/limits | +| new-api / Tyk / Respan / GPT-Load / uni-api / Traefik Hub `new:` | new-api subscription + prepaid wallet; Tyk monthly budgets at LLM/App/team level with ~30s block-list pulse (enforcement Enterprise-only, Community never blocks); Respan alert/Block thresholds, customer 402; GPT-Load quotas as never-resetting totals or periodic windows from one minute to one year — soft enforcement, charged after completion; uni-api dollar-credit quotas (30-day default, 0 blocks); Traefik Hub `costLimit` | https://github.com/QuantumNous/new-api, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/budgeting.md, https://www.respan.ai/docs/documentation/features/gateway/limits, https://www.gpt-load.com/docs/access-keys, https://raw.githubusercontent.com/yym68686/uni-api/main/README.md, https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit | +| Status codes `new:` | Not converged — Bifrost 402, Portkey 412, Respan 402/429, LiteLLM 400/401/429 | https://github.com/maximhq/bifrost/blob/dev/plugins/governance/main.go, https://portkey.ai/docs/api-reference/inference-api/error-codes, https://docs.litellm.ai/docs/proxy/users | +| Attribution `new:` | agentgateway invoice-grade: Bedrock per-request STS session names into CloudTrail/CUR, Vertex billing labels into Cloud Billing export | https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/attribution.md | + +Verdict: gap #2 (high, 15 products ship USD budgets; one-api/new-api/Azure +APIM enforce token quotas instead). + +### Caching + +llm-proxy: no response cache; the dedup window rejects identical in-flight +requests with 409 +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`) — pressure +control, neither cache nor coalescing. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | 8+ backends, SHA-256 over recognized params, streams reassembled then replayed synthetically | https://docs.litellm.ai/docs/proxy/caching | +| Portkey | Simple + semantic, TTL 60s–90d, namespacing | https://portkey.ai/docs/product/ai-gateway/cache-simple-and-semantic | +| Cloudflare | SHA-256 over provider+endpoint+model+auth+body, TTL 60s–1mo | https://developers.cloudflare.com/ai-gateway/features/caching/ | +| Helicone | Edge cache; OSS Rust proxy hashing seed+path+body, RFC 7234 freshness over Moka/Redis, buffered bodies, 2xx-only | https://docs.helicone.ai/features/advanced-usage/caching.md, https://github.com/helicone/ai-gateway | +| OpenRouter | Opt-in exact-match with free hits | https://openrouter.ai/docs/guides/features/response-caching.md | +| Higress | Redis ai-cache keyed by GJSON paths (exact match, streaming replay); `new:` DashVector semantic mode | https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-cache, https://higress.ai/en/docs/ai/scene-guide/semantic-cache | +| Respan `new:` | Exact "messages only" matches, per-customer/model scoping, hit/savings dashboard | https://www.respan.ai/docs/documentation/features/gateway/caching | +| Tyk `new:` | Community exact-match plugin; enterprise semantic (cosine ≥0.95 over Redis/memory vector stores) | https://tyk.io/docs/ai-management/ai-studio/semantic-caching | +| vLLM stack `new:` | Alpha, feature-gated semantic cache (sentence-transformer, threshold 0.95) | https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/experimental/README.md | +| Semantic elsewhere | Bifrost (hash + cosine over vector stores), Kong ai-semantic-cache, Azure APIM via RediSearch | https://docs.getbifrost.ai/features/semantic-caching, https://developer.konghq.com/plugins/ai-semantic-cache/, https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities | +| Single-flight | Rare: Cloudflare documents the duplicate-origin race and OpenRouter non-coalescing; Portkey OSS, Helicone OSS, and LiteLLM cache code examined at source has no in-flight dedup; unverified for the rest | (synthesis of the sourced rows above) | +| Verified absences `new:` | agentgateway (advertised "semantic caching" absent from its Kubernetes API reference); AWS AgentCore, Tyk core, GPT-Load, uni-api document none | https://agentgateway.dev/ | + +Verdict: gap #10 (medium, 8 exact-match products). + +### Guardrails + +llm-proxy: no content inspection, redaction, or moderation. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | ~50 guardrail services across seven hook phases incl. pre-call, post-call, MCP; `new:` reversible Presidio PII masking with `output_parse_pii` re-insertion | https://docs.litellm.ai/docs/proxy/guardrails/quick_start, https://docs.litellm.ai/release_notes, https://docs.litellm.ai/docs/tutorials/presidio_pii_masking.md | +| Portkey | 20+ deterministic checks plus LLM checks (PII, moderation, language, gibberish), deny=446 / flag=246 driving fallback orchestration; `new:` pre-LLM PII redaction irreversible by design | https://portkey.ai/docs/product/guardrails, https://portkey.ai/docs/product/guardrails/pii-redaction | +| Cloudflare | S1–S13 hazards (Llama Guard 3 8B) + P1 injection (Prompt Guard 2), per-category flag/block; DLP shared with Cloudflare One | https://developers.cloudflare.com/ai-gateway/features/guardrails/, https://developers.cloudflare.com/ai-gateway/features/dlp/ | +| Kong | ai-sanitizer redacts 20+ PII entity types bidirectionally | https://developer.konghq.com/plugins/ai-sanitizer/ | +| Bifrost (enterprise) | CEL rules, Gitleaks secret detection, LLM-as-judge | https://docs.getbifrost.ai/enterprise/guardrails | +| LangSmith | In-path PII/secret redaction with placeholder restoration, incl. streams | https://docs.langchain.com/langsmith/llm-gateway-data-policy | +| Helicone | Prompt-injection screening via Meta Prompt Guard (advanced: Llama Guard), OpenAI models only | https://docs.helicone.ai/features/advanced-usage/llm-security.md | +| new-api | Sensitive-word screening on prompts and/or responses; `new:` off by default since rc.45 | https://github.com/QuantumNous/new-api/blob/main/model/request_policy.go, https://github.com/QuantumNous/new-api/releases | +| Higress | Moderation delegated to Aliyun's paid service | https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-security-guard | +| Cloud vendors | Azure AI Content Safety; AWS Bedrock Guardrails | https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities, https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html | +| Tyk `new:` | Tengo filters with redaction/quarantine over normalized messages, calling Lakera/Azure/Presidio/Bedrock/HTTP classifiers; streaming re-evaluation; tool-result screening from 2.3 | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/filters.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/guardrails.md | +| F5 `new:` | Processor chain (injection ML detector, PII matcher incl. `mcp`, language-id, system-prompt injection) with modify/reject/annotate | https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/processors-overview.html | +| agentgateway `new:` | Ordered prompt guards: regex + built-in PII (mask/reject/audit), OpenAI moderation, vendor guards, webhooks, failClosed default; masking reject-only on streams | https://agentgateway.dev/docs/standalone/latest/documentation/llm/prompt-guards/overview | +| GPT-Load `new:` | Experimental JEV guardrails via external decision service; global RE2 redaction with reversible per-AccessKey encryption restored in responses and streams | https://www.gpt-load.com/docs/advanced/guardrails, https://www.gpt-load.com/docs/advanced/redaction | +| uni-api / Respan `new:` | Optional omni-moderation of user messages; free 8-category blocking via span-01-security, org-wide | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md, https://www.respan.ai/docs/documentation/features/gateway/guardrails | +| AWS AgentCore `new:` | Deny-by-default Cedar policies on MCP `tools/call` with Bedrock Guardrails confidence-score conditions | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/use-gateway-with-policy.html | +| Documented absence | Agent Router: no guardrails as of October 2026 — transport/auth only; per its docs, not a code audit | https://theagentrouter.ai/docs/capabilities/security/ | + +Verdict: gap #7 (medium, 17 products). + +### Observability + +llm-proxy: in-process atomic counters, request ids, plain/JSON logs, +graceful shutdown; up to 1000 latency samples collected +(`crates/llm-proxy-core/src/metrics.rs:34-35`) with `Snapshot.latencies` +plus `calculate_percentile`/`calculate_p95`/`calculate_p99` computed +in-process (`crates/llm-proxy-core/src/metrics.rs:210-239`) — but no +exposure surface: no `/metrics`, percentiles never exported, no TTFT +measurement, no trace export. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Request/latency/TTFT/queue/overhead histograms, identity-labeled token/spend counters, budget-remaining gauges; OTel export with `traceparent` propagation | https://docs.litellm.ai/docs/proxy/prometheus, https://docs.litellm.ai/docs/proxy/logging | +| agentgateway | OTel GenAI semconv names directly (`gen_ai.server.request.duration`, `time_to_first_token`, `time_per_output_token`, token-usage histograms) on :15020; OTLP tracing with per-request labels and CEL metric removal | https://agentgateway.dev/docs/standalone/latest/documentation/observability/metrics/reference/, https://agentgateway.dev/docs/standalone/latest/documentation/observability/metrics/overview | +| Bifrost | Per-provider/model/virtual-key cost, token, latency-histogram, TTFT, key-health metrics with SQLite/Postgres logs and web UI; `llm.call` OTel span per request, per-attempt spans over OTLP HTTP/gRPC | https://docs.getbifrost.ai/features/observability/prometheus, https://docs.getbifrost.ai/features/observability/default, https://docs.getbifrost.ai/features/otel.md | +| Kong | `ai_metrics` request/cost/token counters (token_type labels incl. cache-write TTL splits), LLM latency histograms, TTFT under gen_ai conventions | https://developer.konghq.com/ai-gateway/monitor-ai-llm-metrics/ | +| Cloudflare | OTLP under GenAI semconv, per-user spend anomaly detection | https://developers.cloudflare.com/ai-gateway/observability/otel-integration/ | +| Vercel | Per-attempt routing logs, OTel trace drains | https://vercel.com/docs/ai-gateway/observability-and-spend/logs, https://vercel.com/docs/ai-gateway/observability-and-spend/trace-drains | +| OpenRouter | OTLP trace broadcast to ~20 destinations | https://openrouter.ai/docs/guides/features/broadcast/webhook.md | +| TensorZero | Three Prometheus metrics + OTLP traces (pre-archival) | https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/export-prometheus-metrics.mdx | +| vLLM stack | Prometheus/Grafana/OTel in-chart | https://github.com/vllm-project/production-stack | +| Higress | ai-statistics per-route/cluster/model/consumer token and latency counters | https://higress.ai/en/docs/latest/user/plugins/ai/api-o11y/ai-statistics | +| oc-go-cc | p95/p99 at `/health` (`ref/oc-go-cc/internal/metrics/metrics.go:132-154`) | `ref/oc-go-cc` | +| Tyk `new:` | Secure-by-default Prometheus `/metrics` under GenAI semconv (`gen_ai_server_request_duration_seconds`, time-to-first-token, time-per-output-token) plus `aistudio_*` governance counters, Server-Timing header splitting gateway vs upstream latency, opt-in OTel with always-on W3C traceparent | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/observability.md | +| IBM ContextForge `new:` | OTLP to eight backends + JWT-protected Prometheus | https://ibm.github.io/mcp-context-forge/latest/manage/observability/ | +| AWS AgentCore `new:` | CloudWatch metrics (latency to first token p50/p90/p99, throttles, per-target time), vended logs incl. bodies with OTel-compatible spans | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-gateway-metrics.html | +| Helicone `new:` | TTFT and p50/p95/p99 in dashboards rather than exported histograms | https://helicone-helicone-7.mintlify.app/observability/metrics | +| Semconv | OTel client-inference span shape standardized | https://raw.githubusercontent.com/open-telemetry/semantic-conventions-genai/main/docs/gen-ai/client-inference.md | + +Streaming SLO mechanics `new:` LiteLLM SSE `: ping` keep-alives +(`keepalive_seconds` 1–300), `cancel_on_disconnect` (logged 499), and +`always_include_stream_usage` for billing when the final chunk never arrives +(https://docs.litellm.ai/docs/proxy/config_settings); OpenRouter documents +per-provider cancel-billing (abort propagates at ~24 providers; +Bedrock/Groq/Google/Mistral keep processing and bill) and a final +usage-chunk accounting frame with in-band mid-stream errors +(https://openrouter.ai/docs/api_reference/streaming); oc-go-cc emits +`:keepalive` every 3s (`ref/oc-go-cc/internal/handlers/messages.go:239-266`); +uni-api ships an SSE heartbeat +(https://raw.githubusercontent.com/yym68686/uni-api/main/README.md). +Keep-alive is parity for llm-proxy +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:701,883`); no cited +product documents SLO machinery (burn-rate alerts, error budgets) — +percentiles are left to PromQL and TTFT measurement points differ. + +Payload and attribution: llm-proxy keeps digests only +(`crates/llm-proxy-storage/src/sqlite.rs:130-131`) and records only the key +id; Helicone records full bodies with per-request omit headers +(https://docs.helicone.ai/features/advanced-usage/omit-logs.md), LiteLLM +stores messages with redaction controls +(https://docs.litellm.ai/docs/proxy/logging), Cloudflare logs payloads with +per-request opt-out and `new:` Logpush exports bodies customer-key-encrypted +(a fresh AES-256-GCM key per log wrapped with the customer's RSA-4096 public +key — https://developers.cloudflare.com/ai-gateway/observability/logging/, +https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/), +and OpenRouter retains content only behind explicit opt-in +(https://openrouter.ai/docs/guides/features/logs.md). Metadata/tag headers — +Helicone `Helicone-Property-*`, LiteLLM `metadata.tags`/`x-litellm-tags` + +User-Agent auto-tagging, Portkey `x-portkey-metadata`, Cloudflare +`cf-aig-metadata` (excluded from the cache key), Vercel +`providerOptions.gateway.user`/`tags`, Glama `x-glama-metadata` — are the +dashboard grouping key everywhere. + +Verdict: gap #11 (medium, OTel, 9 products), gap #12 (medium, Prometheus, +9), gap #17 (low, percentile/TTFT exposure, 9), additional gap #20 +(stream-cancellation settlement, 2). + +### MCP + +llm-proxy: no MCP client, server, or gateway functionality. + +Protocol: Streamable HTTP + OAuth 2.1 resource-server auth standardized in +the 2025-11-25 spec (https://modelcontextprotocol.io/specification/2025-11-25/basic/transports, +https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization); +the 2026-07-28 release candidate removes sessions and adds gateway-friendly +per-operation routing headers (https://blog.modelcontextprotocol.io/posts/2026-07-28-release-candidate). + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | One endpoint for all MCP tool traffic, six-level permissions, per-tool allowlists | https://docs.litellm.ai/docs/mcp, https://docs.litellm.ai/docs/mcp_control | +| Cloudflare | MCP portals aggregating up to 80 HTTP servers behind Access with managed upstream OAuth | https://developers.cloudflare.com/cloudflare-one/access-controls/ai-controls/mcp-portals/ | +| agentgateway | Federation with CEL authorization per method; MCP OAuth2 resource-server auth; `new:` v1.7 ExtMCP guardrail chain and inline CEL processor | https://agentgateway.dev/docs/standalone/latest/documentation/mcp/mcp-authz.md, https://agentgateway.dev/docs/standalone/latest/documentation/mcp/about.md, https://agentgateway.dev/docs/standalone/latest/documentation/mcp/guardrails/about.md | +| Bifrost | Federated tools at /mcp with virtual-key-scoped filtering; `new:` execution explicit by default (separate `/v1/mcp/tool/execute`), opt-in auto-approval, sandboxed code mode across 3+ servers | https://docs.getbifrost.ai/mcp/overview, https://docs.getbifrost.ai/mcp/overview.md | +| Kong | REST↔MCP bridging in four modes; `new:` 2.1.0 stateless 2026-07-28 revision with `ttlMs`/`cacheScope` hints and W3C trace-context | https://developer.konghq.com/plugins/ai-mcp-proxy/, https://developer.konghq.com/ai-gateway/changelog/ | +| Azure APIM | Exports managed REST APIs as MCP servers | https://learn.microsoft.com/en-us/azure/api-management/mcp-server-overview | +| AWS AgentCore | Converts APIs/Lambda into MCP tools with per-tool credential injection; `new:` stateful sessions with elicitation/sampling, semantic tool search | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html, https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-features.html | +| IBM ContextForge | Federates MCP/A2A/REST/gRPC | https://github.com/IBM/mcp-context-forge | +| Glama | Hosted MCP gateway with per-tool ACLs | https://glama.ai/mcp/gateway | +| Higress | Hosts MCP servers in-gateway | https://higress.cn/en/docs/ai/mcp-quick-start/ | +| Tyk `new:` | Gateway v5.13+ proxies remote MCP servers (OAuth 2.1, per-key tool allowlists, per-tool rate limits and circuit breakers); AI Studio exposes tools at `/tools/{slug}/mcp` | https://tyk.io/docs/ai-management/mcp-gateway/overview.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/tools.md | +| Respan `new:` | Remote MCP server exposing logs, traces, prompts, monitors | https://www.respan.ai/docs/documentation/mcp | +| OpenRouter `new:` | MCP server at mcp.openrouter.ai issuing spend-capped 7-day keys via OAuth | https://openrouter.ai/docs/guides/overview/mcp-server.md | + +Verdict: gap #8 (medium, 11 products). + +### Protocol breadth + +llm-proxy: two client formats (OpenAI Chat, Anthropic Messages), four +upstream formats, count_tokens. Embeddings/audio/images/batch/realtime are +scoped future work (docs/research/endpoints.md). + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Chat, Responses, batches, files, assistants, images, audio, rerank, RAG, realtime, OCR, containers, fine-tuning, A2A, search | https://docs.litellm.ai/release_notes | +| Portkey OSS | Handlers: completions, embeddings, batches, files, fine-tunes, images, speech, transcriptions, translations, realtime WebSocket, Responses, Anthropic + count_tokens | https://github.com/Portkey-AI/gateway/tree/main/src/handlers | +| OpenRouter | Chat, Responses, Anthropic Messages plus embeddings, rerank, image/video, TTS/STT, batch, files, containers | https://openrouter.ai/docs/_llms/api-reference.txt | +| Vercel | Three client formats plus embeddings, rerank, Decisions; `new:` nine modalities incl. video and a typed Decision API | https://vercel.com/docs/ai-gateway/sdks-and-apis, https://vercel.com/docs/ai-gateway/modalities | +| Cloudflare `new:` | REST API `/ai/v1/chat/completions`, `/ai/v1/responses`, `/ai/v1/messages` | https://developers.cloudflare.com/ai-gateway/usage/rest-api/ | +| LangSmith `new:` | Beta gateway serves `/v1/chat/completions`, `/v1/messages`, `/v1/responses` behind one key | https://docs.langchain.com/langsmith/llm-gateway | +| Helicone `new:` | OpenAI Responses at `/v1/responses` for OpenAI and Anthropic models | https://docs.helicone.ai/gateway/concepts/responses-api.md | +| agentgateway `new:` | Client routes OpenAI Chat, Anthropic Messages, OpenAI Responses, Gemini generateContent, translated both directions; 23 native providers incl. Bedrock, Vertex, GitHub Copilot; A2A proxying and REST-as-tools | https://agentgateway.dev/docs/standalone/latest/documentation/llm/about.md, https://agentgateway.dev/docs/standalone/latest/documentation/about/introduction.md | +| AWS AgentCore `new:` | chat/completions, messages, responses, Bedrock converse/InvokeModel with cross-dialect translation | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-targets-inference.html | +| new-api | Four client protocols with cross-conversion and realtime WebSocket; `new:` `/v1/tools/search` and `/v1/tools/fetch` billed per call | https://github.com/QuantumNous/new-api, https://github.com/QuantumNous/new-api/releases | +| Agent Router | Four client formats under configurable prefixes, ~20 providers | https://theagentrouter.ai/docs/capabilities/llm-integrations/supported-endpoints | +| Higress | Auto-detects OpenAI vs Claude, ~30 backends | https://higress.ai/en/docs/latest/plugins/ai/api-provider/ai-proxy | +| LLMG | 70+ providers auto-registered from env | https://modpotatodotdev.github.io/LLMG/providers/all/ | +| Kong | Native provider passthrough across 17 providers | https://developer.konghq.com/plugins/ai-proxy/ | +| Tyk `new:` | Four surfaces (vendor-native, OpenAI translation, Anthropic→Bedrock bridge, legacy); governed batches/Files/Responses only in unreleased 2.3.0 | https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/proxy.md, https://github.com/TykTechnologies/ai-studio/blob/main/CHANGELOG-v2.3.0.md | +| uni-api `new:` | Chat, responses (+compact), messages, images, embeddings, audio, moderations, search across 16+ engines | https://raw.githubusercontent.com/yym68686/uni-api/main/README.md | +| GPT-Load `new:` | OpenAI Chat/Responses/Images/Embeddings/Rerank, Mistral-native OCR/audio, Anthropic, Gemini incl. `:embedContent`, 35 channels | https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md | +| Respan `new:` | OpenAI chat, Responses, Anthropic-native over 1000+ models | https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart | +| F5 `new:` | OpenAI-only client schema (chat subset + /v1/models), Anthropic upstream executor only | https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html | +| Martian / Not Diamond `new:` | Martian: 312 models / 49 providers at three endpoints; Not Diamond a routing-decision API, not a proxy | https://api.withmartian.com/v1/models, https://docs.withmartian.com/api-reference/endpoints, https://docs.notdiamond.ai/reference/token_model_select_v2_modelrouter_modelselect_post.md | +| Deliberately narrow | Langfuse Rust ai-gateway relays OpenAI + Anthropic byte-preserving | https://github.com/langfuse/langfuse/tree/main/ai-gateway | + +Translation fidelity is only partially comparable from public docs. +llm-proxy normalizes 10 content kinds (`docs/protocol.md:80-91`) and records +normalized usage incl. cache/reasoning splits +(`crates/llm-proxy-storage/src/sqlite.rs:116-121`); agentgateway normalizes +cached-prompt-token accounting (Anthropic/Bedrock exclude, OpenAI/Azure/ +Gemini include) and reports normalized plus provider-raw counts for billing +parity (https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/rate-limits.md, +https://agentgateway.dev/docs/standalone/latest/documentation/llm/observability.md); +TensorZero normalizes structured outputs to constrained decoding or forced +tool calls (https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/generate-structured-outputs.mdx). +Per-field coverage of provider-specific options was not audited for any +product here. + +Verdict: gap #6 (high, Responses client endpoint; 15 products, Tyk only in +unreleased 2.3.0; upstream adapter exists at +`crates/llm-proxy-provider/src/adapter/`, client surface does not at +`crates/llm-proxy-server/src/routes/mod.rs:69-79`). + +### BYOK, data residency, saturation, and SDKs + +`new:` comparison context, not ranked gaps. + +| Axis | Market | llm-proxy | +|------|--------|-----------| +| BYOK passthrough | Cloudflare per-request keys with aliased Secrets-Store selection (https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/); Vercel ordered per-provider credentials via `providerOptions.gateway.byok` with system fallback (https://vercel.com/docs/ai-gateway/authentication-and-byok/byok); OpenRouter prioritized/fallback BYOK with per-key model/user filters (https://openrouter.ai/docs/guides/overview/auth/byok.md); Agent Router ext-authz metadata or `x-aigw-api-key` header (https://theagentrouter.ai/docs/capabilities/security/upstream-auth); Bifrost `x-bf-direct-key` opt-in (https://docs.getbifrost.ai/features/keys-management.md); Portkey OSS by design (https://github.com/Portkey-AI/gateway) | Opt-in passthrough BYOK ships: `passthrough_auth: bool` per provider (`crates/llm-proxy-core/src/provider_config.rs:378-385`) makes the inbound client's bearer/x-api-key token the upstream credential (`crates/llm-proxy-server/src/routes/core_pipeline.rs:727-730`, passed to `resolve_target` at `:753-759`); a missing client token fails 401 'missing client auth token for passthrough provider' (`crates/llm-proxy-server/src/routes/error_response.rs:54`); shipped 2026-06-20 (commit 201e29e) | +| Data residency | Vercel ZDR-signed providers only, pinned regions (https://vercel.com/docs/ai-gateway/security-and-compliance); OpenRouter eu/us endpoints, per-key `allowed_data_regions` (https://openrouter.ai/docs/guides/features/in-region-routing.md); LiteLLM `region_name` fallback filtering (https://docs.litellm.ai/docs/proxy/reliability) | No data-posture-aware routing | +| Saturation | LiteLLM fails over-capacity fast with 429 (https://docs.litellm.ai/docs/routing); uni-api cgroup-aware `RESOURCE_ADMISSION_MODE` admit-or-queue (https://raw.githubusercontent.com/yym68686/uni-api/main/README.md); vLLM stack priority-header routing (https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/routers/routing_logic.py); Helicone no queueing on rate limits (https://docs.helicone.ai/features/advanced-usage/custom-rate-limits.md) | 429 with no queueing or prioritization (`crates/llm-proxy-server/src/middleware.rs:136`) | +| SDKs | Vercel native AI SDK integration (https://vercel.com/docs/ai-gateway/sdks-and-apis); OpenRouter generated clients in three languages (https://openrouter.ai/docs/_llms/api-reference.txt); Portkey SDK a superset of the OpenAI SDK (https://portkey.ai/docs/product/ai-gateway/universal-api); Bifrost zero-code drop-ins for six SDKs (https://raw.githubusercontent.com/maximhq/bifrost/main/README.md) | Stock OpenAI/Anthropic SDKs via base-URL swap; no dedicated SDK | + +### Upstream cloud authentication + +`new:` llm-proxy: single static bearer `api_key` +(`crates/llm-proxy-core/src/provider_config.rs:374-385`); wave-2 grep for +`sigv4|bedrock|vertex|azure|entra|workload-identity` matched only +docs/research design notes and zero Rust/TOML files — AWS SigV4, Azure +Entra, GCP ADC upstreams unreachable. Converged market pattern: a +per-provider auth object separating credential acquisition (ambient cloud +identity chains vs explicit secrets) from request attachment (bearer header +vs SigV4 signature), plus an expiry-driven credential cache. + +| Product | What it ships | Source | +|---------|---------------|--------| +| agentgateway | Rust like llm-proxy; all in-process on official SDK crates (aws-config/aws-sigv4, azure_identity, google-cloud-auth, verified in its Cargo.toml): per-backend mutually-exclusive methods (static/aws/azure/gcp/copilot/jwtSign/oauthTokenExchange), SigV4 computed last after every other request mutation, AssumeRole with per-request CEL session tags cached and refreshed before expiry, Entra via full DefaultAzureCredential chain with per-policy caches, GCP ADC access/id tokens incl. impersonated and GDCH service accounts | https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/backend-authn/, https://raw.githubusercontent.com/agentgateway/agentgateway/main/crates/agentgateway/Cargo.toml | +| LiteLLM | boto3 chains; SigV4 over a filtered header set so forwarded client headers cannot break the canonical request; DualCache IAM policy (static ~59 min, ambient 600s, STS TTL minus 60s; AssumeRole uncached); Vertex token manager with single-flight proactive refresh (FRESH/STALE/INVALID) plus one 401 re-auth retry — source-verified `ref/litellm/litellm/llms/bedrock/base_aws_llm.py:74-148,1400-1439`, `ref/litellm/litellm/llms/vertex_ai/vertex_llm_base.py:480-603` | https://docs.litellm.ai/docs/providers/bedrock, https://docs.litellm.ai/docs/providers/azure, https://docs.litellm.ai/docs/providers/vertex | +| Kong | Per-type provider auth on the AI Model Provider entity (aws assume-role incl. batch_role_arn, azure managed identity/client secret, gcp service-account JSON/Workload Identity Federation) | https://developer.konghq.com/ai-gateway/entities/ai-model-provider/ | +| Agent Router | BackendSecurityPolicy CRD (APIKey/AWSCredentials/AzureCredentials/GCPCredentials) with controller-side OIDC token exchange writing rotated secrets at expiry minus 5 minutes; extproc SigV4 self-corrects the signing region from the resolved Bedrock host | https://theagentrouter.ai/docs/capabilities/security/upstream-auth, https://raw.githubusercontent.com/theagentrouter/agent-router/main/internal/backendauth/aws.go | +| Azure APIM | Managed identities to Azure AI | https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities | +| AWS AgentCore | Per-target vaulting (token-vault ARNs, OAuth flows, SigV4 via the gateway role) | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-outbound-auth.html | +| Upstream mTLS | Only agentgateway (backendTLS file cert/key/root; SPIFFE SVID rotation Enterprise); LiteLLM, Kong, Portkey, Agent Router document none | https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/backend-tls/ | + +Verdict: gap #4 (high, 6 products). + +### Prompt caching and cache locality + +`new:` llm-proxy: `CoreContent::Text` carries `cache: +Option`, but `CacheControl` has only a `type` field +(`Ephemeral`/`Other`) — no TTL, so Anthropic's 1-hour lifetime is +unrepresentable; no adapter translates markers, injects them, or routes for +cache locality (`crates/llm-proxy-protocol/src/core.rs:142,261-286`, +verified 2026-10-27). Cache read/write usage columns already land in the +ledger (`crates/llm-proxy-storage/src/sqlite.rs:116-121`). Market treats +prompt caching as a routing and translation concern: + +| Product | What it ships | Source | +|---------|---------------|--------| +| OpenRouter | Bidirectional marker translation (Anthropic `cache_control` ↔ OpenAI `prompt_cache_breakpoint`, TTLs dropped); auto top-level `cache_control` on the last cacheable block; 10-minute sticky routing only when cache-read price beats prompt price; sticky keys from `session_id`/`x-session-id`/`prompt_cache_key`/message hash | https://openrouter.ai/docs/features/prompt-caching | +| LiteLLM | Injection at configured points (role/index/`tool_config` → Bedrock `cachePoint`, OpenAI `prompt_cache_breakpoint`), 4-block cap (v1.84.9), one-flag Anthropic auto-caching (v1.94.0+), markers → Gemini `cachedContents`, 300s deployment pinning by SHA-256 of cacheable prefix + tools (`ref/litellm/litellm/router_utils/prompt_caching_cache.py:56-197`) | https://docs.litellm.ai/docs/tutorials/prompt_caching, https://docs.litellm.ai/docs/completion/prompt_caching | +| Vercel | `caching: 'auto'` classifies explicit-marker vs implicit providers, places two markers (last message; message before last user message, system fallback), advisory `cache_anchor_items`/`cache_ttl` degrading with warnings; session affinity pins provider+credential with per-provider lifetime (5m Anthropic/Bedrock/Google, 10m OpenAI/Azure, 1h DeepSeek, minus 15s, capped 1h), session-ID hierarchy ending in a conversation fingerprint, minimum-prompt-size gating, reported affinity outcome | https://vercel.com/docs/ai-gateway/models-and-providers/automatic-caching, https://vercel.com/docs/ai-gateway/models-and-providers/session-affinity | +| Bifrost | Pins session IDs (incl. auto-adopted Claude Code/Codex/OpenCode headers) to last-serving provider+key for 1h | https://docs.getbifrost.ai/providers/session-affinity.md | +| GPT-Load | Soft affinity from AccessKey/protocol/first-input prefix, LRU 10,000 | https://www.gpt-load.com/docs/internals/scheduling | +| new-api | Per-model channel affinity | https://github.com/QuantumNous/new-api | +| Helicone | Passes Anthropic `cache_control` through, documents pricing multipliers | https://docs.helicone.ai/gateway/concepts/prompt-caching | + +Verdict: gap #14 (medium, 7 products). + +### Model lifecycle metadata + +`new:` llm-proxy: `parse_models` keeps only +id/display_name/context_length +(`crates/llm-proxy-provider/src/discovery.rs:272-288`, verified 2026-10-27); +`/v1/models` hardcodes `created: 0`, null `created_at` +(`crates/llm-proxy-server/src/routes/models.rs:24-36`). Nothing warns or +blocks on a deprecated model. + +| Who | What it ships | Source | +|-----|---------------|--------| +| OpenAI | Model object optional `shutdown_date`; notice ≥6 months GA, 3 months specialized, ~2 weeks preview | https://developers.openai.com/api/docs/api-reference/models, https://developers.openai.com/api/docs/deprecations | +| Anthropic | Tri-state `lifecycle` (active/deprecated/retired) + `deprecated_at`/`retires_at`; retired stays listed while inference fails | https://platform.claude.com/docs/en/api/models-list | +| Bedrock | `modelLifecycle` (ACTIVE/LEGACY + timestamps) drops EOL models from listings; 6-month or 45-day Legacy window | https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html, https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html | +| Gemini | Model resource has no lifecycle fields | https://ai.google.dev/api/models | +| OpenRouter | Catalog `expiration_date` (live check: 14 of 464 entries dated, expired delisted), `canonical_slug` permanent aliases, provider-feed deprecation pipeline | https://openrouter.ai/docs/guides/overview/models, https://openrouter.ai/docs/guides/providers/integration | +| LiteLLM | Cost map `deprecation_date` on 874 entries, `/model/deprecations` urgency buckets, daily-lock Slack alerts — never blocks or rewrites | https://github.com/BerriAI/litellm/blob/main/litellm/proxy/common_utils/model_deprecation.py, https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | +| `new:` all gateways cited | None emits a request-time signal (sunset-style header, response field, staged rejection); no successor-model mapping ships anywhere cited | (synthesis of the sourced rows above) | + +Verdict: gap #18 (low, 2 gateways + 3 providers). + +### Admin and tooling + +llm-proxy: CLI serve/stop/status/validate/init/models/autostart +(`apps/llm-proxy/src/cli.rs:23`), read-only usage admin API +(`crates/llm-proxy-api/src/routes.rs:53-55,116-117`), musl single-binary releases, +per-provider egress forward proxying +(`crates/llm-proxy-provider/src/transport.rs:293`). + +| Surface | Market | Source | +|---------|--------|--------| +| Admin UIs | LiteLLM DB-backed control plane + Next.js UI; gproxy embedded console; Bifrost web UI with live log streaming; agentgateway UI with key management and CEL playground + experimental agctl CLI; new-api/one-api multi-user consoles with payments; Vercel CLI + REST + MCP control-plane tools; `new:` Higress console, Tyk AI Studio console with edge microgateways, IBM ContextForge admin UI with provider CRUD and model testing, GPT-Load health/log/route-check/usage UI, Respan dashboard | https://docs.litellm.ai/docs/proxy/virtual_keys, https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md, https://docs.getbifrost.ai/features/observability/default, https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/costs.md, https://github.com/QuantumNous/new-api, https://vercel.com/docs/ai-gateway/sdks-and-apis, https://higress.cn/en/docs/latest/user/quickstart/, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/edge-gateways.md, https://ibm.github.io/mcp-context-forge/latest/api/llm-proxy/, https://www.gpt-load.com/docs/monitor, https://www.respan.ai/docs/ | +| Prompt management | TensorZero prompts as config variants; Langfuse versioned prompts with stale-while-revalidate caching; Portkey immutable versioning A/B-tested via loadbalance configs | https://github.com/tensorzero/tensorzero/blob/main/docs/experimentation/run-adaptive-ab-tests.mdx, https://langfuse.com/docs/prompt-management/features/caching.md, https://portkey.ai/docs/product/prompt-engineering-studio/prompt-versioning | +| Audit trails `new:` | LiteLLM entity lifecycle audit with before/after values and S3 export; Kong RSA-signed audit logs with TTL purge; Cloudflare config-change audit on all plans; Portkey audit logs API; new-api security audit log with step-up verification | https://docs.litellm.ai/docs/proxy/audit_logs, https://developer.konghq.com/gateway/audit-logs, https://developers.cloudflare.com/ai-gateway/reference/audit-logs/, https://portkey.ai/docs/product/enterprise-offering/audit-logs, https://github.com/QuantumNous/new-api/releases | +| Extensibility `new:` | LiteLLM CustomLogger callbacks + request-lifecycle hooks (modify/reject/stream/header) and custom auth/key-policy hooks; Bifrost Go `.so` plugins with hot reload incl. MCP hooks; Kong Lua/PDK plugins, serverless snippets, external runners; Higress and APISIX proxy-wasm in Go/Rust/C++; Portkey compiled TypeScript plugin manifests; Tyk one gRPC-isolated SDK for gateway/UI/analytics plugins. Hook names, registration, and lifecycle are bespoke — no common API emerged. Agent Router ships no custom extension point (ext_proc request closed stale) | https://docs.litellm.ai/docs/proxy/call_hooks, https://docs.litellm.ai/docs/observability/custom_callback, https://docs.getbifrost.ai/plugins, https://developer.konghq.com/custom-plugins/reference, https://developer.konghq.com/plugins/pre-function/, https://github.com/alibaba/higress/tree/main/plugins, https://apisix.apache.org/docs/apisix/plugins/ai-proxy/, https://github.com/Portkey-AI/gateway/blob/main/plugins/README.md, https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/plugins-overview.md, https://theagentrouter.ai/ | +| Config lifecycle `new:` | agentgateway hot-reloads everything except a startup-only list (adminAddr/storage/database/logging/tracing) via debounced file watch (re-registering on atomic replace), validate-then-swap, last-known-good fallback, `config_synchronized` metric, `--validate-only`, file/hybrid/readOnly modes; uni-api compiles validated snapshots atomically and swaps an all-`Arc` immutable snapshot under one write with revision-CAS overlays; LiteLLM converges pods by 30s DB polling, YAML-bootstrap-then-DB-overlay, router-update errors caught and swallowed (`ref/litellm/litellm/proxy/proxy_server.py:5027-5099`); Higress xDS push in milliseconds; Kong data planes fed from Konnect; APISIX dynamic from etcd; oc-go-cc fsnotify+SIGHUP (`ref/oc-go-cc/internal/config/watcher.go:19-101`); openproxy SIGUSR2 zero-downtime upgrade. No product cited ships versioned config with in-gateway history/diff/rollback — Kong decK keeps history in git | https://agentgateway.dev/docs/standalone/latest/documentation/setup/update, https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/state_manager.rs, https://github.com/yym68686/uni-api/blob/main/src/runtime/state.rs, https://docs.litellm.ai/docs/proxy/configs, https://docs.litellm.ai/docs/proxy/prod, https://github.com/higress-group/higress/blob/main/docs/architecture.md, https://developer.konghq.com/ai-gateway/, https://apisix.apache.org/docs/apisix/admin-api/, https://raw.githubusercontent.com/x5iu/openproxy/master/README.md, https://raw.githubusercontent.com/Kong/deck/main/README.md | +| Rust performance | LiteLLM migrating translation to a bundled Rust core (beta, off by default); Portkey self-hosted gateway runs a Rust front-end from v2.30.0; TensorZero claimed 0.94 ms p99 at 10k QPS — self-published and archived. llm-proxy overhead was not measured; no performance comparison asserted | https://docs.litellm.ai/docs/proxy/rust_gateway, https://portkey.ai/docs/self-hosting/major-performance-improvements.md, https://github.com/tensorzero/tensorzero/blob/main/docs/gateway/benchmarks.mdx | + +Verdict: gap #16 (low, hot reload, 8 products — llm-proxy rejects +`server.hot_reload` at `crates/llm-proxy-core/src/provider_config.rs:1531-1538`); +additional gap #19 (web admin UI, 11 products). Prompt management, audit +trails, and hook systems have no llm-proxy counterpart but were surveyed as +context. + +### Transport, state, and deployment topology + +llm-proxy: plain-HTTP listener; TLS only on the upstream client (reqwest +rustls-tls, `crates/llm-proxy-provider/Cargo.toml:23`); a 2026-10-14 grep +for server-side TLS (rustls/axum-server/certificate) across workspace +Cargo.toml files returns no listener TLS dependency — HTTPS fronting is +delegated to a reverse proxy; openproxy shows the in-proxy alternative +(rustls TLS 1.3 listener, +https://raw.githubusercontent.com/x5iu/openproxy/master/README.md). All +state is single-node: local SQLite ledger (`config.toml.example:56-68`); +rate limiter, dedup window, metrics, and catalog refresh in-process — N +instances mean N independent limits and N ledgers. + +| Product | What it ships | Source | +|---------|---------------|--------| +| LiteLLM | Limits, cooldowns, and spend in Redis/Postgres for multi-proxy deployments | https://docs.litellm.ai/docs/proxy/load_balancing | +| gproxy | Rate-limit counters shared via a cache backend | https://gproxy.leenhawk.com/guides/permissions/ | +| Bifrost | OSS governance explicitly single-node ("running multiple OSS Bifrost nodes with a Postgres backend is not supported") — the same shape llm-proxy has | https://docs.getbifrost.ai/features/governance/budget-and-limits | +| new-api | Per-node Redis limits count independently | https://github.com/QuantumNous/new-api | +| AWS AgentCore `new:` | VPC-internal targets and identity providers through VPC Lattice private endpoints | https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-vpc-egress.html | +| Cloudflare `new:` | Custom gateway domains; custom-provider registry for arbitrary HTTPS providers | https://developers.cloudflare.com/ai-gateway/configuration/custom-domains/, https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/ | +| Kong `new:` | 2.2.0 ships FIPS 140-3 and distroless variants | https://developer.konghq.com/ai-gateway/changelog/ | + +Verdict: single-node state matches the OSS Bifrost posture; listener TLS +delegated to a reverse proxy. + +## Complete gap register + +Severities are the research runs' final finding tiers; the audited analysis +orders the same gaps by impact — ranked by impact on a self-hosted proxy +serving coding-agent workloads: request-path reliability and spend/rate +control first, ecosystem interop second, operator convenience last. "Ships +at" counts products named across both waves. + +| # | Severity | Gap | Ships at | Evidence | Detail | +|---|----------|-----|----------|----------|--------| +| 1 | high | Request-path retries, fallback chains, provider cooldowns | 22 products (LiteLLM, Portkey, Bifrost, Cloudflare, Agent Router, Higress, Kong, OpenRouter, Vercel, TensorZero, one-api, new-api, gproxy, oc-go-cc, agentgateway, AWS AgentCore, Tyk, F5, uni-api, GPT-Load, Respan, llm-api-key-proxy) | https://docs.litellm.ai/docs/proxy/reliability | No request-path retry/fallback in llm-proxy; scoped future work (`docs/research/core-protocol.md:170-186`) | +| 2 | high | Per-key spend budgets with enforcement and periodic resets | 15 USD budgets (LiteLLM, Bifrost, OpenRouter, Vercel, Cloudflare, LangSmith, gproxy, BricksLLM, agentgateway, Portkey, Tyk, Respan, GPT-Load, uni-api, Traefik Hub); one-api/new-api/Azure APIM enforce token quotas instead | https://docs.getbifrost.ai/features/governance/budget-and-limits.md | Pricing estimation-only, no enforcement (`crates/llm-proxy-core/src/provider_config.rs:472`) | +| 3 | medium | Token-based (TPM) and concurrency rate limits — per-key RPM already exists for authenticated callers (token bucket keyed by key id, core_pipeline.rs:172-176, since 2026-10-13); unauthenticated callers fall back to per-IP (narrowed 2026-10-28: per-key RPM now exists) | 14 (LiteLLM, Kong, Agent Router, Higress, Bifrost, gproxy, new-api, agentgateway, AWS AgentCore, Traefik Hub, Respan, GPT-Load, uni-api, TensorZero) | https://docs.litellm.ai/docs/proxy/users | TPM and concurrency caps absent; limiter is a burst-tolerant token bucket keyed by key id when authenticated else client IP (`crates/llm-proxy-server/src/middleware.rs:117-129`) | +| 4 | high | Cloud-native upstream auth (SigV4/STS, Entra refresh, ADC) | 6 (agentgateway, LiteLLM, Kong, Agent Router, Azure APIM, AWS AgentCore) | https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/backend-authn/ | Static bearer only (`crates/llm-proxy-core/src/provider_config.rs:374-385`) | +| 5 | high | Upstream credential pools with rotation, cooldowns, quota windows | 8 (llm-api-key-proxy, gproxy, Bifrost, Higress, new-api, openproxy, GPT-Load, uni-api) | https://github.com/Mirrowel/LLM-API-Key-Proxy | One static key per provider (`crates/llm-proxy-core/src/provider_config.rs:375`); rotator defaults `ref/llm-api-key-proxy/src/rotator_library/config/defaults.py:139-158` | +| 6 | high | OpenAI Responses client endpoint (`/v1/responses`) | 15 (LiteLLM, Portkey OSS, OpenRouter, Vercel, LangSmith, Helicone, new-api, Agent Router, agentgateway, AWS AgentCore, Cloudflare, Respan, uni-api, GPT-Load, Martian); Tyk only in unreleased 2.3.0 | https://vercel.com/docs/ai-gateway/sdks-and-apis | Upstream Responses adapter exists, client surface does not (`crates/llm-proxy-server/src/routes/mod.rs:69-79`) | +| 7 | medium | Guardrails (PII redaction, moderation, injection detection) | 17 products | https://docs.litellm.ai/docs/proxy/guardrails/quick_start | llm-proxy ships none | +| 8 | medium | MCP gateway functionality | 11 (LiteLLM, Cloudflare, agentgateway, Bifrost, Kong, Azure APIM, AWS AgentCore, IBM ContextForge, Higress, Glama, Tyk) | https://docs.litellm.ai/docs/mcp | No MCP client, server, or federation | +| 9 | medium | Client-key lifecycle API without restart | 12 (LiteLLM, Bifrost, OpenRouter, Vercel, rust-llm-proxy, one-api, Portkey, agentgateway, GPT-Load, Respan, Tyk, uni-api) | https://portkey.ai/docs/product/enterprise-offering/org-management/api-key-rotation | Keys are static config (`config.toml.example:42-54`) | +| 10 | medium | Exact-match response caching | 8 exact-match (LiteLLM, Portkey, Cloudflare, Helicone, OpenRouter, Higress, Respan, Tyk); semantic at Bifrost, Kong, Azure APIM, LiteLLM Qdrant, vLLM alpha | https://portkey.ai/docs/product/ai-gateway/cache-simple-and-semantic | No cache; dedup rejects the second caller (`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`) | +| 11 | medium | OTel GenAI trace export | 9 (Cloudflare, Vercel, OpenRouter, agentgateway, LiteLLM, Bifrost, Tyk, IBM ContextForge, vLLM stack) | https://developers.cloudflare.com/ai-gateway/observability/otel-integration/ | No OTLP span export | +| 12 | medium | Prometheus /metrics scrape endpoint | 9 (LiteLLM, agentgateway, Bifrost, Kong, TensorZero, vLLM stack, Higress, Tyk, IBM ContextForge) | https://docs.litellm.ai/docs/proxy/prometheus | Counters in-process only (`crates/llm-proxy-core/src/metrics.rs`) | +| 13 | medium | Weighted/multi-strategy load balancing per model group | 12 (LiteLLM, Portkey, Kong, gproxy, Bifrost, OpenRouter, Cloudflare Auto Router, Higress, uni-api, GPT-Load, AWS AgentCore, Tyk) | https://developer.konghq.com/plugins/ai-proxy-advanced/ | Alias resolves to exactly one provider (`crates/llm-proxy-core/src/provider_config.rs:392-394`); inline: https://developers.cloudflare.com/ai-gateway/features/auto-router/, https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-load-balancer/README_EN.md | +| 14 | medium | Cross-provider prompt-cache plumbing and cache-locality routing | 7 (OpenRouter, LiteLLM, Vercel, Helicone, Bifrost, GPT-Load, new-api) | https://vercel.com/docs/ai-gateway/models-and-providers/automatic-caching | `CacheControl` cannot express TTLs; nothing translates or pins (`crates/llm-proxy-protocol/src/core.rs:261-286`) | +| 15 | medium | Upstream credential encryption at rest / secret-manager references | 8 (gproxy, LiteLLM, Kong, Tyk, IBM ContextForge, new-api, GPT-Load, Portkey) | https://docs.litellm.ai/docs/secret_managers/overview.md | Plaintext TOML | +| 16 | low | Config hot reload without restart | 8 (agentgateway, uni-api, Higress, Kong, APISIX, LiteLLM, oc-go-cc, openproxy) | https://agentgateway.dev/docs/standalone/latest/documentation/setup/update | `server.hot_reload` rejected (`crates/llm-proxy-core/src/provider_config.rs:1531-1538`) | +| 17 | low | No exposure surface for latency percentiles (no /metrics; snapshots and p95/p99 helpers exist in-process only, metrics.rs:210-239) | 9 (LiteLLM, agentgateway, Bifrost, oc-go-cc, vLLM stack, Kong, Tyk, Helicone, AWS AgentCore) | https://agentgateway.dev/docs/standalone/latest/documentation/observability/metrics/reference/ | Latency samples and percentile helpers in-process only (`crates/llm-proxy-core/src/metrics.rs:34-35,210-239`); TTFT unmeasured | +| 18 | low | Model lifecycle metadata (deprecation dates, successor aliases) | 2 gateways (LiteLLM, OpenRouter); 3 providers publish fields (OpenAI, Anthropic, Bedrock) | https://openrouter.ai/docs/guides/overview/models | Discovery discards it (`crates/llm-proxy-provider/src/discovery.rs:272-288`) | +| 19 | — (report rank 19) | Web admin UI | 11 (LiteLLM, gproxy, Bifrost, agentgateway, new-api, one-api, Higress, Tyk, IBM ContextForge, GPT-Load, Respan) | https://docs.litellm.ai/docs/proxy/virtual_keys | Admin is read-only JSON (`crates/llm-proxy-api/src/routes.rs:53-55,116-117`) | +| 20 | — (report rank 20) | Stream-cancellation usage settlement | 2 document cancelled-stream billing/accounting (LiteLLM, OpenRouter) | https://docs.litellm.ai/docs/proxy/config_settings | `ClientCancelled` persists with no usage payload — delivered tokens never settle (`crates/llm-proxy-server/src/routes/core_pipeline.rs:952-985`, `crates/llm-proxy-storage/src/lib.rs:80`); OpenRouter: https://openrouter.ai/docs/api_reference/streaming | + +Prevalence counts include the audited analysis's `new:` additions (Apache +APISIX, Traefik Hub), which sit outside the 67-item inventory. + +Third wave (November 2026): the niche survey +([`niche-repos.md`](niche-repos.md)) surfaces ten candidate gaps that fall +outside this register — timeout hierarchies, request-payload cost +transforms, shadow-eval quality guards, OAuth credential lifecycle for +non-cloud upstreams, upstream cancellation propagation, engine-state-aware +routing, admission queueing, caller SSO/RBAC, hash-verified price catalogs, +and hedged request races. Each is defined and evidenced there and tagged per +capability in [`repos-registry.json`](repos-registry.json); they are +candidates for register entries, not yet ranked. + +Already-scoped future work (docs/research/endpoints.md) — embeddings, +rerank, realtime, transcription/TTS, image/video generation, files, OCR, +batch, fine-tuning, search bridges — is not re-listed. BYOK, data +residency, queueing/prioritization, and SDKs are comparison context, not +ranked gaps. + +## Implementation blueprints + +One subsection per deep dive (6 wave-1, 4 wave-2); market mechanisms are +summarized, with full sourcing in the capability tables above. Where a deep +dive was deferred from near-term candidates, the deferral and its reason +are stated as the analysis gives them. + +### 1. Request-path retries, fallback chains, cooldowns (wave 1) + +Market: per-exception retry policies, typed fallback chains, cooldowns tripped on 429 or >50% failure rate; no fallback after stream start; default-on circuit breakers (Routing table). llm-proxy: the streaming path already awaits the first `CoreEvent` via +`FirstByteResult` before SSE headers commit +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:702-705`); after the +first event, keep the in-band error invariant +(`docs/protocol.md:139-151`). Done when a 429/5xx/connect error from the +primary advances to the configured fallback in that window and the ledger +records both attempts — requires extending the usage-row schema with an +attempt ordinal and provider per upstream attempt (or an attempts-count +column), a storage migration scoped into this candidate +(`crates/llm-proxy-storage/src/sqlite.rs:105-131`). + +### 2. Client-key budgets and virtual-key lifecycle (wave 1) + +Market: layered USD budgets with periodic resets — stackable multi-window limits, calendar-aligned resets, reservation; lazy window reset (https://github.com/maximhq/bifrost/blob/dev/plugins/governance/store.go); Block/Audit modes; status codes not converged (Cost table). llm-proxy: check-then-accrue — reject when the in-memory counter is over +budget, accrue actual cost after completion against the existing SQLite +ledger (`crates/llm-proxy-storage/src/sqlite.rs:105`). Reset windows are a +rolling duration (calendar alignment later). Accrued spend must be +rehydrated from the ledger on startup so a restart does not zero an +in-flight window; the window start is persisted with the key record. +Deliberately not reservation-based: bounded overshoot from in-flight calls +is accepted, and LiteLLM's cost reservation silently skips when cost is +unestimatable (https://github.com/BerriAI/litellm/issues/35524). Done when +an over-budget key gets 402 pre-dispatch, settled cost lands in the ledger, +and accrued spend survives a restart. Virtual-key lifecycle +(issuance/rotation/revocation without restart) is deferred: it needs a +control-plane decision the candidate avoids; current shape +`config.toml.example:42-54`. + +### 3. TPM and per-key/concurrency rate limits (wave 1) + +Market: batched pre-call checks per key/user/team with reserve-then-reconcile TPM; token-bucket vs sliding-window split; TPM (429) combined with long-period quotas (403) (Rate-limiting table). llm-proxy: per-key RPM already exists for authenticated callers (`crates/llm-proxy-server/src/routes/core_pipeline.rs:172-176`); TPM and concurrency caps would reuse the offline token counter +(`crates/llm-proxy-core/src/token/tiktoken.rs`, heuristic documented at +`crates/llm-proxy-core/src/token/counter.rs:5`) for reserve-then-reconcile +TPM — reservation suits tokens because estimates are cheap and refundable — +keyed by CallerId as the RPM limiter already is. Done when over-limit +requests get 429 with retry-after and rate-limit headers, reconciled to +provider-reported usage after completion. + +### 4. Exact-match response caching (wave 1) + +Market: SHA-256 keys over recognized parameters; 60s TTL floors; single-flight absent from the Portkey OSS, Helicone OSS, and LiteLLM cache code examined at source (Caching table). llm-proxy: cache keyed on a canonical `CoreRequest` serialization, +in-memory v1 with a bounded TTL (floor/default/max per the 60s-floor +convention), an entry cap, and LRU eviction under the cap. Done when a +second identical request is served from the store with a HIT header and +zero recorded usage, entries expire at the TTL and evict under the cap, and +the dedup window still rejects concurrent identical misses before origin +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`) — +coalescing (serving the second caller) remains unbuilt. + +### 5. Upstream credential pools (wave 1) + +Market: same-key server-error retries under a global deadline with rotator defaults (`ref/llm-api-key-proxy/src/rotator_library/config/defaults.py:139-158`); key-pool failover with cooldown restore; tier-priority scheduling with credential isolation (Routing table). llm-proxy: optional `[[provider.credentials]]` tables extending the single +`api_key` field (`crates/llm-proxy-core/src/provider_config.rs:375`) with +per-credential cooldown state and quota windows persisted via the existing +storage layer. Done when a credential that 401s is locked out, its cooldown +survives restart, and the credential id appears in ledger rows. + +### 6. Prometheus /metrics and OTel GenAI telemetry export (wave 1) + +Market: OTel GenAI semconv metric names (`gen_ai.server.request.duration`, `time_to_first_token`, `time_per_output_token`); latency histograms with identity-labeled counters and budget-remaining gauges; secure-by-default `/metrics`; standardized client-inference span shape (Observability table). llm-proxy: hand-rendered text exposition walking the metrics snapshot +(`crates/llm-proxy-core/src/metrics.rs`), served on the admin listener. +Done when the endpoint scrapes counters plus a latency histogram and TTFT +for streams. OTel GenAI trace export is deferred (needs a dependency +decision the candidate avoids). + +### 7. Cloud-native upstream auth: SigV4/STS, Entra, ADC, mTLS (wave 2) + +Market: per-provider auth object separating credential acquisition from request attachment plus an expiry-driven credential cache; SigV4 computed last after every other request mutation; DualCache IAM policy (static ~59 min, ambient 600s, STS TTL minus 60s) and single-flight proactive token refresh with one 401 re-auth retry (`ref/litellm/litellm/llms/bedrock/base_aws_llm.py:74-148,1400-1439`, `ref/litellm/litellm/llms/vertex_ai/vertex_llm_base.py:480-603`); OIDC token exchange writing rotated secrets at expiry minus 5 minutes; upstream mTLS only at agentgateway (Upstream-auth table). llm-proxy: static bearer only +(`crates/llm-proxy-core/src/provider_config.rs:374-385`); the wave-2 grep +matched zero Rust/TOML files. Deferred from near-term candidates: needs a +config/schema and dependency decision. + +### 8. Cross-provider prompt-cache plumbing (wave 2) + +Market: bidirectional marker translation (`cache_control` ↔ `prompt_cache_breakpoint`, TTLs dropped; Bedrock `cachePoint`, Gemini `cachedContents`, 4-block cap); auto-injection with advisory anchors that degrade with warnings; sticky cache-locality routing — 10-min pin only when cache-read price beats prompt price, 300s pin by SHA-256 of cacheable prefix + tools, per-provider pin lifetimes capped 1h, LRU-10,000 soft affinity (Prompt-caching table). llm-proxy: add a `ttl` field to `CacheControl` +(`crates/llm-proxy-protocol/src/core.rs:261-266`) and pass markers through +both client adapters so Anthropic 5m/1h semantics survive a round trip; +cache read/write usage columns already exist +(`crates/llm-proxy-storage/src/sqlite.rs:116-121`). Done when a request +carrying a 1h marker on `/v1/messages` produces the same marker upstream +and the ledger records cache-write tokens. + +### 9. Hot-reloadable configuration (wave 2) + +Market: debounced file watch with validate-then-swap, last-known-good fallback, `config_synchronized` metric, `--validate-only`, startup-only exclusion list; immutable-snapshot swap with revision-CAS overlays; 30s DB polling with swallowed router-update errors (`ref/litellm/litellm/proxy/proxy_server.py:5027-5099`); fsnotify+SIGHUP; SIGUSR2 zero-downtime upgrade; no versioned config with in-gateway history/diff/rollback (Admin table). llm-proxy: `server.hot_reload=true` rejected at load +(`crates/llm-proxy-core/src/provider_config.rs:1531-1538`). Deferred from +near-term candidates: needs a config/schema decision. + +### 10. Model lifecycle metadata in catalogs and /v1/models (wave 2) + +Market: providers publish `shutdown_date`, tri-state `lifecycle` + `deprecated_at`/`retires_at` (retired stays listed, inference fails), `modelLifecycle` (6-month or 45-day window); Gemini none. Gateways: catalog `expiration_date` (14 of 464 dated, expired delisted) + `canonical_slug` aliases; cost-map `deprecation_date` on 874 entries, urgency buckets, daily-lock Slack alerts, never blocks; no request-time signal or successor mapping anywhere cited (Model-lifecycle table). llm-proxy: extend `StaticModelCatalogEntry` +(`crates/llm-proxy-core/src/provider_config.rs:735-747`, currently +`deny_unknown_fields`, so new fields need `#[serde(default)]`) and +discovery's `parse_models` to capture per-kind lifecycle fields (OpenAI +`shutdown_date`, Anthropic `lifecycle`/`retires_at`, Bedrock-style +`modelLifecycle`), surfaced per client shape in `/v1/models` +(`crates/llm-proxy-server/src/routes/models.rs:24-36`) with retired entries +kept listed. Done when both client protocols list deprecation dates and a +warn-level log fires on deprecated-model use; blocking stays opt-in and out +of scope. + +## Build-order candidates + +From the analysis's near-term list; each names its code touchpoint and +done-condition. Gaps without a candidate (OTel traces, MCP, key lifecycle, +guardrails, load-balancing strategies, the Responses client endpoint, admin +UI, cloud-native upstream auth, hot reload, credential encryption at rest, +stream-cancellation usage settlement) are deferred: each needs a +dependency, config/schema, or control-plane decision the candidates avoid. + +1. Pre-first-event retry/fallback — the pre-commit window already exists + (`crates/llm-proxy-server/src/routes/core_pipeline.rs:702-705`, + `FirstByteResult`), so fallback slots in without breaking the in-band + error invariant; done includes the attempts migration + (`crates/llm-proxy-storage/src/sqlite.rs:105-131`). +2. Client-key budgets, check-then-accrue — the ledger already records + settled cost, so enforcement is a counter over it; reservation rejected + because LiteLLM's silently skips unestimatable cost + (https://github.com/BerriAI/litellm/issues/35524); done: 402 pre-dispatch, + settled cost in ledger, spend survives restart. +3. TPM (per-key RPM already exists, `core_pipeline.rs:172-176`) — the + offline token counter exists and token estimates are cheap and + refundable; done: 429 with retry-after and rate-limit headers, + reconciled to provider-reported usage. +4. Prometheus `/metrics` with latency/TTFT histograms — counters and + latency samples already exist (`crates/llm-proxy-core/src/metrics.rs`); + done: admin-listener endpoint scraping counters plus latency histogram + and stream TTFT. +5. Exact-match response cache — canonical `CoreRequest` serialization gives + a stable key and the dedup window already discriminates identical + requests; done: HIT header, zero usage on hit, TTL expiry, LRU eviction + under cap, dedup still rejects concurrent misses. +6. Credential pools — `[[provider.credentials]]` extends the single + `api_key` field without changing request handling; done: 401 lockout, + cooldown survives restart, credential id in ledger rows. +7. Prompt-cache TTL and marker passthrough — `CacheControl` is the only + missing field and the ledger columns already exist; done: 1h marker on + `/v1/messages` round-trips upstream, cache-write tokens land in the + ledger. +8. Model lifecycle capture — discovery already parses model lists, so this + is field work plus `#[serde(default)]` on `StaticModelCatalogEntry`; + done: both client protocols list deprecation dates, warn-level log on + deprecated-model use, blocking opt-in and out of scope. + +## Research coverage appendix + +Every inventory item from both waves, with outcome. "Not detailed in the +analysis" marks items the audited report carried no usable findings for. + +### Wave 1 — products (19) + +| Item | Outcome | Where | +|------|---------|-------| +| LiteLLM | Findings noted | Auth, Routing, Rate limiting, Cost, Caching, Guardrails, Observability, MCP, Protocol, Upstream auth, Prompt caching, Model lifecycle, Admin; `ref/litellm` pins | +| Portkey | Findings noted | Auth, Routing, Cost, Caching, Guardrails, Admin, Protocol | +| Kong AI Gateway | Findings noted | Auth, Routing, Rate limiting, Caching, Guardrails, Observability, MCP, Protocol, Upstream auth, Admin, Transport | +| Cloudflare AI Gateway | Findings noted | Auth, Routing, Rate limiting, Cost, Caching, Guardrails, Observability, MCP, Protocol, BYOK, Transport | +| Envoy AI Gateway (renamed Agent Router) | Findings noted | Market map status; Routing, Rate limiting, Guardrails (documented absence), Protocol, BYOK, Upstream auth | +| Higress | Findings noted | Auth, Routing, Rate limiting, Caching, Guardrails, Observability, MCP, Protocol, Admin and tooling (xDS config push) | +| kgateway | Findings noted — Gateway API edge features (JWT/OAuth2/OIDC + ext-proc auth, local + global rate limiting, retries with per-try timeouts and outlier detection, priority failover, transformations, mirroring); Envoy-based AI Gateway deprecated in v2.1.x, removed in v2.2.0, became agentgateway; no in-tree AI gateway, LLM cost tracking, prompt guardrails, token rate limiting, or MCP | Market map | +| TensorZero (archived 2026-06-12) | Findings noted | Market map status; Routing, Rate limiting, Protocol, Observability, Admin (benchmarks) | +| Helicone | Findings noted | Cost, Caching, Guardrails, Observability (TTFT, payload, metadata), Protocol, BYOK | +| Langfuse and LangSmith | Findings noted | Langfuse: Protocol (narrow relay), Admin (prompt caching); LangSmith: Protocol, Cost, Guardrails | +| OpenRouter | Findings noted | Auth, Routing, Cost, Caching, Observability (incl. streaming SLO), Protocol, BYOK, Prompt caching, Model lifecycle | +| Vercel AI Gateway | Findings noted | Auth, Routing, Cost, Observability, Protocol, BYOK, Prompt caching | +| Glama | Findings noted (thin: MCP gateway per-tool ACLs; `x-glama-metadata`) | MCP, Observability | +| one-api and new-api | Findings noted | Auth, Routing, Rate limiting, Cost, Guardrails, Protocol, Prompt caching, Admin, Transport; dormancy status | +| Vendored reference checkouts (oc-go-cc, llm-proxy, llm-api-key-proxy under ref/) | Findings noted for oc-go-cc, llm-api-key-proxy, and `ref/litellm`; the `ref/llm-proxy` checkout (fb588f6 per `ref/REFS`) — an in-process Python routing library (proxyllm v0.2.3), uncited by the analysis — is researched under Reference repositories | Routing, Observability, Admin, config lifecycle, blueprints; Reference repositories | +| README reference projects (gproxy, LLMG, openproxy, rust-llm-proxy) | Findings noted | Auth, Routing, Rate limiting, Cost, Protocol, Admin, Transport, blueprints | +| BricksLLM and Maxim | Findings noted — BricksLLM per-key USD-cap pioneer, dormant since 2025-01-05; Maxim covered through Bifrost (maximhq) | Market map, Cost tracking | +| vLLM production stack | Findings noted | Caching (alpha semantic), Observability, BYOK (priority routing) | +| Cloud provider LLM gateways (Azure APIM, AWS) | Findings noted — Azure APIM: Rate limiting, Caching, MCP, Upstream auth; AWS Bedrock AgentCore: Routing, Rate limiting, Guardrails, Observability, MCP, Protocol, Upstream auth, Transport | respective sections | + +### Wave 1 — themes (8) + +| Theme | Outcome | Where | +|-------|---------|-------| +| Virtual keys and multi-tenant auth | Findings noted | Auth and keys | +| Usage metering and cost tracking | Findings noted | Cost tracking; Observability (payload, metadata) | +| Routing resilience and load balancing | Findings noted | Routing and resilience | +| Caching | Findings noted | Caching | +| Guardrails and content safety | Findings noted | Guardrails | +| Observability and tracing | Findings noted | Observability | +| MCP gatewaying | Findings noted | MCP | +| Prompt management and smart routing | Findings noted | Admin (prompt management); Routing (complexity router, CEL rules) | + +### Wave 1 — deep dives (6) + +| Deep dive | Outcome | Where | +|-----------|---------|-------| +| Request-path retries/fallback/cooldowns | Findings noted | Routing; blueprint 1 | +| Client-key budgets and virtual-key lifecycle | Findings noted (budgets); lifecycle deferred with reason | Cost tracking; blueprint 2; gaps 2 and 9 | +| TPM and per-key/concurrency rate limits | Findings noted | Rate limiting; blueprint 3 | +| Exact-match response caching | Findings noted | Caching; blueprint 4 | +| Upstream credential pools | Findings noted | Routing; blueprint 5; gap 5 | +| Prometheus /metrics and OTel GenAI export | Findings noted (Prometheus carried; OTel deferred) | Observability; blueprint 6; gaps 11–12 | + +### Wave 2 — scouts (10) + +| Scout | Outcome | Where | +|-------|---------|-------| +| Bifrost (Maxim) | Findings noted | Auth, Routing, Rate limiting, Cost, Caching, Guardrails, Observability, MCP, Protocol, Prompt caching, Admin | +| agentgateway | Findings noted | Auth, Routing, Rate limiting, Cost, Observability, MCP, Protocol, Upstream auth, Admin; cache and prompt-cache absences | +| AWS Bedrock AgentCore Gateway | Findings noted | Routing, Rate limiting, Guardrails, Observability, MCP, Protocol, Upstream auth, Transport | +| IBM ContextForge | Findings noted | Auth, Observability, MCP, Admin; gap 15 | +| Tyk AI Gateway | Findings noted | Auth, Routing, Rate limiting, Cost, Caching, Guardrails, Observability, MCP, Protocol, Admin; gap 15 | +| F5 AI Gateway | Findings noted | Market map status (transition); Routing, Guardrails, Protocol | +| uni-api and GPT-Load | Findings noted | Market map; Auth, Routing and resilience, Rate limiting, Cost, Caching (absence), Guardrails, Observability, Protocol, Prompt caching, Admin; gap 5 | +| Martian and Not Diamond | Findings noted | Market map status (Martian live 2026-10-27); Protocol | +| OpenPipe and Keywords AI (now Respan) | Findings noted — OpenPipe: sunset status only; Respan: detailed | Market map status (OpenPipe); Auth, Routing, Rate limiting, Cost, Caching, Guardrails, MCP, Protocol, Admin (Respan) | +| Upstash AI Gateway | Negative finding — verified not to exist (11 live checks incl. sitemap, llms.txt, GitHub, npm, DNS) | Market map | + +### Wave 2 — doc sweeps (10) + +| Sweep | Outcome | Where | +|-------|---------|-------| +| LiteLLM | Findings noted (folded into `new:` entries) | mid-stream fallback, Presidio masking, tag budgets, rust_gateway beta | +| Portkey | Findings noted | key rotation, circuit breaker, secret references/KMS, usage policies | +| Kong AI Gateway | Findings noted | Vault entity, changelog 2.1.0/2.2.0 | +| Cloudflare AI Gateway | Findings noted | REST API surfaces, Logpush encryption, spend-limit fallback | +| one-api and new-api | Findings noted | rc.45 screening default, `/v1/tools/*` | +| Higress | Findings noted | semantic cache mode, console | +| OpenRouter | Findings noted | limits-API 402 taxonomy, streaming accounting, MCP server, prompt caching | +| vLLM production stack | Findings noted | alpha semantic cache, priority routing | +| Vercel AI Gateway | Findings noted | nine modalities, OIDC tokens, BYOK, budgets | +| TensorZero | Findings noted (pre-archival docs) | scoped rate-limit rules, benchmarks | + +### Wave 2 — frontiers (10) + +| Frontier | Outcome | Where | +|----------|---------|-------| +| Protocol surface beyond chat | Findings noted | Protocol breadth (`new:` entries) | +| Upstream auth complexity | Findings noted | Upstream cloud authentication | +| Config and operations | Findings noted | Admin and tooling (config lifecycle) | +| Extensibility and hooks | Findings noted | Admin and tooling (hook systems) | +| Advanced quota shapes | Findings noted | Cost tracking (budget shapes), Rate limiting | +| Model lifecycle management | Findings noted | Model lifecycle metadata | +| Security and compliance | Findings noted | Auth (encryption, secret refs), Observability (Logpush encryption), BYOK (residency), Transport (FIPS) | +| Client experience | Findings noted | BYOK/SDKs; Protocol (`/v1/responses`) | +| Context and prompt aids | Findings noted | Prompt caching and cache locality | +| Streaming SLO features | Findings noted | Observability (streaming SLO mechanics) | + +### Wave 2 — deep dives (4) + +| Deep dive | Outcome | Where | +|-----------|---------|-------| +| Cloud-native upstream auth (SigV4/STS, Entra OIDC refresh, GCP ADC/workload identity, mTLS) | Findings noted; deferred from candidates with reason | Upstream cloud authentication; blueprint 7 | +| Cross-provider prompt-cache plumbing (marker translation, auto-injection, sticky cache-locality routing) | Findings noted | Prompt caching and cache locality; blueprint 8 | +| Hot-reloadable configuration | Findings noted; deferred from candidates with reason | Admin and tooling; blueprint 9 | +| Model lifecycle metadata in catalogs and /v1/models | Findings noted | Model lifecycle metadata; blueprint 10 | + +### Wave 3 — niche-repository hunts (November 2026) + +Niche hunts across the Rust/Go/TypeScript/Python long tail — +agent-facing local proxies, semantic and prompt caches, MCP gateways, +guardrail/DLP toolkits, queue dispatchers and load balancers for local +inference, subscription-to-API adapters, research routers, +local-inference front doors, and curated meta-lists — produced a +deduplicated find list of 121 candidate repositories; 23 were researched in +full and are recorded modularly (identity, maintenance, capabilities with +evidence URLs, verified absences, gap-register mapping) in +[`niche-repos.md`](niche-repos.md), which also tables the full 121-find +coverage appendix. All 23 appear in [`repos-registry.json`](repos-registry.json). + +| Repo | Outcome | Where | +|------|---------|-------| +| NVIDIA-NeMo/Switchyard | Findings noted | niche-repos.md (retries/cooldowns #1, chain deadline, #6, #13, #11/#12, redaction #7) | +| EeroEternal/xlb | Findings noted; dormant six-day blaze | niche-repos.md (timeout hierarchy, #1, #3, #12/#17, cancel propagation) | +| os-tack/ostk-cache | Findings noted; dormant, private deps | niche-repos.md (#14, payload cost transforms, usage rewriting) | +| raine/claude-code-proxy | Findings noted | niche-repos.md (#1, OAuth credential lifecycle, #13, #6, #19) | +| EricLBuehler/mistral.rs | Findings noted (serving-side cousin) | niche-repos.md (#8 client+server modes, #12/#17, protocol fidelity) | +| smg-project/smg | Findings noted | niche-repos.md (engine-state routing, admission queueing, #1, #2/#3, SSO/RBAC, #8, #11/#12) | +| eliasbiondo/antirion | Findings noted; 18 days old, pre-1.0 | niche-repos.md (#1/#13 hedging, #2/#3, #7, #10, SSO/RBAC, #8, #19) | +| katanemo/plano | Findings noted | niche-repos.md (#1/#13, #3, #7 filter chains, #6, #8, #11/#12) | +| api7/aisix | Findings noted | niche-repos.md (#1/#13, #3, #9, OIDC auth, #7, #10/#14, #8/#A2A, #16) | +| MikkoParkkola/mcp-gateway | Findings noted; PolyForm NC — ideas only | niche-repos.md (#8 tool-poisoning/pinning, #1, #2, #3, #10, #16 8 ms reload) | +| ThinkWatchProject/ThinkWatch | Findings noted; BSL 1.1 | niche-repos.md (SSO/RBAC, #9, #8 per-user OAuth, #7, #2/#3, #1/#13) | +| ghuntley/preflight | Findings noted; dev dormant, CI green | niche-repos.md (#7 Gitleaks DLP + OCR, #12, #16, backpressure) | +| agiwhitelist/tokdiet | Findings noted; dormant PoC | niche-repos.md (compaction, shadow-eval guard, #2, #14 mutation guards) | +| router-for-me/CLIProxyAPI | Findings noted; ToS-gray pattern reference | niche-repos.md (OAuth lifecycle, #5, #1, #14, #6, #19/#16) | +| Wei-Shaw/sub2api | Findings noted; ToS-flagged design reference | niche-repos.md (#5, #14, #1, #2, price catalog, #6) | +| musistudio/claude-code-router | Findings noted | niche-repos.md (#13 rules, #1, #5, #9/#3, #6, #8 ToolHub, #19) | +| mozilla-ai/otari | Findings noted; pre-1.0 | niche-repos.md (#2 holds/settlement, #9, #1/#13, #3, #8, #7, #11/#12) | +| deanxv/done-hub | Findings noted | niche-repos.md (#2 cache-token ledger, OAuth channels, #1, #3, #12, #8, #7) | +| yolorouter | Findings noted | niche-repos.md (#5, #1, #14, compaction, #2/#3/#9, timeout hierarchy, price catalog) | +| NoahStepheno/completion-to-response | Findings noted; archived | niche-repos.md (#6 blueprint: Responses synthesis + event-contract tests) | +| zilliztech/GPTCache | Findings noted; maintenance-mode | niche-repos.md (#10 semantic taxonomy, verification gate, SSE replay) | +| messkan/prompt-cache | Findings noted; quiet | niche-repos.md (#10 two-threshold + gray-zone judge, #1, #12) | +| Chleba/ollamaMQ | Findings noted | niche-repos.md (admission queueing, engine-state routing, #3/#13, #16/#19) | + + + +- Documentation research only: no competitor code was built or run; no + hands-on benchmarking. llm-proxy's own overhead was not measured, so no + performance comparison is asserted (TensorZero's 0.94 ms p99 is + self-published and archived). +- Per the research runs' recorded notCovered list: no pricing, SLA, or + market-share comparisons; closed betas and undocumented capabilities not + covered; per-field translation coverage of provider-specific options + (reasoning controls, tool-choice variants, response formats) was not + audited for any product. +- Cache coalescing unverified either way beyond Cloudflare, OpenRouter, + Portkey OSS, Helicone OSS, and LiteLLM; Agent Router's guardrail absence + is per its docs, not a code audit. Gap-register prevalence counts are + counts of surveyed products named per item across both waves, not a + market census. diff --git a/docs/research/harvest-manifest.json b/docs/research/harvest-manifest.json new file mode 100644 index 0000000..6bab9fe --- /dev/null +++ b/docs/research/harvest-manifest.json @@ -0,0 +1,725 @@ +{ + "generated": "2026-11-04", + "repos": [ + { + "repo": "0xzr/freellmpool", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "1mcp-app/agent", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "AgentOps-AI/agentops", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "AmoyLab/Unla", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ENTERPILOT/GoModel", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "EinStack/glide", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "EricLBuehler/mistral.rs", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "Inebrio/Routerly", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "LMCache/LMCache", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "LostRuins/koboldcpp", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "MartialBE/one-hub", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "MikkoParkkola/mcp-gateway", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "NVIDIA-NeMo/Guardrails", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "NVIDIA-NeMo/Switchyard", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "Nya-Foundation/NyaProxy", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "RouteWorks/RouterArena", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "StackOneHQ/defender", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ThinkWatchProject/ThinkWatch", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "Wei-Shaw/sub2api", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "abetlen/llama-cpp-python", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "api7/aisix", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "aws-samples/bedrock-access-gateway", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "blue-pen5805/llm-proxy-on-cloudflare-workers", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "cita-777/metapi", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "cuihuan/awesome-ai-gateway", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "data-privacy-stack/presidio", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "eclipse-lmos/lmos-router", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ericc-ch/copilot-api", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ferro-labs/ai-gateway", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "guardrails-ai/guardrails", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "inference-gateway/inference-gateway", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "janhq/jan", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "katanemo/plano", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "kvcache-ai/Mooncake", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "lidge-jun/opencodex", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "meta-llama/PurpleLlama", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "metatool-ai/metamcp", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "mostlygeek/llama-swap", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "mozilla-ai/llamafile", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "mozilla-ai/otari", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "mudler/LocalAI", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "musistudio/claude-code-router", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "oobabooga/textgen", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "open-webui/mcpo", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "protectai/llm-guard", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "raine/claude-code-proxy", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "redis/redis-vl-python", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "router-for-me/CLIProxyAPI", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "samanhappy/mcphub", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "smart-mcp-proxy/mcpproxy-go", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "smg-project/smg", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "sparfenyuk/mcp-proxy", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "tensorchord/Awesome-LLMOps", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "theroyallab/tabbyAPI", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ulab-uiuc/LLMRouter", + "cloned": false, + "issues": 100, + "pulls": 100 + }, + { + "repo": "zilliztech/GPTCache", + "cloned": true, + "issues": 100, + "pulls": 100 + }, + { + "repo": "ip2a/mcpstore", + "cloned": false, + "issues": 99, + "pulls": 82 + }, + { + "repo": "KenyonY/openai-forward", + "cloned": false, + "issues": 100, + "pulls": 80 + }, + { + "repo": "NadirRouter/NadirClaw", + "cloned": false, + "issues": 100, + "pulls": 80 + }, + { + "repo": "Xerxes-2/clewdr", + "cloned": false, + "issues": 100, + "pulls": 76 + }, + { + "repo": "zuisong/gemini-openai-proxy", + "cloned": false, + "issues": 100, + "pulls": 72 + }, + { + "repo": "howardpen9/awesome-ai-api-proxy", + "cloned": false, + "issues": 100, + "pulls": 69 + }, + { + "repo": "coaidev/coai", + "cloned": false, + "issues": 100, + "pulls": 68 + }, + { + "repo": "1rgs/claude-code-proxy", + "cloned": false, + "issues": 100, + "pulls": 64 + }, + { + "repo": "Nayjest/lm-proxy", + "cloned": false, + "issues": 92, + "pulls": 69 + }, + { + "repo": "protectai/rebuff", + "cloned": false, + "issues": 100, + "pulls": 60 + }, + { + "repo": "BenedictKing/ccx", + "cloned": false, + "issues": 100, + "pulls": 44 + }, + { + "repo": "codefuse-ai/ModelCache", + "cloned": false, + "issues": 75, + "pulls": 58 + }, + { + "repo": "stulzq/azure-openai-proxy", + "cloned": false, + "issues": 100, + "pulls": 27 + }, + { + "repo": "PublicAffairs/openai-gemini", + "cloned": false, + "issues": 87, + "pulls": 31 + }, + { + "repo": "jianshuo/ccglass", + "cloned": false, + "issues": 70, + "pulls": 45 + }, + { + "repo": "eliasbiondo/antirion", + "cloned": true, + "issues": 62, + "pulls": 46 + }, + { + "repo": "12britz/awesome-ai-gateways", + "cloned": false, + "issues": 54, + "pulls": 50 + }, + { + "repo": "InditexTech/cerbia", + "cloned": false, + "issues": 53, + "pulls": 45 + }, + { + "repo": "deadbits/vigil-llm", + "cloned": false, + "issues": 75, + "pulls": 22 + }, + { + "repo": "youssefvdel/qwengate", + "cloned": false, + "issues": 63, + "pulls": 30 + }, + { + "repo": "deanxv/done-hub", + "cloned": true, + "issues": 65, + "pulls": 15 + }, + { + "repo": "VoAPI/VoAPI", + "cloned": false, + "issues": 76, + "pulls": 0 + }, + { + "repo": "zhu327/gemini-openai-proxy", + "cloned": false, + "issues": 60, + "pulls": 12 + }, + { + "repo": "openziti/llm-gateway", + "cloned": false, + "issues": 34, + "pulls": 28 + }, + { + "repo": "Router-One/awesome-llm-gateway", + "cloned": false, + "issues": 28, + "pulls": 28 + }, + { + "repo": "sihyeong/Awesome-LLM-Inference-Engine", + "cloned": false, + "issues": 25, + "pulls": 23 + }, + { + "repo": "Writesonic/GPTRouter", + "cloned": false, + "issues": 25, + "pulls": 15 + }, + { + "repo": "m0n0x41d/anthropic-proxy-rs", + "cloned": false, + "issues": 21, + "pulls": 16 + }, + { + "repo": "luohy15/y-router", + "cloned": false, + "issues": 23, + "pulls": 12 + }, + { + "repo": "burugo/one-mcp", + "cloned": false, + "issues": 24, + "pulls": 7 + }, + { + "repo": "messkan/prompt-cache", + "cloned": true, + "issues": 17, + "pulls": 11 + }, + { + "repo": "EeroEternal/xlb", + "cloned": true, + "issues": 10, + "pulls": 10 + }, + { + "repo": "occludra/gateway", + "cloned": false, + "issues": 10, + "pulls": 10 + }, + { + "repo": "talhaanwarch/awesome-llmops-platforms", + "cloned": false, + "issues": 10, + "pulls": 9 + }, + { + "repo": "ai-gateways/awesome-ai-gateways", + "cloned": false, + "issues": 9, + "pulls": 9 + }, + { + "repo": "CassiopeiaCode/CosyRedactGateway", + "cloned": false, + "issues": 9, + "pulls": 6 + }, + { + "repo": "hbsgithub/deno-azure-openai-proxy", + "cloned": false, + "issues": 10, + "pulls": 5 + }, + { + "repo": "upstash/semantic-cache", + "cloned": false, + "issues": 9, + "pulls": 6 + }, + { + "repo": "dext7r/ollama-api-pool", + "cloned": false, + "issues": 8, + "pulls": 6 + }, + { + "repo": "yolorouter/yolorouter", + "cloned": true, + "issues": 10, + "pulls": 3 + }, + { + "repo": "thu-nics/TokenRouter", + "cloned": false, + "issues": 5, + "pulls": 5 + }, + { + "repo": "ulab-uiuc/GraphRouter", + "cloned": false, + "issues": 9, + "pulls": 1 + }, + { + "repo": "ulab-uiuc/Router-R1", + "cloned": false, + "issues": 10, + "pulls": 0 + }, + { + "repo": "Chleba/ollamaMQ", + "cloned": true, + "issues": 5, + "pulls": 4 + }, + { + "repo": "stanford-futuredata/FrugalGPT", + "cloned": false, + "issues": 8, + "pulls": 1 + }, + { + "repo": "anyscale/llm-router", + "cloned": false, + "issues": 5, + "pulls": 3 + }, + { + "repo": "ghuntley/preflight", + "cloned": true, + "issues": 3, + "pulls": 3 + }, + { + "repo": "rohanarun/computer-use-cache", + "cloned": false, + "issues": 3, + "pulls": 3 + }, + { + "repo": "yenanjing/awesome-model-routing", + "cloned": false, + "issues": 4, + "pulls": 2 + }, + { + "repo": "MilkThink-Lab/RouterEval", + "cloned": false, + "issues": 4, + "pulls": 1 + }, + { + "repo": "achetronic/claude-oauth-proxy", + "cloned": false, + "issues": 2, + "pulls": 2 + }, + { + "repo": "akl7777777/shell-api", + "cloned": false, + "issues": 3, + "pulls": 1 + }, + { + "repo": "dongfangzan/local-openai2anthropic", + "cloned": false, + "issues": 3, + "pulls": 1 + }, + { + "repo": "NoahStepheno/completion-to-response", + "cloned": true, + "issues": 2, + "pulls": 1 + }, + { + "repo": "unclecode/antomix", + "cloned": false, + "issues": 3, + "pulls": 0 + }, + { + "repo": "os-tack/ostk-cache", + "cloned": true, + "issues": 1, + "pulls": 1 + }, + { + "repo": "stevessr/Veloera", + "cloned": false, + "issues": 1, + "pulls": 1 + }, + { + "repo": "agiwhitelist/tokdiet", + "cloned": true, + "issues": 1, + "pulls": 0 + }, + { + "repo": "Jhongwe1/edge-llm-gateway", + "cloned": false, + "issues": 0, + "pulls": 0 + }, + { + "repo": "RixAPI/Rix-API", + "cloned": false, + "issues": 0, + "pulls": 0 + }, + { + "repo": "ShirokamiRyzen/Neko-router", + "cloned": false, + "issues": 0, + "pulls": 0 + }, + { + "repo": "kasuboski/openai-gateway", + "cloned": false, + "issues": 0, + "pulls": 0 + }, + { + "repo": "kushals256/jevcache", + "cloned": false, + "issues": 0, + "pulls": 0 + }, + { + "repo": "yasuoiwakura/openai-ollama-api-bridge", + "cloned": false, + "issues": 0, + "pulls": 0 + } + ] +} \ No newline at end of file diff --git a/docs/research/niche-repos.md b/docs/research/niche-repos.md new file mode 100644 index 0000000..2b02ecd --- /dev/null +++ b/docs/research/niche-repos.md @@ -0,0 +1,1193 @@ +# Niche repository survey — third research wave (November 2026) + +Method: a third research wave, run in November 2026 after the two October +waves recorded in +[`competitive-landscape-compendium.md`](competitive-landscape-compendium.md). +Niche hunts across the Rust/Go/TypeScript/Python long tail — agent-facing +local proxies, semantic and prompt caches, MCP gateways, guardrail/DLP +toolkits, queue dispatchers and load balancers for local inference, +subscription-to-API adapters, research routers, local-inference front doors, +and curated meta-lists — produced a deduplicated find list of **121 candidate +repositories**; the **23 below were researched in full** (identity, language, +form factor, maintenance status, capabilities with primary-source evidence +URLs, and the absences the research verified). All findings and URLs come +from the wave-3 research runs; nothing was built or run. + +Sections are modular: each is self-contained and liftable on its own. Gap +numbers reference the register in the compendium (#1–#20); capabilities that +fall outside that register are tagged `new — ` and summarized as +candidate gap rows at the end of this intro; `context` marks informative but +unranked material (comparison axes the compendium deliberately leaves +unranked, already-scoped future work from `docs/research/endpoints.md`, or +parity with what llm-proxy already ships). + +Candidate new gaps this wave surfaces (each backed by at least one niche +repo): **timeout hierarchies** (phase-shaped TTFB/idle budgets), +**request-payload cost transforms** (compaction, tool-output compression, +history hygiene), **shadow-eval quality guards** for such transforms, +**OAuth credential lifecycle** for non-cloud upstreams, **upstream +cancellation propagation** on client disconnect, **engine-state-aware +routing**, **admission queueing** with priority/fair-share, **caller +SSO/RBAC**, **externally synced price catalogs**, and **hedged request +races**. + +Tagging policy: #6 tags rows that serve the OpenAI Responses API on the +client side (the register's definition); multi-protocol or endpoint breadth +without a Responses endpoint is `context`, as is breadth already scoped in +`docs/research/endpoints.md`. #19 (web admin UI) tags browser-served +consoles only — terminal TUIs and HTTP admin APIs are tagged `context` even +when they are rich admin surfaces. + +Maintenance statistics (stars, commits, release dates, activity verdicts) +are the wave-3 runs' reads of each repository's GitHub page, releases page, +or API (and clone refs where pinned), captured November 2026; each section +closes its Maintenance paragraph with a **Sources** line linking them. + +## Sections + +Switchyard (NVIDIA-NeMo) · xLB (EeroEternal) · ostk-cache (os-tack) · +claude-code-proxy (raine) · mistral.rs · SMG · Antirion · Plano · AISIX · +mcp-gateway (MikkoParkkola) · ThinkWatch · preflight (ghuntley) · tokdiet · +CLIProxyAPI · sub2api · Claude Code Router · Otari · done-hub · yolorouter · +completion-to-response · GPTCache · PromptCache (messkan) · ollamaMQ + +## NVIDIA-NeMo/Switchyard + +**Identity.** Apache-2.0 model-routing toolkit from NVIDIA NeMo, successor to +NVIDIA-AI-Blueprints/llm-router. Rust Cargo workspace (edition 2024, +`rust-version` 1.96.1): embeddable routing library `libsy` (beta), optional +HTTP LLM client with protocol translation (alpha), a runner for external +runtimes (NeMo Relay), and a standalone OpenAI/Anthropic/Responses-compatible +server explicitly labeled demo-stability ("not a supported production path"); +Python, C-ABI, and Node/TypeScript bindings plus a PyPI package. + +**Maintenance.** Very active — last commit 2026-11-03, ~100 commits since +2026-10-01, last push 2026-11-04; v0.3.0 stable (2026-09-22), v0.4.0a1 +pre-release, v0.4.0 targeted late November 2026 (roadmap issue #1026); 3,483 +stars / 354 forks. Source claims verified by the research run against a +shallow clone of main @ f60a7a2. + +**Sources.** Stats and activity: https://github.com/NVIDIA-NeMo/Switchyard +(repo page; commit and release history); capability code citations verified +against the clone at +https://github.com/NVIDIA-NeMo/Switchyard/tree/f60a7a2291ea1f8068f769fb920314c3280fb168; +v0.4.0 roadmap: https://github.com/NVIDIA-NeMo/Switchyard/issues/1026. + +| Capability | Evidence | Gap | +|---|---|---| +| Ordered per-candidate fallback: each candidate exhausts its retry budget (Retry-After honored, backoff 250 ms→2 s) before a rewritten fallback request advances | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/run.rs | #1 | +| Per-backend failure cooldown: `failure_cooldown_ms` (default 5000) parks an atomic unavailable-until timestamp after transient failures; terminal cooldown surfaces as 503 | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/reference/toml_schema.md | #1 | +| Rate-limit-aware fallback policy: `fallback_on_rate_limit` decides whether an exhausted 429 falls through or returns verbatim; forwarded credentials retry without poisoning shared cooldown | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/run.rs | #1 | +| Whole-attempt-chain deadline: one `timeout_ms` covering every attempt, retry delay, and complete stream reads | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/backend.rs | new — timeout hierarchy | +| Native triple-protocol server: `/v1/chat/completions`, `/v1/messages`, `/v1/responses` plus `/v1/decision`, count_tokens variants, and `/v1/responses/compact` | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/lib.rs | #6 | +| Agent-aware routing suite: composite judge + between-turn stage router, LLM classifier with message-hash fallback, plan-execute, advisor gates, prefill router | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/routing_algorithms/overview.md | #13 | +| Prometheus `/metrics` plus OTel spans under GenAI semantic conventions (cache-read/creation and reasoning tokens per call) | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/metrics.rs | #11, #12 | +| Append-only JSONL routing log: durable per-request routing records (chosen model, fallbacks, usage, cost) queryable via `/v1/stats` | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/routing_log.rs | #2 | +| Credential redaction middleware scrubbing provider keys from response headers and decompressed bodies | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/redaction.rs | #7 | +| Caller-credential forwarding (use your Claude login upstream) incl. a Claude Code plugin | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/examples/claude/README.md | context (parity: llm-proxy ships opt-in passthrough BYOK — `crates/llm-proxy-core/src/provider_config.rs:378-385`, `crates/llm-proxy-server/src/routes/core_pipeline.rs:727-730`) | +| Responses continuation state and `responses_reasoning` replay policy; context compaction operation | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/reference/toml_schema.md | #6, new — payload cost transforms | +| Theoretical KV-cache hit-rate tracking (prefix-fingerprint match rate as a routing-quality signal) | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/stats/cache_eligibility.rs | new — engine-state-aware routing | +| Unreleased main: speculative parallel model/decision calls with loser cancellation, host aborts, serializable routing state | https://github.com/NVIDIA-NeMo/Switchyard/blob/main/CHANGELOG.md | context (announced) | + +**Absent (verified).** No client-facing auth/key store, no per-client rate +limiting or budgets, no response caching (eval-only cache estimation), no +content guardrails, no MCP, no queryable durable store beyond JSONL, no +middleware suite, no per-provider egress forward proxy; the standalone server +is demo-stability. + +## xLB (EeroEternal) + +**Identity.** Single-crate Rust (edition 2021, tokio + httparse, ~11 source +files) streaming-first thin-L7/L4 load balancer positioned as "the HAProxy +role in the AI stack" in front of gateways like LiteLLM, New API, AISIX; it +forwards bytes, not tokens, pooling backend connections across long-lived +streaming POSTs. One static binary driven by TOML; musl/darwin tarballs and +GHCR/Docker Hub images. MIT. + +**Maintenance.** Experimentally young and currently dormant — created +2026-10-14, 38 commits and 9 releases (v0.0.1–v0.0.9) compressed into +Oct 14–19 2026, then silent; last push 2026-10-19 (~2.5 weeks before +2026-11-04); 173 stars, single maintainer, 0 open issues. + +**Sources.** Stats and activity: https://github.com/EeroEternal/xlb (repo +page) and https://github.com/EeroEternal/xlb/releases (v0.0.1–v0.0.9, +2026-10-14/19). + +| Capability | Evidence | Gap | +|---|---|---| +| Phase-shaped timeout hierarchy: `connect` (5 s), optional `first_byte` TTFB budget, `idle` per-direction in-stream silence, `total` ceiling, `fin` half-close grace; cross-field validation | https://github.com/EeroEternal/xlb/blob/main/docs/architecture.md | new — timeout hierarchy | +| Wall-clock retry budget anchored at request arrival, jointly bounding connect retries, stale-socket failovers, and pre-first-byte reroutes; exhaustion rejects with a reason metric | https://github.com/EeroEternal/xlb/blob/main/src/config.rs | #1 | +| Pre-execution-refusal vs in-doubt error split: non-idempotent POST/PUT/PATCH reroute only on unambiguous pre-execution statuses; `respect_idempotency_key` restores full reroute | https://github.com/EeroEternal/xlb/blob/main/docs/changelog.md | #1 | +| Pre-first-byte HTTP status rerouting with buffered request-body replay (≤64 MiB); hard invariant: never retried after the first response byte | https://github.com/EeroEternal/xlb/blob/main/README.md | #1 | +| Thin-L7 HTTP/1.1 framing with cross-client POST connection pooling: LIFO idle pools, non-blocking liveness probes, keep-alive rewrite, L4 fallback for h2/Upgrade/binary | https://github.com/EeroEternal/xlb/blob/main/docs/architecture.md | context | +| Health checking with hysteresis thresholds, authenticated probes (CR/LF-validated headers), pool purge on unhealthy, passive dial-failure ejection with cooldown | https://github.com/EeroEternal/xlb/blob/main/docs/gateways.md | #1 | +| `xlb_ttft_delta_ms` — histogram of proxy-added first-byte latency — plus 16 further Prometheus metrics on a dedicated listener | https://github.com/EeroEternal/xlb/blob/main/docs/observability.md | #12, #17 | +| Per-backend concurrency caps with RAII permits and capacity-aware leastconn/roundrobin preference; 503 `queue_full` on slot-acquire timeout | https://github.com/EeroEternal/xlb/blob/main/config.toml | #3, new — admission queueing (queue_full backpressure) | +| Client-abort propagation: client EOF/RST raced against backend reads so disconnect immediately closes the upstream socket; TCP keepalive on both legs | https://github.com/EeroEternal/xlb/blob/main/docs/changelog.md | new — upstream cancellation | +| Sanitized multi-attempt trace correlation: secret-redacting trace-id derivation, logged with attempt counters on every reroute | https://github.com/EeroEternal/xlb/blob/main/src/proxy/http1.rs | context | +| Integration suite (~87 KB) pinning no-retry-after-first-byte, phase timeouts, idempotency split, retry ceiling, pooled-death failover — a ready-made spec for retry-contract tests | https://github.com/EeroEternal/xlb/blob/main/tests/integration.rs | context | + +**Absent (verified, mostly stated non-goals).** No client auth, no +quotas/budgets/cost tracking, no admin API or hot reload, no LLM protocol +awareness (bodies opaque), no per-client rate limiting, no dedup/caching/ +guardrails/token counting/MCP; HTTP/2 and WebSocket clients get raw L4 +fallback only. + +## ostk-cache (os-tack) + +**Identity.** Rust local proxy (single `ostk-cache` server binary plus +`ostk-cache-hooks` installer and `ostk-cache-stats` reporter over a +tokio-free core crate) from ostk.ai that sits between agent harnesses +(Claude Code, Codex, Cursor) and Anthropic/OpenAI, rewriting request bodies +so stable context sits at byte-stable boundaries that maximize provider +prompt-cache hits; SSE/multipart forwarded verbatim. Dual MIT/Apache-2.0. + +**Maintenance.** Effectively dormant — 104 commits all by one author, last +commit 2026-06-12 (the v0.4.0 bump), no activity in the ~5 months to +2026-11-04; 10 stars. Not buildable from the public repo alone: it depends +on three private crates from sibling repo os-tack/haystack. + +**Sources.** Stats and activity: https://github.com/os-tack/ostk-cache +(repo page; 104 commits by one author) and +https://github.com/os-tack/ostk-cache/releases (v0.1.0 2026-05-06 through +v0.4.0 2026-06-12). + +| Capability | Evidence | Gap | +|---|---|---| +| Prompt-cache hit-rate engineering by request mutation: single 1 h `cache_control` collapse, HUD line, proxy-owned breakpoint placement | https://github.com/os-tack/ostk-cache/blob/main/README.md | #14 | +| Volatile billing-token stripper: schema-free recursive removal of Claude Code's per-turn `cch=;` correlator that busts ~1 MB cached prefixes | https://github.com/os-tack/ostk-cache/blob/main/src/billing_strip.rs | new — payload cost transforms | +| History hygiene: `` stripping from past turns with current-turn protection and tool_use/tool_result pairing preserved | https://github.com/os-tack/ostk-cache/blob/main/src/system_reminder_strip.rs | new — payload cost transforms | +| WARM/DEAD cache-write policy state machine with TTL-tier forecasting (median of trailing 5 inter-request gaps; 5 m/1 h write premiums) | https://github.com/os-tack/ostk-cache/blob/main/src/write_policy.rs | #14 | +| Byte-stable synthetic history projection (`rebuild`) with WARM freeze — re-serving the prior synthetic verbatim because re-render churn was 83% of cache-write spend | https://github.com/os-tack/ostk-cache/blob/main/src/rebuild.rs | new — payload cost transforms | +| Cross-provider cache-policy seam: Anthropic explicit breakpoints vs GPT `prompt_cache_retention`/`prompt_cache_key`, with normalized UsageSnapshot reconciliation | https://github.com/os-tack/ostk-cache/blob/main/src/provider_policy.rs | #14 | +| Streaming SSE usage rewriting: monotonic-max `input_tokens` overlay plus fabricated Anthropic-shaped errors to trigger client-side compaction | https://github.com/os-tack/ostk-cache/blob/main/src/usage_truth.rs | new — payload cost transforms | +| Progressive soft-cap body reduction pipeline (tool_result ejection → pair pruning → unused tool defs → structured 413 with reduction payload) | https://github.com/os-tack/ostk-cache/blob/main/README.md | new — payload cost transforms | +| Per-turn JSONL ledger with byte columns, stats CLI, and a two-window A/B passthrough protocol with a 5% noise threshold | https://github.com/os-tack/ostk-cache/blob/main/docs/PASSTHROUGH.md | context | +| Content-addressed file-handle inlining (~96-byte `file_id` refs) with a fail-open contract — every failure mode passes the original body | https://github.com/os-tack/ostk-cache/blob/main/src/rewrite_middleware.rs | new — payload cost transforms | +| Host lifecycle-hooks ingestion (SessionStart/PreToolUse/… posted to a `/hook/event` endpoint, rotated JSONL) | https://github.com/os-tack/ostk-cache/blob/main/README.md | context | +| Cross-session harness transcript tailing (Claude Code/Codex session logs → activity summary + tool_use_id seek index) | https://github.com/os-tack/ostk-cache/blob/main/src/transcript_tail.rs | context | +| Opt-in full HTTP capture with redaction, sharded rotation, bounded-channel writer, and self-loop guard | https://github.com/os-tack/ostk-cache/blob/main/src/http_capture.rs | context | + +**Absent (verified).** No auth/multi-tenancy (loopback trust, forwards the +caller's API key), no retries/failover/aliasing (one upstream per process), +no rate limits or spend caps, no local response caching, no MCP beyond a +README mention, no metrics/admin API (local files only), kernel-daemon modes +Unix-only; not verifiable by build (private deps). + +## claude-code-proxy (raine) + +**Identity.** Single-crate Rust CLI (`src/` layout) running a local +Anthropic-compatible HTTP proxy that lets Claude Code drive +subscription-backed upstreams — Codex/ChatGPT, Kimi, Grok, Cursor Agent, +OpenCode Go — with `serve/monitor/demo/models` subcommands plus per-provider +`auth login|device|status|logout`. MIT; distributed via Homebrew, install +script, prebuilt GitHub Releases (not crates.io). + +**Maintenance.** Active — latest commit 2026-10-31, v0.1.48 released +2026-11-01; 687 commits, 694 stars, ten releases Sep–Nov 2026. + +**Sources.** Stats and activity: https://github.com/raine/claude-code-proxy +(repo page), https://github.com/raine/claude-code-proxy/releases (v0.1.48, +2026-11-01), and the run-cited release commit +https://github.com/raine/claude-code-proxy/commit/49cf152. + +| Capability | Evidence | Gap | +|---|---|---| +| Request-path retries: 429/500/502/503/504 with 2 s→30 s exponential backoff honoring Retry-After; Codex empty-completion retry; subscription-limit fail-fast | https://raw.githubusercontent.com/raine/claude-code-proxy/main/src/retry.rs | #1 | +| Full OAuth credential lifecycle for subscription upstreams: browser PKCE, device-code, browser-polling flows; per-request token refresh; macOS Keychain or permissioned `auth.json` | https://claude-code-proxy.raine.dev/reference/command-reference/ | new — OAuth credential lifecycle | +| Model-ID-driven routing on a single listener: registered `gpt-*` IDs with `-fast`/`-ultrafast` service-tier suffixes, `cursor:` prefixes, Anthropic-style aliases via configurable `aliasProvider` | https://claude-code-proxy.raine.dev/using/models-and-routing/ | #13 | +| Embedded monitor TUI (attachable over SSH, auto-reconnect, 250 ms polling) plus loopback-only `/monitor` snapshot endpoint: sessions, lifecycles, tokens, throughput | https://claude-code-proxy.raine.dev/using/monitor-tui/ | context (terminal TUI; #19 covers browser-served consoles) | +| Subscription-quota usage and reasoning-token surfacing in usage reports (spend against the subscription, not USD prices) | https://raw.githubusercontent.com/raine/claude-code-proxy/main/CHANGELOG.md | context | +| Opt-in traffic capture grouped by Claude Code session, failed-response error captures, redacting rotating JSONL logs | https://claude-code-proxy.raine.dev/reference/files-and-storage/ | context | +| Multi-protocol inbound (`/v1/messages`, opt-in `/v1/chat/completions` and `/v1/responses`) plus five upstream translators incl. WebSocket transport with HTTP fallback | https://claude-code-proxy.raine.dev/reference/http-api/ | #6 | +| In-memory session-state continuity keyed by `x-claude-code-session-id` (Cursor conversation IDs, optional Codex `previous_response_id`/compaction) | https://claude-code-proxy.raine.dev/how-it-works/ | #14 | +| Per-provider offline token estimators for count_tokens (o200k_base plus image/reasoning/framing estimates) | https://claude-code-proxy.raine.dev/how-it-works/ | context (parity-plus: llm-proxy has offline tiktoken — `crates/llm-proxy-core/src/token/tiktoken.rs`; per-provider estimators are the refinement) | +| Opt-in image generation/editing endpoints and a changelog-era transcription route (current status unverified) | https://claude-code-proxy.raine.dev/reference/http-api/ | context (scoped future work in endpoints.md) | +| Homebrew/installer/Nix distribution; 64 MiB bodies; graceful SIGTERM | https://raw.githubusercontent.com/raine/claude-code-proxy/main/README.md | context | + +**Absent (verified).** No client-auth validation (accepted but never +checked; non-loopback exposure warned against), no rate limiting, dedup, +budgets, or quotas, no admin/mutation API ("no administrative mutation API, +credential API, or remote shutdown route"), no persistent usage ledger +(monitor history in-memory, resets on restart), no multi-tenant keys, no +MCP, no first-party Anthropic/Gemini API upstreams, no per-provider egress +forward proxy (global env proxies only). + +## mistral.rs (EricLBuehler) + +**Identity.** Rust LLM inference engine on the Candle ML framework (~15-crate +workspace): `mistralrs serve` CLI/server binary, embeddable +`mistralrs-server-core` axum router crate, Python SDK, Docker images; MIT. +Included here as the serving-side cousin — the closest architectural peer on +the engine side of the dual-protocol problem llm-proxy solves at the proxy +layer. + +**Maintenance.** Very active — last commit 2026-11-03, v0.9.5 (2026-10-19) +plus 27 commits on master; ~7.8k stars. + +**Sources.** Stats and activity: https://github.com/EricLBuehler/mistral.rs +(repo page) and https://github.com/EricLBuehler/mistral.rs/releases (v0.9.5, +2026-10-19). + +| Capability | Evidence | Gap | +|---|---|---| +| Side-by-side OpenAI `/v1` and Anthropic `/v1/messages` + count_tokens in one server; the internal hub is the OpenAI ChatCompletionRequest itself | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/anthropic.rs | #6 (serves `/v1/responses`), context (llm-proxy solves the same dual-protocol problem with a neutral IR — `docs/protocol.md:7-9`) | +| Full canonical Anthropic SSE event machine plus custom `agentic_tool_call_progress`/`agentic_tool_approval_required`/`file_produced` events | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/anthropic.rs | context | +| Anthropic error-envelope fidelity: typed statuses incl. 409/529, exact "prompt is too long" wording for context overflow, generic messages hiding internal error detail | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/serve/anthropic-messages-api.md | context | +| Agent permission system (auto/ask/deny) with streamed approval events, `/v1/agent/approvals/{id}` resolution, `remember_for_session`, request-may-only-tighten ratchet | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/permissions-and-approvals.mdx | context (llm-proxy executes no tools) | +| Per-request context limits (smallest of model/paged-KV/`--max-model-len`) with per-API typed overflow errors, reported on `/v1/models` | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/permissions-and-approvals.mdx | context | +| KV prefix caching (incl. multimodal) default-on for Metal plus in-situ re-quantization endpoint `/re_isq` | https://github.com/EricLBuehler/mistral.rs#what-makes-it-fast | #14 (engine-side) | +| Model-name routing with default/aliases and runtime load/unload/reload — a request to an unloaded model waits out an automatic reload | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/serve/multiple-models.mdx | context | +| MCP client: Process/HTTP/WebSocket transports, per-server bearer tokens, `tool_prefix` namespacing, timeouts, concurrency caps | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-mcp/README.md | #8 | +| MCP server mode: second listener serving JSON-RPC 2.0 at `POST /mcp`, exposing the served model as a single `chat` tool | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/expose-as-mcp.md | #8 | +| Prometheus `/metrics` with vLLM-derived TTFT/inter-token/queue histograms and client-disconnect outcome detection via an observed body wrapper | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/metrics.rs | #12, #17 | +| Request-ID echo plus Anthropic-convention `request-id` header; JSON access logs | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-cli/src/args/server.rs | context | +| Endpoint breadth: Responses with background jobs + cancel, embeddings, audio speech/transcriptions/translations, image gen, files/containers, `/v1/skills`, LoRA mgmt, system doctor | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/route_registry.rs | context (llm-proxy's endpoints.md future list) | +| Stateful server-side sessions incl. live Python interpreter state, export/import, LRU cap, no-ID prefix heuristic | https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/persist-sessions.mdx | context | +| Embeddable axum router builder (`MistralRsServerRouterBuilder`) + self-hosted OpenAPI/Swagger docs | https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/lib.rs | context | + +**Absent (verified).** No auth of any kind (docs: "use a reverse proxy"), no +rate limiting/quotas/cost/usage tracking, no gateway functions (no upstream +abstraction, retries, or failover), no response cache (KV only); Anthropic +fidelity gaps: thinking signatures always empty, `cache_control` +breakpoints/TTLs unimplemented (`cache_creation_input_tokens` always 0), +nested tool_result blocks and `is_error` not translated; sessions +in-memory only. + +## SMG — Shepherd Model Gateway (smg-project) + +**Identity.** Engine-agnostic LLM gateway in native Rust (Python +bindings/launcher), shipped as a server binary + `smg launch` CLI, +installable via Docker/Helm/pip/cargo; Apache-2.0, LightSeek-backed. Form +factor: self-hosted gateway fronting inference fleets (vLLM, TRT-LLM, +SGLang, Gemini and more). + +**Maintenance.** Exceptionally active — commits dated 2026-11-04 (the +research day, ~34 that day), v1.14.0 shipped 2026-11-03, 22 releases through +2026; 699 stars, 4,479 commits. Research was docs-based (lightseek.org/smg +v1.14.0); no local checkout exists under `ref/`. + +**Sources.** Stats and activity: https://github.com/smg-project/smg (repo +page) and the run's 15 opened pages under https://lightseek.org/smg/ +(v1.14.0 docs). + +| Capability | Evidence | Gap | +|---|---|---| +| KV-cache-aware routing as the default policy: four index modes (engine KV events, token-ID radix, text radix, TTL'd hash), match threshold, spill gate, per-model overrides | https://lightseek.org/smg/v1.14.0/concepts/routing/cache-aware | new — engine-state-aware routing | +| Engine-state-aware overload protection: queue-depth and KV-usage vetoes from `/v1/loads`, `/metrics`, gRPC `GetLoads`, KV-event records, or ZMQ, with optional 503 shedding + Retry-After | https://lightseek.org/smg/v1.14.0/concepts/reliability/overload-protection | new — engine-state-aware routing | +| Two-level tokenization cache: L0 exact-match (256 MB budget, approximate-LRU) + L1 prefix-at-special-token-boundaries (50 MB) so multi-turn chats tokenize only the suffix | https://lightseek.org/smg/v1.14.0/concepts/performance/tokenizer-caching | context | +| WASM plugin middleware (OnRequest/OnResponse) hot-deployed via `POST/GET/DELETE /wasm` without restart, sandboxed with fixed limits | https://lightseek.org/smg/v1.14.0/concepts/extensibility/wasm-plugins | context (extensibility surveyed as context in waves 1–2) | +| Priority admission scheduling: four classes via `x-smg-priority`, reserved floors, strict-priority drain, starvation promotion, preemption before first byte (a streamed byte is never preempted) | https://lightseek.org/smg/v1.14.0/concepts/reliability/priority-scheduling | new — admission queueing | +| Per-tenant token/request budgets with reserve-then-settle semantics (own estimate reserved, signed delta settled; failures never leak reservations; 429 with Retry-After) | https://lightseek.org/smg/v1.14.0/concepts/reliability/tenant-rate-limiting | #2, #3 | +| Gateway-wide admission control: concurrency permits over the whole stream, bounded FIFO queue, optional token-bucket refill, 429/503 with Retry-After and error-code headers | https://lightseek.org/smg/v1.14.0/concepts/reliability/rate-limiting | #3, new — admission queueing | +| Request-path retries: fixed retryable set, 50 ms→×1.5→30 s with ±20% jitter, bodies buffered ≤1 MiB, worker re-selection skipping failed workers, never retry a started stream | https://lightseek.org/smg/v1.14.0/concepts/reliability/retries | #1 | +| Per-worker three-state circuit breakers (429 deliberately uncounted, ≤1 failure charged per request, health-check promotion to half-open) | https://lightseek.org/smg/v1.14.0/concepts/reliability/circuit-breakers | #1 | +| Split control/data-plane auth: OIDC/JWT SSO with JWKS auto-discovery, SHA-256 hashed keys with constant-time compare, admin/user RBAC roles, tenant keys, control-plane audit log | https://lightseek.org/smg/v1.14.0/concepts/security/authentication | new — caller SSO/RBAC, #9 | +| Gateway-side MCP client: stdio + streamable HTTP (pooled dynamic servers), tool aliasing/dedup, ordered allow/deny approval policies, internal-server hiding, tool-loop caps | https://lightseek.org/smg/v1.14.0/concepts/extensibility/mcp | #8 | +| Very wide API surface: `/v1/responses` (MCP tool loops, stored conversations), `/v1/messages`, Gemini `/v1/interactions`, embeddings, rerank, classify, transcriptions, tokenize, SGLang `/generate`, `/v1/decisions`, realtime WS/WebRTC, vendor dialect profiles | https://lightseek.org/smg/v1.14.0/reference/api/openai | #6 + endpoints future work | +| 150+ Prometheus metrics across six layers (TTFT/TPOT histograms, breaker/KV-event health, capped cardinality) plus OTel trace export | https://lightseek.org/smg/v1.14.0/reference/metrics | #11, #12 | +| Async admin API with job model: worker CRUD (202 + pollable jobs), runtime tokenizer CRUD, `POST /flush_cache`, `GET /inflight`, heap profiles | https://lightseek.org/smg/v1.14.0/reference/api/admin | context (HTTP admin API; #19 covers browser-served consoles) | +| Sticky sessions/routing keys (header-derived, 4 h idle pin) plus HA mesh with SWIM gossip and CRDT state incl. radix-tree delta gossip | https://lightseek.org/smg/v1.14.0/reference/configuration | #14, context (multi-instance) | + +**Absent (verified).** No USD cost tracking, pricing tables, or spend budgets +(anywhere in the config reference); no durable per-request usage ledger +(chat history storage aside — llm-proxy's SQLite ledger is ahead here — +`crates/llm-proxy-storage/src/sqlite.rs:105-131`); +per-instance tenant limits (~N× across replicas); no per-client-IP limiting; +no request dedup; WASM plugins cannot influence routing, are skipped on +failure, and bypass OnResponse on streams; no shipped Grafana dashboards. + +## Antirion (eliasbiondo) + +**Identity.** Apache-2.0 LLM/AI gateway written as a 27-crate Rust workspace +shipping one binary — or a distroless Docker image — with the Next.js admin +console, Admin API, embedded store, and CLI (serve/validate/schema/ +healthcheck) built in; data plane :8080, admin :8081. Positioned (and +benchmarked) against Bifrost and LiteLLM. + +**Maintenance.** Active but 18 days old at research time — repo created +2026-10-17, v0.1.0/v0.1.1 released 2026-10-18 with cosign/SLSA/SBOM, last +push 2026-10-31, 534 commits; one human maintainer (commits co-authored +with Claude); 662 stars. Pre-1.0, bus factor one. + +**Sources.** Stats and activity: https://github.com/eliasbiondo/antirion +(repo page; created 2026-10-17) and +https://github.com/eliasbiondo/antirion/releases (v0.1.0/v0.1.1, +2026-10-18). + +| Capability | Evidence | Gap | +|---|---|---| +| Route trees with fallback chains, weighted Thompson-sampling splits, least-cost/latency/busy and classifier pickers; session/prefix affinity; route simulation | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-resilience/src/engine/hedge.rs | #1, #13 | +| Hedged request races: extra attempts every `delay` up to `max_extra` while none has answered; losers cancelled and settled with health verdicts | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-resilience/src/engine/hedge.rs | new — hedged requests | +| Reservation-based limits engine (GCRA/sliding window) with per-attempt holds settled before the last response byte, degraded store mode with queued-settlement replay, lease-expiry release for crashed nodes | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-limits/src/engine.rs | #2, #3 | +| Budgets with forecasts, overage grants/approvals, event feed, per-period refused/downgraded counts — run entirely off the request path | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-runtime/src/budgets/mod.rs | #2 | +| Guardrails: PII/secrets/regex/moderation/LLM-judge with format-preserving redaction, block/redact/flag/divert actions, zero-data-retention mode | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-guardrails/src/lib.rs | #7 | +| Exact + semantic response cache with single-flight, epochs, memory slab and Valkey/Lua backends, console-manageable rules | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-cache/src/semantic.rs | #10 | +| Enterprise identity in-tree: virtual keys, JWT, BYOK, full SAML 2.0 (XML dsig, c14n), OIDC SSO, SCIM, TOTP/WebAuthn, RBAC roles, audit log with change approval | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-identity/src/saml/mod.rs | new — caller SSO/RBAC, #9 | +| MCP gateway (both protocol eras, encrypted session tokens, tool approval, OpenAPI-as-tools) plus an A2A proxy | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-mcp/src/lib.rs | #8 | +| WASI 0.2 + webhook plugin hooks, sha256-pinned, per-call stores capped at 64 MiB / 5 ms, no files/env/network | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-ext/src/wasm/mod.rs | context | +| OTLP metrics/traces with GenAI semantic names plus an alerting engine with SLOs and Slack/PagerDuty/email/webhook sinks | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-obs/src/metrics/otlp.rs | #11, #12 | +| Embedded admin console + OpenAPI 3.1 Admin API (ETag/If-Match, Idempotency-Key, `?wait=`, config generations with diff/rollback, `[REDACTED]` secrets) | https://github.com/eliasbiondo/antirion/blob/console/README.md | #19 | +| Reproducible open-loop benchmark harness vs Bifrost/LiteLLM with published raw per-cell results and validity flags | https://github.com/eliasbiondo/antirion/blob/main/docs/benchmarks.md | context | +| Connect-time SSRF validation on egress (URL rules + per-resolved-address rules, trust classes) | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-egress/src/ssrf.rs | context | +| In-repo fuzzing (13 cargo-fuzz targets), cargo-deny, reproducible builds, JSON-Schema config (`antirion schema`) | https://github.com/eliasbiondo/antirion/tree/main/fuzz/fuzz_targets | context | +| Thread-per-core data plane with least-loaded accept lanes and jemalloc; zero-config first start; cluster mode on PostgreSQL + Valkey; Helm chart; signed releases | https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-server/src/threads.rs | context | + +**Absent / caveats (verified).** Linux amd64/arm64 glibc-only binaries; no +Python/JS SDKs; no audio/image/video/files/batch/fine-tuning endpoints; no +durable request replay beyond retry budgets; ~3 weeks of production +exposure; benchmark cells largely flagged invalid (EC2 network floor) — +cross-gateway numbers need care. + +## Plano (katanemo) + +**Identity.** Katanemo's (a DigitalOcean company) open-source AI-native +proxy server and out-of-process data plane for agentic apps, built on Envoy; +the data plane is Rust — proxy-wasm cdylib plugins (`llm_gateway`, +`prompt_gateway`, `hermesllm`) inside Envoy plus the Rust "brightstaff" +model-serving process — with a TypeScript `planoai` CLI. Apache-2.0; run +via `planoai up config.yaml` or a single Docker image; declarative YAML. + +**Maintenance.** Active — last commit and release 0.4.39 both 2026-10-24, +11 releases since mid-July 2026, active issues/PRs; 7.1k stars. Docs at +docs.planoai.dev; planoai.dev self-reports HuggingFace routing 115+ LLMs +across 1M+ interactions through it (vendor claim, https://planoai.dev/). + +**Sources.** Stats and activity: https://github.com/katanemo/plano (repo +page) and https://github.com/katanemo/plano/releases (0.4.39, 2026-10-24). + +| Capability | Evidence | Gap | +|---|---|---| +| NL-intent agent orchestration listener: a small orchestrator model infers intent per turn and routes to plain-HTTP `/v1/chat/completions` agents | https://docs.planoai.dev/guides/orchestration.html | context (agent-level routing) | +| Three-mode LLM routing with automatic fallback: exact provider/model, semantic aliases, or routing preferences (4B router), first model primary and rest fallbacks on 429/5xx | https://docs.planoai.dev/guides/llm_router.html | #1, #13 | +| Ordered guardrail filter chains: MCP- or HTTP-filter services that inspect/mutate/short-circuit (4xx = policy violation before the agent/LLM) | https://docs.planoai.dev/concepts/filter_chain.html | #7 | +| Token-budget rate limits keyed on arbitrary header selectors (or `*` wildcard) via the `governor` crate | https://docs.planoai.dev/resources/configuration_reference.html | #3 | +| Cheapest/fastest selection policy from live cost + latency catalogs (models.dev / custom / Prometheus query, refreshed) | https://docs.planoai.dev/resources/configuration_reference.html | #13, new — synced price catalogs | +| Model-affinity session pinning (`X-Model-Affinity`) with in-memory or Redis cache for multi-replica deployments | https://docs.planoai.dev/guides/llm_router.html | #14 | +| Agentic Signals: model-free behavioral quality detectors (3 layers, 7 categories) emitted as OTel attributes — loop detection incremental O(1) | https://docs.planoai.dev/concepts/signals.html | context | +| OpenTelemetry tracing with sampling, custom span attributes, exporters (incl. PostHog), traceparent propagation, `planoai trace` inspector | https://docs.planoai.dev/resources/configuration_reference.html | #11 | +| Prometheus metrics (:19901/stats, brightstaff :9092 RED + LLM TTFT/per-token metrics) plus a shipped self-contained Prometheus+Grafana compose with a 19-panel dashboard | https://docs.planoai.dev/guides/observability/monitoring.html | #12, #17 | +| Envoy-format access logs split into ingress / LLM / internal streams | https://docs.planoai.dev/guides/observability/access_logging.html | context | +| Ingress breadth: `/v1/chat/completions`, `/v1/responses`, `/v1/messages` cross-provider with streaming in both SDKs | https://docs.planoai.dev/concepts/llm_providers/client_libraries.html | #6 | +| Ten first-class providers + `provider/*` wildcard auto-configuration of every offered model, per-provider passthrough auth and custom headers | https://docs.planoai.dev/concepts/llm_providers/llm_providers.html | context | +| Persistent conversation state in Postgres backing `/v1/responses` continuation + `overrides.optimize_context_window` history trimming | https://docs.planoai.dev/resources/configuration_reference.html | new — payload cost transforms, context | +| MCP-native filter ecosystem and typed declarative `prompt_targets` (local nearest-neighbor intent matching via `acap`) | https://github.com/katanemo/plano/blob/main/crates/llm_gateway/Cargo.toml | #8 | +| planoai CLI lifecycle + single Docker image + K8s deployment guidance (rollout-restart on config change) | https://docs.planoai.dev/resources/deployment.html | context | + +**Absent (verified).** No client auth/keys/admin API (any api_key accepted +locally), no usage ledger or billing-grade query API, no response caching or +dedup, no token-count endpoint or embeddings/rerank/audio/image families, +no hot config reload, no documented egress forward proxy or listener TLS; +`crates/prompt_gateway`/`hermesllm` contents unverified. + +## AISIX (api7) + +**Identity.** Rust-native Apache-2.0 AI gateway from API7 (the original +Apache APISIX creators) — an 18-crate Cargo workspace built as the OSS data +plane of commercial AISIX Enterprise/Cloud; single `aisix` static binary +proxying OpenAI- or Anthropic-format clients to OpenAI, Anthropic, +Gemini/Vertex, Bedrock, Azure OpenAI and any OpenAI-compatible upstream; +declarative `resources.yaml` (JSON-Schema checked, SIGHUP/etcd hot reload), +read-only admin listener, Prometheus listener, Docker image, small CLI. + +**Maintenance.** Very active — created 2026-04-17, last push 2026-11-04, 987 +commits; GA v1.0.0 2026-09-04, ten releases in eight weeks (v1.8.0, +2026-10-29); 212 stars / 48 forks (young, single corporate backer). + +**Sources.** Stats and activity: https://github.com/api7/aisix (repo page: +created 2026-04-17, 987 commits) and https://github.com/api7/aisix/releases +(v1.0.0 2026-09-04 through v1.8.0 2026-10-29). + +| Capability | Evidence | Gap | +|---|---|---| +| Six multi-target routing strategies (failover default, smooth-weighted RR, consistent hash, least-cost/latency/busy) with priority tiers, retry knobs, `retry_on_429`, `fallback_on_statuses`, per-model cooldowns and background health ejection | https://raw.githubusercontent.com/api7/aisix/main/README.md | #1, #13 | +| Multi-granularity rate limits: per-key/per-model rps/rpm/rph/rpd + tpm/tpd + in-flight concurrency, 429 pre-call with Retry-After and X-RateLimit-* headers; per-process or Redis (single/cluster/sentinel) shared counters, fail-open on Redis loss | https://raw.githubusercontent.com/api7/aisix/main/config.example.yaml | #3 | +| Caller API keys stored as SHA-256 hashes with RFC 3339 expiry, disable/enable, glob model allowlists, two-key rotation path, `x-aisix-api-key` header alternative | https://raw.githubusercontent.com/api7/aisix/main/README.md | #9 | +| OIDC/JWT caller authentication: JWKS (RS/PS/ES/EdDSA) or HMAC, aud/nbf/exp validation, required scopes, bound claims, priority-ordered claim→key mapping, coalesced JWKS refresh with stale-while-refresh | https://raw.githubusercontent.com/api7/aisix/main/README.md | new — caller SSO/RBAC | +| Layered guardrails at input/output/both: keyword/regex/semantic/PII built-ins plus Presidio, Lakera, OpenAI Moderation, Bedrock, Azure, Alibaba, custom scripts; monitor mode | https://raw.githubusercontent.com/api7/aisix/main/README.md | #7 | +| Response cache: exact-match of non-streaming chat completions on a normalized key (temperature compared at 0.001) with memory/Redis backends, TTL, applies_to selectors, purge generations, Cache-Control bypass; semantic layer on Redis 8+ search; automatic Anthropic prompt-cache breakpoints | https://raw.githubusercontent.com/api7/aisix/main/config.example.yaml | #10, #14 | +| MCP gateway: aggregated `ANY /mcp` and `/mcp/{server}` with gateway-held credentials, OpenAPI→MCP exposure, stateless 2026-07-28 revision, MCP conformance suite as merge-blocking CI | https://raw.githubusercontent.com/api7/aisix/main/README.md | #8 | +| A2A agent gateway at `/a2a/{agent}` (JSON-RPC + SSE, per-agent protocol pin, card URL rewriting, shared key governance) | https://raw.githubusercontent.com/api7/aisix/main/README.md | context | +| Broad endpoint families (chat, completions, responses, messages + count_tokens, embeddings, rerank, systemone/decisions, images, async video, audio, files/batches/fine-tuning, realtime WS, `/passthrough/:provider/*`) and five native adapter families (Bedrock SigV4, Vertex OAuth2, Azure) | https://raw.githubusercontent.com/api7/aisix/main/README.md | #6, #4 + endpoints future work | +| Observability: dedicated `:9090/metrics` with request- vs attempt-level counters (successful_fallbacks, TTFT/e2e histograms, auth decisions), per-attempt usage events with cache_status and error classes, OTLP GenAI/Datadog/SLS/S3 exporters with bounded queue, pprof and heap dumps | https://raw.githubusercontent.com/api7/aisix/main/README.md | #11, #12, #17 | +| Ops hardening: thread-per-core with SO_REUSEPORT, multi-listener TLS + PROXY protocol v1/v2, real_ip trusted CIDRs, outbound SSRF protection, upstream pool tuning incl. mTLS, SIGHUP/etcd hot reload, graceful drain, <10 MiB idle memory | https://github.com/api7/aisix/releases | #16 + context | +| Ensemble models (panel + judge) and semantic (embedding) routing incl. `jev` classifier mode | https://raw.githubusercontent.com/api7/aisix/main/README.md | #13 | +| Lock-free hot-path config reads via ArcSwap snapshots with reclamation of replaced snapshots | https://raw.githubusercontent.com/api7/aisix/main/crates/aisix-core/src/snapshot.rs | context | +| Budgets and spend caps — explicitly Enterprise/Cloud-only ("not part of the AISIX OSS resource model") | https://github.com/api7/aisix/releases | #2 (gated) | +| `aisix validate` offline JSON-Schema validation, read-only admin listener with OpenAPI 3 + Scalar UI, 183-scenario e2e suite, bench directory, MCP conformance harness | https://raw.githubusercontent.com/api7/aisix/main/README.md | context | + +**Absent (verified).** OSS is data-plane only — budgets/RBAC/audit/encrypted +provider keys/control-plane dashboard are Enterprise/Cloud; no local usage +store or usage-query API in OSS (usage leaves via exporters only — llm-proxy's +SQLite ledger + usage API is a differentiator — +`crates/llm-proxy-storage/src/sqlite.rs:105-131`, +`crates/llm-proxy-api/src/routes.rs:53-55`); exact cache covers only +non-streaming chat completions; token windows only per-minute/per-day, fixed +windows, Redis-outage fail-open; legacy HTTP+SSE MCP unsupported; MCP serving +stateless (no session state); A2A guardrails input-only; roadmap-not-shipped: +prompt templates, scheduled key rotation, Vault/KMS, SAML, alerting, SDKs. + +## mcp-gateway (MikkoParkkola) + +**Identity.** Self-hosted MCP gateway/proxy in Rust shipped as a single +~12–13 MB binary (plus crates.io, Homebrew, npm, Docker/Helm), sitting +between an AI client and many backend MCP servers/REST APIs behind a +9–17-tool "meta-tool" surface with on-demand discovery (or a 2-tool Code +Mode). **PolyForm Noncommercial 1.0.0** — strictly an ideas reference for +llm-proxy, never code reuse (MIT versions were deprecated/yanked). + +**Maintenance.** Very active — last commit 2026-11-02, ~3,191 commits, +releases every 1–4 weeks (stable v3.5.1 2026-09-04; v4.0.0-beta.3 +2026-10-06); 85 stars, single author. + +**Sources.** Stats and activity: https://github.com/MikkoParkkola/mcp-gateway +(repo page; ~3,191 commits) and +https://github.com/MikkoParkkola/mcp-gateway/releases (v3.5.1 2026-09-04, +v4.0.0-beta.3 2026-10-06). + +| Capability | Evidence | Gap | +|---|---|---| +| Meta-tool MCP gateway surface with on-demand tool discovery (gateway_search_tools/gateway_invoke), aggregating stdio/Streamable-HTTP/SSE/WebSocket backends and REST APIs; A2A adapter | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md | #8 | +| Fail-closed tool-poisoning and prompt-injection validator scanning every backend tool description (HIGH patterns fail closed; response-injection scanner; firewall JSON-RPC −32002) | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/security-controls.toml | #7, #8 | +| SHA-256 capability pinning with rug-pull quarantine: 110+ built-in capability YAMLs, pinned files failing closed on mismatch, changed tool definitions trigger unload-and-quarantine | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md | #8 | +| End-user identity propagation: gateway-signed ES256 assertions, client-token passthrough, and RFC 8693 token exchange, all fail-closed; per-user OAuth isolation; RFC 8707 resource indicators | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/CHANGELOG.md | new — caller SSO/RBAC (identity axis) | +| Per-slot circuit breakers (per backend and per-user session-pool slot) plus a tunable business-logic error-budget kill switch revivable only via a meta-tool; 429 excluded from budget accounting | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/ARCHITECTURE.md | #1 | +| Per-client rate limiting with burst bucket (token bucket per caller key, not per-IP) | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml | #3 | +| Cost governance with blocking tiers: currency, daily/per-tool/per-key budgets, 50/80/100% alert tiers where 100% blocks, `costs.json` persistence with anti-tamper hardening, cost-report meta-tool | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml | #2 | +| SHA-256-keyed response cache plus three-state idempotency cache (in-flight → completed, SHA-256 over server:tool + canonical args, TTLs, failure eviction) and prompt_cache_key preservation | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/ARCHITECTURE.md | #10 | +| ~8 ms hot reload of backends and config without dropping connections (notify + meta-tools, transition tracker merging in-memory state) | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md | #16 | +| Key server: SHA-256 hashed expiring API keys with OIDC, policies, admin token; single-use dashboard bootstrap link; agent auth | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml | #9 | +| Hash-chain transparency audit log of tool invocations, tested Prometheus alert rules, stats meta-tool | https://github.com/MikkoParkkola/mcp-gateway/releases | #12 (metrics.rs counters and shipped Prometheus alert rules; the hash-chain audit trail itself is comparison context) | +| SSRF guard with resolves-then-pins DNS (rebinding defense), special-use IP blocking, cleartext-credential refusal at config load, Origin/Host cross-site validation, mTLS, HMAC+nonce message signing | https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/security-controls.toml | context | +| Full deployment stack: Docker (incl. `-full` with Node/uv/git/ssh), Helm, K8s manifests, Prometheus; setup wizard importing Claude/Cursor/Zed configs, `doctor`, web UI/dashboard | https://github.com/MikkoParkkola/mcp-gateway/tree/main/deploy | #19 | + +**Absent (verified).** It is an MCP tool gateway, not an LLM API proxy — no +chat/messages/models client endpoints, no wire translation, no SSE token +translation, no embeddings/audio/image/batch, no offline token counting; no +SQL usage ledger (JSON + meta-tools only), no egress forward proxy observed, +no per-model aliasing/failover; "retry forwarding not yet implemented" for +the stateless 2026-07-28 protocol per its own changelog. + +## ThinkWatch (ThinkWatchProject) + +**Identity.** Rust Cargo workspace (server, gateway, mcp-gateway, auth, +common, test-support) with a React 19 console — a self-hosted dual-port +"bastion host for AI access": AI+MCP gateway on :3000, management +console/admin API on :3001, deployed via Docker Compose or Helm. Source +available under **BSL 1.1** (free up to 10M billable tokens + 10k MCP calls +per month; GPL-2.0+ changeover announced but not in effect). + +**Maintenance.** Very active — repo created 2026-04-02, 913 commits, last +push 2026-11-04, v3.11.0 released 2026-11-03 after ~12 releases in the +prior month; 814 stars. + +**Sources.** Stats and activity: +https://github.com/ThinkWatchProject/ThinkWatch (repo page; 913 commits, +created 2026-04-02) and +https://github.com/ThinkWatchProject/ThinkWatch/releases (v3.11.0, +2026-11-03); the run used the GitHub API and the releases page. + +| Capability | Evidence | Gap | +|---|---|---| +| OIDC/SSO sign-in (Zitadel, Okta, Entra, Keycloak, Auth0) with JWT access + rotating refresh tokens, replay detection, IP-bound sessions, Argon2id passwords | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | new — caller SSO/RBAC | +| RBAC with five built-in roles (admin/operator/user/viewer/service) + custom roles over models, tools, and admin pages, incl. MCP tool-level access control | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | new — caller SSO/RBAC | +| `tw-` virtual keys stored as SHA-256 hashes, scoped to AI and/or MCP gateway, with rotation grace periods, inactivity timeouts, per-key limits | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | #9 | +| MCP gateway where tool calls run as the real user: per-user OAuth/token connections so upstream audit logs show who acted; unconnected callers get the authorization URL; AES-256-GCM tokens at rest | https://github.com/ThinkWatchProject/ThinkWatch/tree/main/crates/mcp-gateway/src | #8 | +| PII/secrets redaction with placeholder restore in streams: 131 rules in 20 groups, provider keys recognized by value, per-value exceptions | https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.11.0 | #7 | +| Dangerous-command and content-filter guards on requests and streamed answers (held-back tail matching, refuse/delete/replace/record actions) | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#behaviour-worth-knowing | #7 | +| Sliding-window rate limits (requests or tokens, 1 min–1 week) and spend budgets (day/week/month) on users/keys/roles with 50/80/95/100% alerts, 429 + Retry-After, fail-open default with fail-closed switch, client-facing `GET /v1/usage` | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#behaviour-worth-knowing | #2, #3 | +| Cost accounting with chargeback CSVs, month-end forecast, cost centers, per-model quota weights | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | #2 | +| ClickHouse audit trail (six tables, 90/30-day TTLs) of every model request and tool call with optional pre-redacted bodies, SIEM forwarding (Syslog/Kafka/signed webhooks), S3 offload >256 KiB | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/.env.example | context | +| Weight/latency/health routing with a `tw-breaker` circuit breaker and per-route sliding-minute RPM/TPM caps pushing traffic to the next route | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | #1, #13 | +| Transient upstream retry window (default 120 s) before any content reaches the client, with reasoning-effort auto-adaptation on rejection | https://github.com/ThinkWatchProject/ThinkWatch/releases | #1 | +| Provider-encrypted conversation pinning: conversations whose history a provider encrypted stay pinned 30 days with a clear SealedHistoryRefused error otherwise | https://github.com/ThinkWatchProject/ThinkWatch/releases | #14 | +| Four client dialects on one port (OpenAI Chat, Responses, Anthropic Messages, Gemini) translated to OpenAI/Anthropic/Google/Azure/Bedrock (SigV4 + event-stream decoding) upstreams | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights | #6, #4 | +| Bearer-gated `/metrics` endpoint, unmounted when the token is unset | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/.env.example | #12 | +| Docker Compose + Helm deployment, distroless images, CPU/memory-adaptive execution, runbooks | https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#deployment | context | + +**Absent / caveats (verified).** BSL 1.1, not OSI open source; README claims +TOT but the security doc does not document it (treat MFA as unverified); no +in-process TLS termination; chat-only protocol breadth (no embeddings, +rerank, transcription/TTS, image/video, batch, realtime — the same families +llm-proxy defers); no SAML (OIDC only); advertised LLM response caching +unverified (cache.rs purpose not read); heavy footprint (PostgreSQL + Redis ++ ClickHouse + optional S3). + +## preflight (ghuntley) + +**Identity.** Rust single-crate (edition 2024, axum 0.8 + tokio + reqwest + +rusqlite, ~3.6k lines) local HTTP inspection proxy that scans +OpenAI-compatible LLM requests and attachments for secrets before they +reach the model, replacing findings with stable `[REDACTED:rule]` +placeholders; axum server (127.0.0.1:8081) + CLI, a Bubblewrap-sandboxed +attachment worker, a Gitleaks rules updater, Nix flake + hardened NixOS +service module. By Geoffrey Huntley; the outbound DLP stage in front of his +`ghuntley/underclass` proxy. MIT. + +**Maintenance.** Effectively dormant human development — the entire +implementation landed in 5 commits on 2026-09-20 (repo creation day), no +release/tag/CHANGELOG, no human commits since (two unmerged dependabot PRs); +the weekly rules workflow still runs green (latest success 2026-11-02) +without needing commits; 123 stars. + +**Sources.** Stats and activity: https://github.com/ghuntley/preflight +(repo page; five commits on 2026-09-20), the run's clone of main @ eb13fc0, +and workflow runs at +https://github.com/ghuntley/preflight/actions/workflows/rules.yml (green +through 2026-11-02). + +| Capability | Evidence | Gap | +|---|---|---| +| Request-path DLP secret scanning with the full Gitleaks rule DB (pinned v8.30.1) compiled in, gated by an explicit 18-rule profile; keyword prefilter → regex → entropy → SHA-256 allowlist carnet | https://github.com/ghuntley/preflight/blob/main/src/scanner.rs | #7 | +| Stable-placeholder redaction with redact / no-go (409) / advisory policy modes; un-rewritable findings fail closed; error bodies carry opaque finding UUIDs, never the secret | https://github.com/ghuntley/preflight/blob/main/src/main.rs | #7 | +| Local OCR and attachment inspection (JPEG/PNG/APNG/PDF, barcodes, metadata) via Tesseract/Poppler/QPDF/ExifTool/ZBar behind typed argv adapters; rebuilt artifacts re-inspected before approval | https://github.com/ghuntley/preflight/blob/main/src/attachments.rs | context (OCR is scoped future work in endpoints.md) | +| Bubblewrap-sandboxed, networkless worker processes (--unshare-all, tmpfs, 120 s, 16 MiB stdout cap); unknown formats and PDF active content fail closed | https://github.com/ghuntley/preflight/blob/main/docs/adr/0002-rust-workers-and-content-cache.md | context | +| Content-addressed verdict/artifact cache (memory + WAL SQLite with secure_delete) binding digest + credential scope + profile fingerprint, single-flight, generation-based purge | https://github.com/ghuntley/preflight/blob/main/src/cache.rs | context | +| Live cache administration (`status|purge [--scope]`) over a mode-0600 Unix control socket | https://github.com/ghuntley/preflight/blob/main/src/main.rs | context | +| SSRF-hardened remote materialization: https-only, no redirects/proxy, DNS resolved once and pinned, all IPs must be global-unicast, byte budgets, then replaced with inline data: URLs | https://github.com/ghuntley/preflight/blob/main/src/resolver.rs | context | +| Concurrency admission control: 16-permit semaphore returning 503 `inspection_capacity`; 2 attachment/4 scan worker slots | https://github.com/ghuntley/preflight/blob/main/src/main.rs | #3, new — admission queueing (backpressure form) | +| Prometheus `/metrics` with lock-free counters and a request-to-headers histogram; aggregate labels only | https://github.com/ghuntley/preflight/blob/main/src/metrics.rs | #12 | +| Secret-free structured JSON logging contract + cross-proxy x-request-id correlation (ingress-minted UUIDv4 propagated upstream) | https://github.com/ghuntley/preflight/blob/main/src/correlation.rs | context | +| SIGHUP atomic config reload with immutable runtime snapshots (failed reload keeps the previous runtime; admitted requests keep their snapshot) | https://github.com/ghuntley/preflight/blob/main/src/main.rs | #16 | +| Split credentials: optional client key, upstream key substitution after hop-by-hop stripping, separate file-API key | https://github.com/ghuntley/preflight/blob/main/src/main.rs | context | +| NixOS hardened service module (DynamicUser, ProtectSystem=strict, MemoryMax…) + QEMU VM test; weekly fully-gated automated Gitleaks rules updates committed with provenance | https://github.com/ghuntley/preflight/blob/main/nixos/module.nix | context | + +**Absent (verified).** OpenAI-shaped passthrough only (no `/v1/messages`, no +translation), hard-wired to a single "underclass" upstream; no +retries/failover/multi-provider routing (explicitly delegated); no +response-side inspection (upstream bodies stream through uninspected); no +rate limiting per client/IP, quotas, budgets, cost tracking, or usage +history; single shared bearer key compared non-constant-time (no key ids, +rotation, or hashing); no MCP; Linux-only, Nix-centric; runtime behavior +against a live provider not verified. + +## tokdiet (agiwhitelist) + +**Identity.** TypeScript/Node 20 CLI (`npx tokdiet start`, alias `td`) +running a local, loopback-only streaming reverse proxy on :7787 with a live +dashboard on :7878 between AI coding agents (Claude Code, Cursor, Codex, +opencode, Aider) and model APIs (Anthropic, OpenAI, Gemini, MiniMax); it +meters tokens/USD and compacts bloated agent context by recoverably paging +out cold tool results. Also a Claude Code marketplace plugin (metering hook +only). Born as "Context Governor" (legacy CTXGOV_* names survive). + +**Maintenance.** Dormant — 12 commits in a 3-day AI-assisted burst +(2026-06-16/18), one npm release (0.1.0, 2026-06-16), no releases/tags since, +one unanswered issue (a pricing bug, Oct 2026); 32 stars. Treat as a +proof-of-concept snapshot. + +**Sources.** Stats and activity: https://github.com/agiwhitelist/tokdiet +(repo page; 12 commits, 2026-06-16/18) and +https://www.npmjs.com/package/tokdiet (0.1.0, 2026-06-16). + +| Capability | Evidence | Gap | +|---|---|---| +| Fail-open mutating streaming proxy core: every stage individually guarded so any internal error degrades to byte-transparent passthrough ("the proxy never breaks the user's workflow") | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/proxy.ts | new — payload cost transforms, context | +| Context "virtual memory" compaction: hot/warm/cold tiers, loss-free dedup, recoverable elision (240-char previews + 12 salient lines, originals persisted to SQLite), optional mid-summarize; published −71% input tokens | https://github.com/agiwhitelist/tokdiet/tree/main/src/compactor | new — payload cost transforms | +| Shadow-eval quality guard: ~5% of compacted requests re-sent uncompacted and scored, 50-sample rolling average vs a 2% max-degradation budget, tripping overall or per-strategy safe-mode; priors persisted across restarts | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/quality/guard.ts | new — shadow-eval quality guard | +| Published 66-task A/B quality-parity benchmark harness: paired governed/ungoverned runs against a real model, objective grader + LLM judge, majority voting, per-category regression list | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/bench/run.mjs | context | +| Page-fault detection (model echoing paged-out `cg-` ids or complaining) with one-shot blob re-injection | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/pagefault.ts | new — payload cost transforms | +| Hierarchical USD budget gate: session → day (UTC) → repo-month precedence, warn/compact/block actions (block = 429) | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/budget.ts | #2 | +| Cost metering from an out-of-band SSE usage tee with a user-editable, hot-reloaded pricing.json incl. cache read/write rates; keys forwarded, never persisted | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/ARCHITECTURE.md | new — synced price catalogs | +| Anthropic prompt-cache-aware and thinking-safe mutation guards: never rewrite at/before a cache_control breakpoint (would invalidate ~0.1×-priced prefixes); never mutate signed/thinking blocks | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/README.md | #14 (safety constraint for any transform) | +| Live SSE dashboard (five screens: session, savings, quality, tool/repo, strategy leaderboard) over better-sqlite3 | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/ARCHITECTURE.md | #19 | +| Claude Code plugin: PreToolUse/PostToolUse metering hooks appending JSONL records; `/tokdiet` command; never blocks a tool | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/hooks/tool-meter.mjs | context | +| Four provider adapters with per-provider upstream override env vars; unknown JSON fields pass through untouched | https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/README.md | context | + +**Absent (verified).** No auth (127.0.0.1 bind is the whole security model), +no retries/fallback/load balancing (one fixed upstream per provider), no +rate limiting, no response caching (preserves upstream prompt-cache prefixes +only), no MCP, no Prometheus/request-id/configurable logging, no cursor-paged +usage API; page-fault recovery impossible for streaming; embedding judge +unimplemented; unknown models silently cost $0.00 (open pricing bug); +dormant. + +## CLIProxyAPI (router-for-me) + +**Identity.** Go 1.26 single-binary proxy server (plus embeddable Go SDK and +an optional desktop client) that turns coding-CLI subscriptions — ChatGPT +Codex, Claude Code, Gemini CLI, Antigravity, AI Studio, Kimi, Grok Build, +Meta Muse Code, Devin — into OpenAI/Gemini/Claude/Codex/Grok-compatible +local APIs with multi-account OAuth credential pools. MIT; 57.5k stars / +8.8k forks / 4,709 commits; large third-party ecosystem. + +**Maintenance.** Extremely active — last commit and v8.0.54 on 2026-11-04 +(research day); near-daily releases. Core value prop (impersonating official +CLI clients against subscription backends) is ToS-gray: treated here as a +pattern reference, not an integration target. + +**Sources.** Stats and activity: +https://api.github.com/repos/router-for-me/CLIProxyAPI (stars/forks/commits, +read 2026-11-04), https://github.com/router-for-me/CLIProxyAPI (the run's +shallow clone at main HEAD 78a9cc9), and +https://github.com/router-for-me/CLIProxyAPI/releases (v8.0.54, +2026-11-04). + +| Capability | Evidence | Gap | +|---|---|---| +| Per-provider OAuth login flows: browser PKCE, device-code, service-account import, SIWC binding, headless Management-API auth URLs | https://github.com/router-for-me/CLIProxyAPI/blob/main/cmd/server/main.go | new — OAuth credential lifecycle | +| Multi-account credential pools with three rotation strategies (round-robin with identity-based rotation, smooth weighted, fill-first to stagger rolling-window caps), priority tiers, prefix pinning | https://github.com/router-for-me/CLIProxyAPI/blob/main/sdk/cliproxy/auth/selector.go | #5 | +| Credential-level retry rounds (default 3) with per-round credential caps, cooldown backoff levels, model-scoped cooling, and custom error classification (stop / stop-and-cooldown / continue) | https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml | #1, #5 | +| Session-affinity sticky routing for prompt/KV-cache reuse: session headers, prompt_cache_key, Responses conversation IDs, or first-message hash; subagents inherit the parent's credential | https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml | #14 | +| Passive upstream quota observation from response headers (Retry-After watermarks, limit-reached flags) kept separate from cooldown state; live quota fetch/reset API | https://github.com/router-for-me/CLIProxyAPI/blob/main/sdk/cliproxy/auth/quota_signals.go | #5 | +| Very wide client + upstream surface: chat/completions, completions, responses (HTTP + WebSocket + compact), messages + count_tokens, images, video, TTS, `/v1/live` realtime incl. WebRTC, Gemini native, Codex aliases | https://github.com/router-for-me/CLIProxyAPI/blob/main/internal/api/server_routes.go | #6 + endpoints future work | +| v8 Management API: GET/PUT/PATCH/DELETE over the whole YAML tree with comment-preserving PATCH, auth-file upload/download/refresh, ~80 runtime toggle routes, auto-downloaded web panel | https://github.com/router-for-me/CLIProxyAPI/blob/main/docs/management-api-v8.md | #19, #16 | +| Cluster mode with a distributed credential-lease control plane ("Home"): JWT/mTLS bootstrap, heartbeats, fences, reclaim grace so nodes share one pool without double-spending | https://github.com/router-for-me/CLIProxyAPI/blob/main/docker-compose.cluster.yml | context (multi-instance) | +| Pluggable credential token stores: file, Postgres, Git repo, S3/MinIO registered through one interface | https://github.com/router-for-me/CLIProxyAPI/blob/main/cmd/server/main.go | context | +| Payload rewrite rules engine: set-if-missing / always-overwrite / strip by JSON path with rich matchers (model, protocol, headers, payload conditions) | https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml | context (request policy layer) | +| Stream-bootstrap buffering: hold handshake/heartbeat frames (48-frame/1 MiB budget) so HTTP-200-embedded rejections can still fail over before headers commit | https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml | #1 (pre-commit fallback refinement) | +| Embeddable Go SDK + c-shared plugin system with a plugin store; TLS listener, zstd/gzip response compression, mDNS LAN discovery, pprof, TUI, example-key safe-mode | https://github.com/router-for-me/CLIProxyAPI/blob/main/docs/sdk-usage.md | context | +| Client fingerprint cloaking / CLI impersonation (pinned CLI wire fingerprints, uTLS, CCH signing, zero-width obfuscation) | https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml | context (ToS-gray; documented as pattern only) | + +**Absent (verified).** No persistent usage/cost ledger (built-in statistics +removed in v6.10.0, delegated to third-party tools; ~60 s in-memory queue +only), no built-in budgets/spend caps/pricing, downstream auth is a flat +plaintext api-keys list (no hashes, no per-key limits — llm-proxy's +argon2id + per-key attribution are stronger — `config.toml.example:42-54`), +no response caching, no +content guardrails, no MCP server, no embeddings/rerank/batch/files/ +fine-tuning. + +## sub2api (Wei-Shaw) + +**Identity.** Go 1.27 + Gin + Ent + Vue 3 multi-tenant +"subscription-to-API" gateway platform (PostgreSQL 15+, Redis 7+) that turns +pooled AI product subscriptions (Claude, Codex, Gemini, Grok, Antigravity, +CN providers) into distributable `sk-` keys with per-user billing, budgets, +payments (Alipay/WeChat/Stripe/EasyPay) and an account-pool scheduler. +LGPL-3.0; 45.4k stars. Its README carries a ToS-violation warning and it +ships provider-adversarial machinery — a **design reference for +account-pool mechanics, not a compliant integration target**. + +**Maintenance.** Extremely active — last push 2026-11-04, ~7,850 commits, +releases every 2–5 days (v0.2.19, 2026-10-28). + +**Sources.** Stats and activity: +https://api.github.com/repos/Wei-Shaw/sub2api (stars/forks/commits, read +2026-11-04) and https://github.com/Wei-Shaw/sub2api/releases (v0.2.19, +2026-10-28). + +| Capability | Evidence | Gap | +|---|---|---| +| Account-pool schedulability state machine: five independent time-based isolation states (overload, rate-limit reset, temp-unschedulable ± model scope, expiry, session windows) persisted as columns | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/account.go | #5 | +| Sticky-session scheduling with prompt-cache affinity: session key from client session IDs, else SHA-256 of `cache_control:{ephemeral}` content, else client digest; Redis binding with 1 h TTL; sticky wait queues; account switches reclassify cache billing | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/gateway_service.go | #14 | +| In-request failover loop keyed to 401/403/429: per-account retry counters, same-account retry delays/deadlines, switch budgets, mid-stream switching after SSE comments only; client disconnects classified 499 and stop failover | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/handler/failover_loop.go | #1, new — upstream cancellation (499 classification) | +| Status-code→account-health taxonomy: 401 OAuth cache invalidation + recoverable cooldown, 402 permanent stop, 429 5 h/7 d quota windows parsed from headers, 529 overload cooldown | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/ratelimit_service.go | #5 | +| Background OAuth token refresh with distributed lock and a stale-snapshot rollback guard; per-platform refreshers | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/token_refresh_service.go | new — OAuth credential lifecycle | +| Append-only token-level cost ledger: 5 m/1 h cache variants, per-kind USD costs, total vs actual (post-multiplier) with rate-multiplier snapshots, first-token latency, requested-vs-upstream model mismatch flag, billing circuit breaker | https://github.com/Wei-Shaw/sub2api/blob/main/backend/ent/schema/usage_log.go | #2 | +| Layered budgets: per-key 5 h/1 d/7 d USD windows, user subscription quotas, and Redis in-flight balance reservation (estimate = input + output price × max_tokens, TTL-renewed, fail-open on Redis loss) | https://github.com/Wei-Shaw/sub2api/blob/main/backend/migrations/064_add_api_key_rate_limits.sql | #2 | +| Per-user-per-group rate multipliers (billing markup), RPM overrides, and a scheduler profit gate with veto cap and anti-livelock handling | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/user_group_rate.go | context (reseller economics) | +| Remote pricing catalog (Wei-Shaw/model-price-repo) with SHA-256 verification, local fallback, override patch merge, 10-minute hash rechecks, hot reload without restart | https://github.com/Wei-Shaw/sub2api/blob/main/deploy/config.example.yaml | new — synced price catalogs | +| Redis concurrency slots per account and per user (sorted-set with expiry sweeps, stale cleanup) feeding multi-layer load-aware selection; SSE pings while queued | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/concurrency_service.go | #3 | +| Broad multi-protocol surface: messages(+count_tokens), responses(+subpaths), chat/completions, embeddings, models, usage/billing, `/v1/live` realtime, images (sync/async/tasks), videos, tts/stt/custom-voices, x_search, Gemini cachedContents CRUD | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/server/routes/gateway.go | #6 + endpoints future work | +| Management MCP server: stateless Streamable-HTTP endpoint exposing four immutable tool sets (user/admin × read/read-write) with bounded envelopes and per-tool annotations | https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/managementmcp/server.go | #8 | +| Synthetic channel monitors (probe/quota modes, AES-256-GCM probe keys, jittered intervals), async prompt-audit guard, dashboard aggregation caches | https://github.com/Wei-Shaw/sub2api/blob/main/backend/ent/schema/channel_monitor.go | context | +| Hardened admin tooling: scoped admin keys with expiry/IP allowlists, passkeys/TOTP/step-up, session fingerprint binding, OIDC connect, Idempotency-Key on admin writes, systemd/compose/Caddy deployment | https://github.com/Wei-Shaw/sub2api/blob/main/deploy/README.md | context | + +**Absent (verified).** No local response caching or request dedup +(idempotency middleware covers admin APIs only); not embeddable (Go +monolith, mandatory PostgreSQL + Redis, no single-binary embedded storage); +plugin surface limited to local OAuth transport plugins; no rerank or OCR; +provider-adversarial features (TLS fingerprint profiles, CLI mimicry, +per-account proxy rotation) make it non-compliant as an integration target; +deep behavior documented mostly in source comments (Chinese docs). + +## Claude Code Router (musistudio) + +**Identity.** TypeScript/Node.js local agent-gateway in three form factors: +Electron desktop app (macOS/Windows/Linux, v3.1.3), npm CLI (`ccr`, gateway +:3456, web UI :3458), Docker Compose — a "control plane" for coding agents +(Claude Code, Codex, Grok CLI, Kimi, OpenCode, Kilo, Pi, ZCode, WorkBuddy) +with routing rules, credential pools, MCP aggregation and usage analytics. +MIT; 38k stars; started 2025-02-25, the day after Claude Code's release. + +**Maintenance.** Active — last commit 2026-10-10, v3.1.3 (2026-10-09), ~30 +releases on a bug-fix-driven cadence; 950 open issues / 237 PRs show backlog +pressure. + +**Sources.** Stats and activity: +https://github.com/musistudio/claude-code-router (repo page: ~38k stars, 989 +commits, issue/PR counts) and +https://github.com/musistudio/claude-code-router/releases (v3.1.3, +2026-10-09). + +| Capability | Evidence | Gap | +|---|---|---| +| Ordered conditional routing rules with request rewriting: dot-path conditions with 9 operators, set/delete/append rewrites, built-in Claude Code detection and subagent auto-routing | https://ccrdesk.top/routing/ | #13 (policy routing) | +| Sandboxed Node.js route scripts (Worker with frozen input + fetch/fs/env API) with size/time/rewrite caps and circuit breaking after 3 failures in 60 s | https://github.com/musistudio/claude-code-router/tree/main/packages/core/src/routing | context (extensibility) | +| Retry policy + ordered model-chain fallback: off / retry-N (408/409/429/5xx + network) / model-chain on any 4xx-5xx; Retry-After honored, 1 s→30 s backoff, `x-ccr-fallback-model` header | https://github.com/musistudio/claude-code-router/releases | #1 | +| Upstream credential pools with per-key priority, weight and quota; rotation on refresh-401 (Kimi/Grok); macOS keychain lookup; optional balance/quota connectors | https://ccrdesk.top/guides/provider/ | #5 | +| Client access keys with expiration (never/7/30/90/custom), instant revoke on delete, per-key local quotas across requests/tokens/images × min/hour/day; constant-time compare | https://ccrdesk.top/configuration/api-keys/ | #9, #3 | +| Request logs with resolved route, credential, latency, tokens, cost, full bodies, and agent-execution-chain reconstruction | https://ccrdesk.top/guides/observability/ | context | +| Usage dashboard (time ranges, cache ratio, success rate, heatmap, model distribution) plus exportable PNG share cards | https://ccrdesk.top/configuration/overview/ | #19 | +| Five client-side protocols with any-to-any translation (OpenAI Chat/Responses, Anthropic Messages, Gemini generateContent/interactions) + auto endpoint protocol/model detection and presets | https://github.com/musistudio/claude-code-router/blob/main/packages/core/src/routing/protocol-endpoints.ts | #6 | +| ToolHub MCP aggregation with lazy tool resolution: `tool_hub.resolve` (retrieval model picks 1–20 tools) + `tool_hub.invoke`; stdio/streamable-http/SSE transports; built-in browser automation; session id forwarded as prompt_cache_key | https://ccrdesk.top/toolhub/ | #8, #14 | +| Fusion: composed virtual models bundling base + capability models (vision/search/image/video/MCP tools) as ordinary routing entries | https://ccrdesk.top/fusion/ | context | +| Desktop MITM proxy mode (CA-cert interception) for clients that ignore base-URL env vars; /health; admin token; model catalog CDN refresh | https://ccrdesk.top/configuration/server/ | context | +| AgentClaw chat-relay access to agents via Weixin/Slack/Discord/Telegram/LINE/Feishu/DingTalk | https://github.com/musistudio/claude-code-router/blob/main/README.md | context | +| Provider presets, deeplink/manifest import, connectivity check that sends real requests before saving | https://ccrdesk.top/guides/provider/ | context | + +**Absent (verified).** No native TLS termination (reverse proxy advised); +request logs retained same-day only (no configurable retention — llm-proxy's +max_age/max_rows pruning is an advantage — `config.toml.example:56-68`); +no response/prompt caching of its +own; no guardrails/content filtering; cost estimation-only; no embeddings, +rerank, batch, transcription/TTS, or realtime endpoints; hot-reload of +routing rules not documented. + +## Otari (mozilla-ai) + +**Identity.** Mozilla.ai's Apache-2.0, Python 3.13+ OpenAI-compatible LLM +gateway ("a gateway you own and run"): FastAPI-style async server in a +hexagonal `src/gateway` package over PostgreSQL/SQLite (SQLAlchemy/alembic), +a React dashboard, and a distributable `otari` CLI; Docker/Compose with +Render and Railway templates; three runtime modes (standalone, hosted +control plane at otari.ai, hybrid data plane). Built on Mozilla's any-llm +provider abstraction. + +**Maintenance.** Extremely active — repo created 2026-04-02, last push and +v0.37.0 both 2026-11-04, 13+ minor releases in the preceding three weeks; +835 stars. Still pre-1.0 with active schema churn (~90 alembic migrations). + +**Sources.** Stats and activity: https://github.com/mozilla-ai/otari (repo +page; 835 commits, created 2026-04-02) and +https://github.com/mozilla-ai/otari/releases (v0.37.0, 2026-11-04). + +| Capability | Evidence | Gap | +|---|---|---| +| Pre-dispatch budget holds: a request's worst case (prompt estimate + entire max_tokens output) held BEFORE dispatch in one conditional update; reconciled to measured cost after; crashed holds returned after a TTL | https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md | #2 | +| Scoped budgets: org/workspace/membership/key ceilings on USD, token and request axes plus rpm/tpm per user, rolling or UTC calendar resets, inert until a spend ceiling applies them | https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md | #2, #3 | +| Structured budget-refusal semantics: 403 with `Otari-Error-Code: budget_exceeded`, an `Otari-Budget-Scope` header naming the refusing budget, Retry-After until window end, and a body stating both amounts | https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md | #2 | +| Workspace-scoped virtual keys: per-key model allowlists (provider wildcards), tool allowlists, own budget ceilings stacking with the owner's, expiry, budget exemption, service-key flag; role-split issuance; regeneration kills the old secret immediately | https://github.com/mozilla-ai/otari/blob/main/docs/api-keys.md | #9 | +| Service keys with per-end-user spend, budgets and per-minute limits from a curated list; pseudonymous per-end-user provider identifiers recorded on usage rows | https://github.com/mozilla-ai/otari/blob/main/docs/end-users.md | new — caller SSO/RBAC (end-user identity axis) | +| Workload-identity federation: backends authenticate as a key with a platform OIDC identity token instead of the key secret | https://github.com/mozilla-ai/otari/blob/main/docs/api-keys.md | new — caller SSO/RBAC | +| Routing policies: on_failure failover (incl. 400/401/429 and empty choices), budget-conditional selection on `budget_used_pct`/`budget_remaining_usd`, weighted router, priority router skipping rate-full models; per-request override headers, error codes, `otari routing explain` CLI | https://github.com/mozilla-ai/otari/blob/main/docs/routing.md | #1, #13 | +| Automatic provider selection with sticky prompt-cache sessions: up to 32 offerings ranked by estimated cost, health, region, latency; colon spelling pins one provider; session bindings reuse provider+credential | https://github.com/mozilla-ai/otari/blob/main/docs/models.md | #14 | +| Multi-dimensional rate-limit rules: per key/user/end-user/deployment/model/budget with rpm, tpm and max_concurrent (15-min slots); `tpm_admission` holds prompt+max_tokens; report mode; rate_limit_store memory or Redis; live edits propagating within 30 s | https://github.com/mozilla-ai/otari/blob/main/docs/configuration.md | #3 | +| Managed MCP tool loop: inline or workspace-stored servers, gateway executes tool calls and re-calls the model (terminal last round without tools), encrypted tokens, SSRF-checked URLs, caller-orchestrated mode | https://github.com/mozilla-ai/otari/blob/main/docs/mcp.md | #8 | +| Layered inference guardrails callers cannot remove: caller opt-in, org-mandated, and operator-mandated on routing policies (block with fail-closed on unavailability) | https://github.com/mozilla-ai/otari/blob/main/docs/guardrails.md | #7 | +| Built-in tools: sandboxed code execution (sandbox_url or hosted e2b), web search and fetch with provider-native interception (auto|otari|provider) and max_uses caps | https://github.com/mozilla-ai/otari/blob/main/docs/tools.md | context | +| Agent traces with a context-window map (8 otari.context.* attributes per span), OTLP ingestion, opt-in sealed content capture, Prometheus `/metrics` | https://github.com/mozilla-ai/otari/blob/main/docs/traces.md | #11, #12 | +| Idempotency-Key replay of stored (encrypted) responses and `streaming_first_chunk_hold_ms` holding the status line for the provider's first chunk so pre-output failures return proper errors and can fall over | https://github.com/mozilla-ai/otari/blob/main/docs/configuration.md | #1 (safe-retry refinement) | +| Agent Guardrails: standalone `otari hook` binary wiring repo-level gates (paths, commands, verifiers, rubrics) into Claude Code/Codex hooks | https://github.com/mozilla-ai/otari/blob/main/docs/agent-guardrails.md | context | + +**Absent (verified).** No response caching (exact or semantic; prompt-cache +tokens tracked for accounting only), no per-IP rate limiting (everything +keyed to key/user/budget/model/deployment), no unauthenticated loopback mode +(bootstrap key minted on first boot; no `mode=none` equivalent — +`config.toml.example:42-54`), no +per-provider egress forward proxy documented, not embeddable and not Rust, +no cargo-dist-style single binary (Docker/uv/brew instead). + +## done-hub (deanxv) + +**Identity.** Single-binary Go (Gin) LLM relay gateway with an embedded +React web UI, forked from MartialBE/one-hub (one-api lineage) with drop-in +DB/Docker compatibility; its distinctive work is agent-CLI OAuth channel +types (Claude Code, Codex, Gemini CLI, Antigravity, Vertex AI Express), +cross-protocol native routing, and a long trail of quota-ledger correctness +fixes. Apache-2.0. + +**Maintenance.** Active — last push 2026-10-27, latest release v1.23.22 +(2026-09-29), ~40 tagged releases Sep 2025–Sep 2026; 472 commits, 815 stars. + +**Sources.** Stats and activity: https://github.com/deanxv/done-hub (repo +page; last push 2026-10-27) and https://github.com/deanxv/done-hub/releases +(v1.23.22, 2026-09-29). + +| Capability | Evidence | Gap | +|---|---|---| +| Multi-bucket cache-token usage ledger: 5 m/1 h cache-write buckets, cache-read, OpenAI cache_write_tokens (deliberately decoupled from Anthropic 1.25×), audio/image/text × input/output, reasoning tokens — each billable at its own ratio | https://github.com/deanxv/done-hub/blob/main/types/common.go | #2 (billing fidelity) | +| Documented quota-ledger correctness fixes incl. the cached_tokens `omitempty` bug (zero values dropped, strict clients retried in a loop); fix ships zero-value serialization tests | https://github.com/deanxv/done-hub/blob/main/README.md#L127-L133 | context (direct lessons for the usage ledger) | +| Flat Anthropic-style cache fields parsed in OpenAI-compatible usage as fallback billing (all cache-write billed at the conservative 5 m ratio) | https://github.com/deanxv/done-hub/commit/bbb2293e34 | context | +| Pre-consume + streaming realtime quota enforcement via a Redis INCRBY Lua script with TTL that can abort mid-stream at the quota; graceful skip when quota ≫ estimate | https://github.com/deanxv/done-hub/blob/main/relay/relay_util/quota.go | #2 | +| Batched DB quota writer with ordered graceful-shutdown flush (StopBatchUpdater before FlushAllBatches or the ticker swaps the map mid-write); int32→bigint quota columns | https://github.com/deanxv/done-hub/blob/main/model/utils.go | context | +| Agent-CLI OAuth channel types: Claude Code console OAuth, Codex OAuth with version-gated UA and per-model instruction files, Gemini CLI published credentials, Antigravity, Vertex Express | https://github.com/deanxv/done-hub/blob/main/providers/codex/base.go | new — OAuth credential lifecycle | +| Cross-channel retry bounded by live channel count with wall-clock retry timeout collapsing to 503, honest retry_skip/retry_timeout logs, and Retry-After-derived cooldowns taking precedence | https://github.com/deanxv/done-hub/blob/main/relay/main.go | #1 | +| Group-based channel distribution with token-pinned groups, backup groups and per-group billing ratios | https://github.com/deanxv/done-hub/blob/main/middleware/distributor.go | #13, #2 | +| API tokens with per-key quota, expiry, unlimited flag, batched used-quota metering | https://github.com/deanxv/done-hub/blob/main/model/token.go | #9, #2 | +| Multi-bucket per-user + per-IP rate limiting with CIDR whitelist and a CriticalRateLimit that whitelisting deliberately does NOT exempt (brute-force protection) | https://github.com/deanxv/done-hub/blob/main/middleware/rate-limit.go | #3 | +| Prometheus metrics with per-channel/model dimensions; upstream request-id correlation; panic counter | https://github.com/deanxv/done-hub/blob/main/metrics/main.go | #12 | +| Built-in MCP server (SSE at /mcp/message + Streamable HTTP) exposing gateway tools to agents | https://github.com/deanxv/done-hub/blob/main/mcp/server.go | #8 | +| Pluggable content-safety checker framework and MaxPromptTokens input guard (400 before any upstream call) | https://github.com/deanxv/done-hub/blob/main/safty/safty.go | #7 | +| Broad endpoint families across 45+ adapters (chat, completions, embeddings, rerank, realtime, transcriptions, speech, translations, moderations, images, Veo video, search, Midjourney tasks); long-context tiered pricing applied incrementally on streams; bulk price import from models.dev | https://github.com/deanxv/done-hub/blob/main/relay/relay_util/quota.go | context (endpoints.md-scoped families; no Responses surface cited), new — synced price catalogs | + +**Absent / caveats (verified).** No request dedup window (llm-proxy has +one — `crates/llm-proxy-server/src/routes/core_pipeline.rs:182-185`); logs +full request/response bodies by default (llm-proxy's digests-only +posture is stronger privacy — `crates/llm-proxy-storage/src/sqlite.rs:130-131`); no offline token counting (billing trusts +upstream-reported usage — the source of its cache-token bugs); realtime +quota enforcement silently degrades to post-hoc billing when Redis is +disabled; full multi-user web-app attack surface (payments, invite/rebate, +OAuth trust gating) llm-proxy deliberately avoids; the exact GLM float-token +fix commit could not be located (README line is the only citable evidence). + +## yolorouter + +**Identity.** Self-hosted, OpenAI-compatible LLM gateway in Go (1.25.7+) +shipped as a single server binary with an embedded multi-user web admin +console (serve/stop/update subcommands); SQLite by default, optional +PostgreSQL. Apache-2.0. Launched July 2026 as a failover gateway for AI +coding CLIs; went multi-protocol with cost-optimization transforms in v0.2.0. + +**Maintenance.** Active — 606 commits, last commit 2026-10-28 (verified in +the research run's shallow clone, HEAD 7ac9750); twenty releases at roughly +weekly cadence, v0.1.0 (2026-07-21) → v0.2.9 (2026-10-27); 254 stars. + +**Sources.** Stats and activity: https://github.com/yolorouter/yolorouter +(repo page: 606 commits) and the run's shallow clone at +https://github.com/yolorouter/yolorouter/tree/7ac9750 (HEAD verified +2026-10-28); releases at https://github.com/yolorouter/yolorouter/releases. + +| Capability | Evidence | Gap | +|---|---|---| +| Upstream key pool with rate-limit benching: plain 429 benches a key for its Retry-After clamped 1 s–10 min; benched keys demoted (soonest-to-recover first), never excluded — an all-cooling pool still dispatches | https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/keypool.go | #5 | +| Background key auto-recovery loop (30 s heartbeat) retesting system-kicked keys and releasing benches without admin action | https://github.com/yolorouter/yolorouter/blob/main/internal/keyrecovery/loop.go | #5 | +| Multi-provider failover with a generation-guarded three-state circuit breaker (soft half-weight faults, time-based probe admission, generation token so pre-transition results cannot mis-book state); key rotation and failover complete before the first byte | https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/circuit/circuit.go | #1 | +| Balanced scheduling mode with per-key prompt-cache affinity: caller keys spread evenly and stuck to one provider so upstream prompt caches stay warm | https://github.com/yolorouter/yolorouter#scheduling-modes | #14 | +| Request-path tool-output compression: anchored-regex detection of go test/pytest/vitest/npm/pnpm logs, git diffs, grep results through a most-specific-first compressor chain, live-zone protected, replace-only-if-shorter, fail-open | https://github.com/yolorouter/yolorouter/blob/main/internal/compress/compress.go | new — payload cost transforms | +| Custom system-prompt injection ("house rules") implemented per egress protocol shape, deterministic to preserve prompt caches, with a published 150-pair on/off benchmark (median +12.6% output reduction, negatives reported honestly) | https://github.com/yolorouter/yolorouter/blob/main/internal/capability/systemprompt/inject.go | new — payload cost transforms | +| Per-caller key governance: budgets (micros), RPM/TPM/concurrency, model allowlists, expiry, instant revocation; media jobs reserve priced bounds against the budget while in flight | https://github.com/yolorouter/yolorouter/blob/main/internal/service/apikey/api_key_service.go | #2, #3, #9 | +| Four chat wire protocols any-to-any (OpenAI Chat/Responses, Anthropic Messages, Gemini) plus images, a pollable video job dialect, and speech; native media providers | https://github.com/yolorouter/yolorouter#protocols | #6 + endpoints future work | +| Seven-phase configurable relay timeouts (connect, TLS, header, first-byte, body-idle, attempt, request) replacing a single wall clock so long-reasoning models are not cut off; ordering validated at startup | https://github.com/yolorouter/yolorouter/blob/main/internal/config/config.go | new — timeout hierarchy | +| Community price catalog (github.com/yolorouter/price-catalog) with one-click model import, background verification of imported mappings (write-nothing probes, vision detection), unpriced candidates never auto-enabled | https://github.com/yolorouter/yolorouter/blob/main/CHANGELOG.md | new — synced price catalogs | +| Request logs with full per-attempt routing chains, final upstream URL, compress skip reasons, and per-modality billing snapshots; console shows full client and upstream bodies | https://github.com/yolorouter/yolorouter/blob/main/internal/model/request_log.go | context, #17 | +| W3C traceparent propagation into the audit log (strict version-00 parsing, never invents an id) | https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/traceparent.go | context | +| Multi-user console with OAuth2/OIDC SSO, per-account scoping, AES-256 upstream keys at rest | https://github.com/yolorouter/yolorouter#why-yolorouter | new — caller SSO/RBAC, #15, #19 (embedded web console) | +| Single-binary ops: embedded console, self-update, systemd/launchd/Windows installers, global or per-provider http/https/socks5/socks5h egress proxy (~30 s hot reload), vision-fallback capability | https://github.com/yolorouter/yolorouter#quick-start | context | + +**Absent (verified).** No MCP support (grep over internal/, cmd/, frontend +found none); no gateway-side response or semantic caching (cache tokens only +metered and priced); no inbound guardrails (content-policy refusals only +become failover signals); no embeddings, audio transcription (speech is +synthesis-only), batch, realtime/WebSocket, or files; no Kubernetes/Helm; +balanced-mode bindings process-local with a 4096-pair LRU cap; compressors +cover a fixed set of tool-output shapes; contributor/star figures from the +landing page (GitHub API rate-limited during research). + +## completion-to-response (NoahStepheno) + +**Identity.** ~800-line Go single-binary HTTP proxy (pure stdlib — go.mod has +zero require blocks) exposing `POST /v1/responses` and translating OpenAI +Responses API requests to any Chat Completions backend (Zhipu, DeepSeek, +Ollama, vLLM, Azure) and back, including full SSE event-chain synthesis; +companion Node gateway and bash scripts add multi-backend routing. Built +during the Codex CLI's Responses-only era. MIT. + +**Maintenance.** Dead — 6 commits between 2026-05-02 and 2026-05-17, one +release (v0.1.0, 2026-05-02), archived read-only on 2026-08-16; 14 stars. +The DeepSeek thinking fix and multi-tool grouping exist only on unreleased +main. + +**Sources.** Stats and status: +https://github.com/NoahStepheno/completion-to-response (repo page: 14 stars; +archived read-only banner dated 2026-08-16; six commits 2026-05-02/17) and +https://github.com/NoahStepheno/completion-to-response/releases (v0.1.0, +2026-05-02). + +| Capability | Evidence | Gap | +|---|---|---| +| OpenAI Responses client endpoint (`POST /v1/responses`) over Chat Completions upstreams — input arrays, instructions, tools, json_schema, max_output_tokens, stream | https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go | #6 (the exact blueprint) | +| Full Responses SSE event-sequence synthesis from chat chunks (response.created → … → response.completed) with per-event output/content indexes | https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go | #6 | +| Complete mapping tables: instructions→system, developer→system, max_output_tokens→max_tokens, finish-reason→status (tool_calls→requires_action), usage renames, resp_/msg_/fc_ id generation, late-system merge ordering rule | https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go | #6 | +| Streaming tool-call reconstruction into `function_call` output items with stable item ids across fragments (and reverse grouping of consecutive function_call inputs) | https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go | #6 | +| Reasoning preservation: `reasoning_content` deltas buffered and attached as `summary` on function_call items (DeepSeek thinking-mode; unreleased main) | https://github.com/NoahStepheno/completion-to-response/blob/main/internal/types/completion.go | #6 (thinking-content mapping) | +| Bearer passthrough with static-key fallback (the entire auth surface) | https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go | context | +| Multi-backend model-ID routing sidecar (Node) advertising alias models via `/v1/models` so Codex lists them | https://github.com/NoahStepheno/completion-to-response/blob/main/codex-gateway.js | context | +| Hardened SSE transport: 64 KB initial / 1 MB max scanner bound, [DONE] handling, flusher requirement, verbatim ≥400 forwarding; 10 s drain shutdown | https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go | context | +| Unit tests pinning the synthesized event contract (ordering + per-chunk counts) — the shape of contract tests a Responses endpoint owes | https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform_test.go | context | + +**Absent (verified).** Archived; no client-key verification, rate limiting, +caching/dedup, retry/fallback/load balancing, cost tracking, persistence, +metrics/tracing/request ids, MCP, or container packaging; Responses coverage +partial: no background mode, `store`/`previous_response_id` parsed but +unused, no built-in tools, no `sequence_number`, no error/incomplete_details +event synthesis, usage omitted from streamed completion, reasoning untested. + +## GPTCache (zilliztech) + +**Identity.** Python library (pip `gptcache`, MIT, by Zilliz — the Milvus +makers) for semantic caching of LLM responses, plus an optional FastAPI +server (`gptcache_server`, Docker `zilliz/gptcache`) exposing a generic +get/put API and an OpenAI-compatible chat proxy. The 2023 original of the +five-stage semantic-cache pipeline taxonomy that later semantic caches are +patterned on (a repeated design lineage, not a verified census). 8.2k stars. + +**Maintenance.** Maintenance-mode / dead-in-practice — 34 releases Apr–Nov +2023, final release v0.1.44 on 2024-08-01, then 2 commits in Jul 2025 and a +one-day burst of 10 commits on 2026-09-22 (cache-reuse benchmark merges, no +release); README states "we no longer add support for new API or models"; +100 open issues. README checkbox list is stale — trust the source tree. + +**Sources.** Stats and status: +https://api.github.com/repos/zilliztech/GPTCache (metadata read by the run: +not archived, default branch main, last push 2026-09-22) and +https://github.com/zilliztech/GPTCache/releases (v0.1.44, 2024-08-01). + +| Capability | Evidence | Gap | +|---|---|---| +| Five-stage pluggable cache pipeline (cache_enable_func → pre_embedding → embedding → data manager → similarity → post-process) with `next_cache` tiering — the canonical taxonomy | https://github.com/zilliztech/GPTCache/blob/main/gptcache/core.py | #10 | +| Rank-threshold similarity adjudication with 11 interchangeable evaluators (exact, vector-search distance, ONNX model, k-reciprocal, Cohere rerank, SBERT cross-encoder, time-window, JEV…) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/similarity_evaluation/time.py | #10 (semantic) | +| Temperature-aware cache skip (softmax over [skip, don't-skip]) and multi-answer post-processors (softmax/first/random) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py | #10 | +| LLM-verification gate on cached answers (yes/no check; a "no" or exception degrades to a miss) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/processor/post.py | #10 | +| Eviction subsystem: soft-evict then threshold hard-delete, mark-then-sweep, periodic index rebuild; MemoryCacheEviction (LRU/FIFO/LFU/RR) and distributed RedisCacheEviction | https://github.com/zilliztech/GPTCache/blob/main/gptcache/manager/eviction/manager.py | #10 | +| Three-axis storage matrix: scalar (SQLite/Postgres/MySQL/DynamoDB/…), vector (Milvus/FAISS/pgvector/Qdrant/…), object store (disk/S3) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/manager/vector_data/manager.py | #10 | +| Session-scoped caching with pluggable hit filters (refuse repeats within a session) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/session.py | #10 | +| Adapter stream capture on miss and SSE synthesis from a cached answer on hit (`_construct_stream_resp_from_cache`) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/openai.py | #10 (hit-replay-as-SSE) | +| FastAPI cache server + Docker image with a language-agnostic get/put API (and a '/cache_skip ' magic prefix bypass) | https://github.com/zilliztech/GPTCache/blob/main/gptcache_server/server.py | context | +| Multi-tier cache chaining via `next_cache` (local L1 falling through to a shared L2) plus `search_only` probing | https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py | #10 | +| Cache-consistency self-heal between scalar and vector stores (embedding compare, forced miss + rewrite on mismatch) | https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py | #10 | +| Per-stage cache telemetry, hit callbacks, and persisted hit reports | https://github.com/zilliztech/GPTCache/blob/main/gptcache/report.py | context | + +**Absent (verified).** Not a gateway: no auth/key management (server +authenticates nothing except cache-file download), no routing/retries/rate +limits/quotas/budgets, no cost tracking, no MCP, no protocol translation hub +(adapters are per-SDK monkey-patches; streaming proxy OpenAI-shaped only), +no external metrics export, no admin API/health checks; multimodal adapters +experimental. + +## PromptCache (messkan) + +**Identity.** Self-hosted semantic-caching proxy for LLM chat completions in +Go 1.24+ (Gin + embedded BadgerDB, single server binary under `cmd/api` with +`internal/` packages — not a library). Sits in front of an OpenAI-compatible +provider, embeds prompts, and returns cached answers above a two-threshold +band, with a small-model "intent verification" pass for gray-zone +candidates. Born Nov 2025 in the self-hosted "cut LLM costs" wave. + +**Maintenance.** Quiet but not dead — 4 releases (v0.1.0 Nov 2025 → v0.4.0 +2026-04-24), last commit 2026-08-18 (~2.5 months before 2026-11-04); 426 +stars, 47 commits, bursty AI-assisted cadence; 4 open issues / 2 PRs +unanswered. + +**Sources.** Stats and activity: https://github.com/messkan/prompt-cache +(repo page) and https://github.com/messkan/prompt-cache/releases (v0.1.0 +Nov 2025 through v0.4.0 2026-04-24; later additions unreleased on main). + +| Capability | Evidence | Gap | +|---|---|---| +| Two-threshold semantic matching: ≥0.70 direct hit, ≤0.30 miss, gray zone between (env-tunable, validated bounds) | https://github.com/messkan/prompt-cache/blob/main/internal/semantic/semantic.go | #10 (semantic) | +| Small-model intent verification for gray-zone hits (gpt-4o-mini / mistral-small / claude-3-haiku yes/no judge) | https://github.com/messkan/prompt-cache/blob/main/internal/semantic/providers.go | #10 | +| SSE streaming end-to-end: misses forward the provider stream while reassembling a non-streaming completion for the cache; hits synthesize OpenAI-style SSE chunks | https://github.com/messkan/prompt-cache/blob/main/RELEASE_v0.4.0.md | #10 (hit synthesis) | +| Runtime threshold and provider tuning via authenticated `GET/PATCH /v1/config` without restart | https://github.com/messkan/prompt-cache/blob/main/RELEASE_v0.4.0.md | context (runtime mutation) | +| Retryable upstream HTTP client (429/500/502/503/504, 500 ms × 2 backoff with 0–25% jitter capped 30 s) — not idempotency-aware | https://github.com/messkan/prompt-cache/blob/main/internal/http/client.go | #1 | +| Prometheus `/metrics` (hits/misses/gray-zone/failures, latency) plus JSON `/v1/stats` with hit-rate | https://github.com/messkan/prompt-cache/blob/main/internal/metrics/metrics.go | #12 | +| Constant-time bearer auth on management endpoints; inference and health stay public by design | https://github.com/messkan/prompt-cache/blob/main/internal/middleware/middleware.go | context | +| Cache lifecycle: 24 h TTL, 100k-entry LRU, hourly cleanup, `POST /v1/cache/warm` bulk insert, GET/DELETE management | https://github.com/messkan/prompt-cache/blob/main/internal/cache/cache.go | #10 | +| Namespace-partitioned cache isolation with ANN partition safety (namespaced lookups fall back to filtered linear scan so candidates never cross partitions) — unreleased on main | https://github.com/messkan/prompt-cache/blob/main/internal/semantic/semantic.go | #10 | +| Multi-provider embedding/verification backends (OpenAI, Mistral, Claude via Voyage embeddings) with an OpenAI→Claude translation lane | https://github.com/messkan/prompt-cache/blob/main/internal/semantic/providers.go | context | +| X-Cache/X-Similarity-Score/X-Cache-Key/X-Provider diagnostic response headers, allocation-free | https://github.com/messkan/prompt-cache/blob/main/internal/middleware/middleware.go | context | + +**Absent (verified).** No rate limiting or request shaping (roadmap only), +no quotas/budgets/cost tracking, no per-tenant auth (single admin token; the +namespacing work is unreleased and explicitly "not tenant isolation"), no +clustering or distributed cache, no dashboard, no MCP, one data-plane +endpoint only; "ANN" index is brute-force O(n·d) with a stale-LRU-index bug +after middle removals; retries blindly re-POST; Claude adapter hardcodes +MaxTokens 1024. + +## ollamaMQ (Chleba) + +**Identity.** Single-crate Rust CLI/proxy server (edition 2024, tokio + +axum 0.8, flat `src/` of 7 files) acting as a message-queue dispatcher and +load balancer in front of Ollama, LM Studio, vLLM-style and "System One" +decision-model backends, with a built-in ratatui TUI dashboard. Zero +protocol translation — every request forwarded byte-for-byte. MIT. + +**Maintenance.** Active — 67 commits, last commit 2026-10-30; ~10 tagged +releases v0.2.3 (2026-03-14) → v0.5.0 (2026-09-27) with Cargo.toml already +bumped to 0.5.1; 130 stars, solo maintainer plus one contributor. + +**Sources.** Stats and activity: https://github.com/Chleba/ollamaMQ (repo +page: 67 commits; last commit 2026-10-30) and +https://github.com/Chleba/ollamaMQ/releases (v0.2.3 2026-03-14 through +v0.5.0 2026-09-27). + +| Capability | Evidence | Gap | +|---|---|---| +| Per-user fair-share scheduling with exponentially decaying scores (300 s half-life), charged at dispatch regardless of outcome so failing clients cannot hold priority | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | new — admission queueing | +| Per-user FIFO queues with multi-layer bounded admission: 100/user queue cap (429), 512 MiB global queued-body budget (503), 64 MiB per-request body (413), 1 GiB blob allowance; metadata reads never queue | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | new — admission queueing | +| Least-connections load balancing with per-backend and per-model concurrency caps keyed on publisher/quant-stripped base names | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | #3, #13 | +| Warm-model state-affinity routing in three tiers: backends with the model loaded in RAM → cold backends → warned last-resort eviction | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | new — engine-state-aware routing | +| API-family detection with learning (a backend rejecting an endpoint twice gets that family excluded) and deterministic model matching (exact/tag/publisher/@quant-stripped, never substrings); strict model pinning with 503 after stuck_timeout | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | context | +| Model load/unload control via TUI keys and an admin HTTP API (GET /admin/models, POST /admin/models/load|unload), serialized per backend | https://github.com/Chleba/ollamaMQ/blob/master/src/control.rs | context (lifecycle management) | +| Declarative model-residency config applied additively at startup and on TUI reload ('r'), each load waiting up to 300 s for idle | https://github.com/Chleba/ollamaMQ/blob/master/appconf.yaml.example | #16 | +| Live ~10 Hz ratatui dashboard doubling as a runtime queue-admin console (VIP/Boost/block-user/block-IP/load/unload keys) | https://github.com/Chleba/ollamaMQ/blob/master/src/tui.rs | context (terminal TUI; #19 covers browser-served consoles) | +| Constant-time single-key auth (X-API-Key/Bearer) plus per-backend egress token substitution overriding client Authorization | https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs | context | +| Background health probing every 10 s with per-backend endpoint memory and offline marking that excludes backends from routing | https://github.com/Chleba/ollamaMQ/blob/master/src/control.rs | #1 (health exclusion) | +| Fail-fast stuck_timeout 503 distinguishing unsatisfiable (pinned, no holder) from merely-busy requests; dispatcher scans past an unroutable queue head | https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs | context | +| Size-rotated JSONL request/response logging with 64 KiB content caps and TUI full-content viewer; headers never logged | https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs | context | +| Aggregated `/api/tags` + `/v1/models` across backends; the full Ollama + OpenAI + System One endpoint surface; opt-in catch-all passthrough | https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs | context | +| Docker/compose/systemd packaging and a poison-tolerant lock recovery trait (~150 lock sites) | https://github.com/Chleba/ollamaMQ/tree/master | context | + +**Absent (verified).** Single static plaintext API key (no hashes, ids, +rotation, budgets, quotas); no usage metering/persistence or cost accounting +(JSONL only — llm-proxy's ledger is well beyond — +`crates/llm-proxy-storage/src/sqlite.rs:105-131`); no request-path +retry/fallback (queues and fail-fasts instead); pure byte passthrough (no +translation, no Anthropic client endpoint, no content normalization/SSE +machine); no response/prompt/semantic caching, guardrails, MCP, metrics +endpoint, OTel, true RPM/TPM windows, TLS termination, or CI. + +## Capability index + +Areas → sections. Numbers refer to the compendium gap register; `new` items +are the wave-3 candidates listed in the intro. + +| Area | Sections | +|---|---| +| Request-path retries, fallback, cooldowns (#1) | Switchyard, xLB, SMG, Antirion, Plano, AISIX, mcp-gateway, ThinkWatch, CLIProxyAPI, sub2api, Claude Code Router, Otari, done-hub, yolorouter, claude-code-proxy (raine), PromptCache, ollamaMQ (health exclusion) | +| Load-balancing / model-selection strategies (#13) | Switchyard, Antirion, Plano, AISIX, ThinkWatch, claude-code-proxy (raine), done-hub, Otari, ollamaMQ | +| Timeout hierarchy (new) | xLB (5 phases), Switchyard (chain deadline), yolorouter (7 phases) | +| Auth, keys, identity (#9; caller SSO/RBAC new) | AISIX, ThinkWatch, SMG, Antirion, mcp-gateway (key server), Otari, done-hub, CLIProxyAPI (absence), yolorouter, Claude Code Router, tokdiet (absence) | +| OAuth credential lifecycle (new) | claude-code-proxy (raine), CLIProxyAPI, sub2api, done-hub | +| Credential pools, rotation, quota state (#5) | CLIProxyAPI, sub2api, yolorouter, Claude Code Router | +| Budgets, cost, pricing (#2; synced price catalogs new) | Antirion, mcp-gateway, tokdiet, sub2api, Otari, yolorouter, done-hub, Switchyard (routing log), Plano (live cost catalogs); SMG and preflight verified absent | +| Rate limits, TPM, concurrency (#3) | xLB, SMG, Plano, AISIX, ThinkWatch, ollamaMQ, yolorouter, done-hub, Claude Code Router, preflight, mcp-gateway, PromptCache (absence) | +| Admission queueing (new) | SMG (priority classes + preemption), ollamaMQ (per-user FIFO + fair-share), xLB (queue_full backpressure), preflight (semaphore backpressure) | +| Engine-state-aware routing (new) | SMG, ollamaMQ, Switchyard (theoretical tracking) | +| Response caching (#10) | Antirion, AISIX, mcp-gateway, GPTCache, PromptCache; done-hub and CLIProxyAPI verified absent | +| Prompt-cache plumbing and cache locality (#14) | ostk-cache, claude-code-proxy (raine), CLIProxyAPI, sub2api, SMG, Claude Code Router, Otari, yolorouter, Plano, tokdiet (mutation guards), ThinkWatch (encrypted-history pinning), mistral.rs (engine-side KV cache) | +| Payload cost transforms + shadow-eval guard (new) | tokdiet, ostk-cache, yolorouter, Plano, Switchyard (compact) | +| Guardrails, DLP, secrets (#7) | preflight, Antirion, Plano, AISIX, ThinkWatch, mcp-gateway (tool poisoning), Otari, done-hub, Switchyard (redaction) | +| Upstream cancellation propagation (new) | xLB, sub2api (499 classification) | +| Observability: Prometheus (#12), OTel (#11), percentiles/TTFT (#17) | Switchyard, xLB, SMG, Antirion, Plano, AISIX, mcp-gateway, preflight, done-hub, yolorouter, PromptCache, ThinkWatch, Otari, mistral.rs | +| MCP (#8) | SMG, Antirion, Plano, AISIX, mcp-gateway, ThinkWatch, Claude Code Router (ToolHub), Otari, sub2api, done-hub, mistral.rs (client + server modes) | +| OpenAI Responses client endpoint (#6) | Switchyard, claude-code-proxy (raine), mistral.rs, SMG, AISIX, Plano, ThinkWatch, CLIProxyAPI, sub2api, Claude Code Router, yolorouter, completion-to-response; wider endpoint breadth is `context` (endpoints.md scope) | +| Cloud-native upstream auth (#4) | AISIX (Bedrock SigV4 / Vertex OAuth2 / Azure adapters), ThinkWatch (Bedrock SigV4 + Azure) | +| Web admin UI (#19) | Antirion (embedded console), mcp-gateway (wizard/doctor/dashboard), CLIProxyAPI (v8 web panel), Claude Code Router (web UI), yolorouter (web console), tokdiet (web dashboard); terminal TUIs (claude-code-proxy, ollamaMQ) and SMG's HTTP admin API are tagged `context` | +| Hot reload (#16) | mcp-gateway (~8 ms), ollamaMQ (TUI reload), CLIProxyAPI (management API), preflight (SIGHUP), AISIX (SIGHUP/etcd) | +| Hedged requests (new) | Antirion | + +## Coverage appendix — all deduplicated wave-3 finds + +Every candidate the nine niche hunts surfaced (121 repositories); ✦ marks +the 23 researched in full above. Stars/commit data as captured by the hunts +in November 2026; "dormant"/"dead" judgments are the hunts'. + +| Repository | What it is | Signals | +|---|---|---| +| ✦ [NVIDIA-NeMo/Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) | Rust model-routing toolkit (libsy + bindings) | Rust · ~3.5k★ · pushed 2026-11-04 · very active | +| ✦ [katanemo/plano](https://github.com/katanemo/plano) | Envoy+proxy-wasm data plane for agentic apps | Rust/TS · ~7.1k★ · pushed 2026-10-24 · active | +| [Xerxes-2/clewdr](https://github.com/Xerxes-2/clewdr) | Rust reverse proxy for Claude (web sessions + Claude Code), OpenAI-compatible out | Rust · ~1.3k★ · pushed 2026-11-01 · active | +| ✦ [ThinkWatchProject/ThinkWatch](https://github.com/ThinkWatchProject/ThinkWatch) | AI+MCP gateway with SSO/RBAC, budgets, audit | Rust · ~815★ · pushed 2026-11-04 · active | +| ✦ [smg-project/smg](https://github.com/smg-project/smg) | Shepherd Model Gateway, engine-agnostic, KV-cache-aware | Rust · ~700★ · pushed 2026-11-04 · active | +| ✦ [eliasbiondo/antirion](https://github.com/eliasbiondo/antirion) | Rust gateway w/ embedded console, benchmarks vs Bifrost/LiteLLM | Rust · ~660★ · pushed 2026-10-31 · young/active | +| ✦ [api7/aisix](https://github.com/api7/aisix) | Rust-native AI gateway from the APISIX creators | Rust · ~212★ · pushed 2026-11-04 · active | +| ✦ [EeroEternal/xlb](https://github.com/EeroEternal/xlb) | Streaming-first thin-L7/L4 AI load balancer | Rust · ~173★ · pushed 2026-10-19 · dormant | +| ✦ [Chleba/ollamaMQ](https://github.com/Chleba/ollamaMQ) | Queue dispatcher/LB for Ollama/LM Studio with TUI | Rust · ~130★ · pushed 2026-10-30 · active | +| ✦ [ghuntley/preflight](https://github.com/ghuntley/preflight) | Rust DLP/secret-scan inspection proxy | Rust · ~123★ · pushed 2026-11-02 · dev dormant (CI green) | +| ✦ [router-for-me/CLIProxyAPI](https://github.com/router-for-me/CLIProxyAPI) | Go proxy exposing CLI subscriptions as APIs | Go · 57.5k★ · pushed 2026-11-04 · extremely active | +| [coaidev/coai](https://github.com/coaidev/coai) | Multi-tenant AI one-stop platform (chat-nio successor) | Go+TS · 9.3k★ · pushed 2026-03-12 · slowed | +| [MartialBE/one-hub](https://github.com/MartialBE/one-hub) | one-api fork with per-channel proxies, MCP, payments | Go · 2.9k★ · pushed 2026-02-19 · bursty | +| [AmoyLab/Unla](https://github.com/AmoyLab/Unla) | Go MCP gateway converting REST/gRPC/WS into MCP servers | Go/TS · 2.25k★ · pushed 2026-10-20 · active | +| [ENTERPILOT/GoModel](https://github.com/ENTERPILOT/GoModel) | Go LiteLLM/Portkey alternative with caching/guardrails/MCP | Go · ~1.3k★ · pushed 2026-11-04 · very active | +| [zhu327/gemini-openai-proxy](https://github.com/zhu327/gemini-openai-proxy) | Minimal OpenAI→Gemini adapter (deprecated by native endpoint) | Go · 720★ · pushed 2025-10-07 · instructive-dead | +| [ferro-labs/ai-gateway](https://github.com/ferro-labs/ai-gateway) | Go gateway: 30 providers, 8 routing strategies, plugin guardrails | Go · 275★ · pushed 2026-11-02 · active | +| ✦ [yolorouter/yolorouter](https://github.com/yolorouter/yolorouter) | Go coding-CLI gateway with key pools and cost transforms | Go · 254★ · pushed 2026-10-28 · active | +| [EinStack/glide](https://github.com/EinStack/glide) | Cloud-native Go LLM gateway with router types | Go · 160★ · pushed 2024-08-12 · dormant | +| [openziti/llm-gateway](https://github.com/openziti/llm-gateway) | Zero-trust OpenAI-compatible proxy w/ semantic model routing | Go · 104★ · pushed 2026-10-14 · active | +| [KenyonY/openai-forward](https://github.com/KenyonY/openai-forward) | asyncio forwarding proxy archetype with cache + rate limits | Python · ~985★ · pushed 2025-03 · unmaintained | +| [Nya-Foundation/NyaProxy](https://github.com/Nya-Foundation/NyaProxy) | Provider-agnostic key-pool + LB policy gateway | Python · ~983★ · pushed 2026-07 · active | +| ✦ [mozilla-ai/otari](https://github.com/mozilla-ai/otari) | Mozilla.ai self-hosted gateway (virtual keys, budgets, MCP) | Python · ~834★ · pushed 2026-11-04 · very active | +| [NadirRouter/NadirClaw](https://github.com/NadirRouter/NadirClaw) | Complexity-routing cost optimizer (PolyForm NC) | Python · ~661★ · pushed 2026-10-22 · active | +| [aws-samples/bedrock-access-gateway](https://github.com/aws-samples/bedrock-access-gateway) | OpenAI⇄Bedrock Converse translator — deprecated for native endpoints | Python · ~1003★ · pushed 2026-09 · deprecated | +| [0xzr/freellmpool](https://github.com/0xzr/freellmpool) | Free-tier catalog + health-checked route pool | Python · ~144★ · pushed 2026-10-30 · active | +| [Nayjest/lm-proxy](https://github.com/Nayjest/lm-proxy) | Minimal FastAPI multi-provider gateway with virtual keys | Python · ~157★ · pushed 2026-11-03 · maintained | +| [Inebrio/Routerly](https://github.com/Inebrio/Routerly) | Node decision-and-control layer with explainable routing, RBAC | TypeScript · ~107★ · pushed 2026-11-04 · active | +| [youssefvdel/qwengate](https://github.com/youssefvdel/qwengate) | Bun gateway exposing chat.qwen.ai accounts as OpenAI API | TypeScript · ~209★ · pushed 2026-08 · gray-zone pattern ref | +| ✦ [agiwhitelist/tokdiet](https://github.com/agiwhitelist/tokdiet) | Local token-metering + context-compaction proxy with A/B proof | TypeScript · ~32★ · pushed 2026-06 · dormant | +| [blue-pen5805/llm-proxy-on-cloudflare-workers](https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers) | Serverless multi-provider proxy w/ credential pools, virtual models | TypeScript · ~88★ · pushed 2026-10-21 · active | +| [zuisong/gemini-openai-proxy](https://github.com/zuisong/gemini-openai-proxy) | Gemini-behind-OpenAI across Deno/Workers/Bun/Node runtimes | TypeScript · ~433★ · pushed 2026-04-02 · slow-moving | +| [CassiopeiaCode/CosyRedactGateway](https://github.com/CassiopeiaCode/CosyRedactGateway) | Cloudflare Worker privacy gateway (Gitleaks-compatible + entropy, reversible) | JavaScript · ~106★ · pushed 2026-10-31 · active | +| [hbsgithub/deno-azure-openai-proxy](https://github.com/hbsgithub/deno-azure-openai-proxy) | Single-file Deno OpenAI→Azure rewriter | TypeScript · ~139★ · pushed 2024-03-23 · dormant | +| [dext7r/ollama-api-pool](https://github.com/dext7r/ollama-api-pool) | Workers proxy pool for Ollama APIs (KV/Postgres/Redis stores) | JavaScript · ~26★ · pushed 2026-01-21 · idle | +| [kasuboski/openai-gateway](https://github.com/kasuboski/openai-gateway) | Hono Worker proxying via AI Gateway; providers from KV + secrets | TypeScript · ~2★ · pushed 2025-04-29 · dormant pattern ref | +| [Jhongwe1/edge-llm-gateway](https://github.com/Jhongwe1/edge-llm-gateway) | Single-Worker full-stack gateway (D1 + Durable Objects + cron) | TypeScript · ~2★ · pushed 2026-08-06 · small, instructive | +| [ShirokamiRyzen/Neko-router](https://github.com/ShirokamiRyzen/Neko-router) | Bun+Elysia gateway: response cache, token telemetry, key quotas | TypeScript · ~5★ · pushed 2026-11-03 · active | +| ✦ [musistudio/claude-code-router](https://github.com/musistudio/claude-code-router) | Flagship agent control plane (rules, pools, ToolHub MCP) | TypeScript · ~38k★ · pushed 2026-10-10 · active | +| [lidge-jun/opencodex](https://github.com/lidge-jun/opencodex) | Own the agent model picker; auto-configures agent clients | TypeScript · ~18.2k★ · pushed 2026-11-04 · very active | +| [ericc-ch/copilot-api](https://github.com/ericc-ch/copilot-api) | Copilot-as-OpenAI/Anthropic adapter (namesake pointer) | TypeScript · ~4.15k★ · pushed 2025-11-10 · stalled | +| [BenedictKing/ccx](https://github.com/BenedictKing/ccx) | Go protocol-translation gateway with console, channels, failover | Go · ~3.98k★ · pushed 2026-11-03 · very active | +| [1rgs/claude-code-proxy](https://github.com/1rgs/claude-code-proxy) | Python Anthropic-endpoint emulation over LiteLLM | Python · ~3.77k★ · pushed 2026-06-23 · slow-moving | +| ✦ [raine/claude-code-proxy](https://github.com/raine/claude-code-proxy) | Rust Anthropic-compatible proxy over subscription upstreams | Rust · ~694★ · pushed 2026-10-31 · active | +| [jianshuo/ccglass](https://github.com/jianshuo/ccglass) | Logging proxy + dashboard inspecting agent traffic | JavaScript · ~893★ · pushed 2026-10-11 · active | +| [luohy15/y-router](https://github.com/luohy15/y-router) | Edge Anthropic→OpenAI translator (archived; superseded upstream) | TypeScript · ~383★ · archived 2026-01-11 · instructive-dead | +| [m0n0x41d/anthropic-proxy-rs](https://github.com/m0n0x41d/anthropic-proxy-rs) | Rust Anthropic→OpenAI-compat proxy (thinking/tools/SSE) | Rust · ~94★ · pushed 2026-05-29 · modest | +| [achetronic/claude-oauth-proxy](https://github.com/achetronic/claude-oauth-proxy) | Go OAuth PKCE token manager for Anthropic Teams/Enterprise | Go · ~10★ · archived 2026-03-18 · instructive-dead | +| [PublicAffairs/openai-gemini](https://github.com/PublicAffairs/openai-gemini) | Serverless Gemini-behind-OpenAI incl. `:search` suffix, extra_body hatch | JavaScript · ~3.67k★ · pushed 2026-03-17 · maintained | +| [stulzq/azure-openai-proxy](https://github.com/stulzq/azure-openai-proxy) | Go OpenAI→Azure translator (path/auth/api-version rewriting) | Go · ~1.33k★ · pushed 2024-06-25 · dormant | +| [yasuoiwakura/openai-ollama-api-bridge](https://github.com/yasuoiwakura/openai-ollama-api-bridge) | Python OpenAI→Ollama-native bridge with queue mode + failover | Python · 3★ · pushed 2026-09-21 · tiny but thoughtful | +| ✦ [NoahStepheno/completion-to-response](https://github.com/NoahStepheno/completion-to-response) | Go Responses-over-ChatCompletions synthesis proxy | Go · 14★ · archived 2026-08-16 · dead-by-choice | +| [dongfangzan/local-openai2anthropic](https://github.com/dongfangzan/local-openai2anthropic) | Bidirectional Anthropic⇄OpenAI translator daemon (OA2A) | Python · 21★ · pushed 2026-06-30 · active | +| [unclecode/antomix](https://github.com/unclecode/antomix) | Anthropic↔~12 providers with in-session $$ commands, fan-out | JavaScript · 27★ · pushed 2025-08-10 · early-stage | +| [ulab-uiuc/LLMRouter](https://github.com/ulab-uiuc/LLMRouter) | De-facto RouteLLM successor: 16+ routers, xRouteBench | Python · ~3.1k★ · pushed 2026-10-19 · active research | +| [Writesonic/GPTRouter](https://github.com/Writesonic/GPTRouter) | Universal-API gateway with health-check fallback | TypeScript · ~454★ · pushed 2024-04 · dormant | +| [anyscale/llm-router](https://github.com/anyscale/llm-router) | Cost/quality routing methodology + paper (arXiv 2406.18665) | Python · ~261★ · pushed 2024-07 · dormant, cited | +| [stanford-futuredata/FrugalGPT](https://github.com/stanford-futuredata/FrugalGPT) | Pioneer of budget-constrained LLM cascades | Notebook · ~291★ · pushed 2025-02 · dormant idea repo | +| [ulab-uiuc/GraphRouter](https://github.com/ulab-uiuc/GraphRouter) | ICLR 2025 GNN router over tasks/queries/LLMs | Python · ~83★ · pushed 2025-12-30 · paper-complete | +| [ulab-uiuc/Router-R1](https://github.com/ulab-uiuc/Router-R1) | NeurIPS 2025 RL-trained multi-round router/aggregator | Python · ~155★ · pushed 2025-12-30 · paper-complete | +| [thu-nics/TokenRouter](https://github.com/thu-nics/TokenRouter) | NeurIPS 2026 token-level routing system | Python · ~75★ · pushed 2026-10-20 · active | +| [MilkThink-Lab/RouterEval](https://github.com/MilkThink-Lab/RouterEval) | EMNLP 2025 routing benchmark (200M+ records, 8.5k LLMs) | Python · ~125★ · pushed 2026-06-03 · maintained | +| [RouteWorks/RouterArena](https://github.com/RouteWorks/RouterArena) | Open router evaluation platform + live leaderboard | Python · ~155★ · pushed 2026-10-31 · active | +| [eclipse-lmos/lmos-router](https://github.com/eclipse-lmos/lmos-router) | Eclipse LMOS agent routing via declarative specs + embeddings | Kotlin · ~42★ · pushed 2026-11-03 · active | +| ✦ [deanxv/done-hub](https://github.com/deanxv/done-hub) | one-hub fork with agent-CLI OAuth channels, quota-ledger fixes | Go · ~815★ · pushed 2026-10-27 · active | +| ✦ [Wei-Shaw/sub2api](https://github.com/Wei-Shaw/sub2api) | Subscription-to-API multi-tenant gateway (design ref; ToS-flagged) | Go · ~45.4k★ · pushed 2026-11-04 · extremely active | +| [cita-777/metapi](https://github.com/cita-777/metapi) | Meta-aggregation of one-api-family panels with balance-aware routing | TypeScript · ~3.34k★ · pushed 2026-09-06 · active | +| [VoAPI/VoAPI](https://github.com/VoAPI/VoAPI) | Commercialized one-api distribution panel (multi-currency billing) | Go · ~1.09k★ · pushed 2026-01-27 · lightly maintained | +| [stevessr/Veloera](https://github.com/stevessr/Veloera) | new-api fork — canonical dissolve-into-upstream dead fork | Go · moved · pushed 2026-03-04 · unmaintained | +| [akl7777777/shell-api](https://github.com/akl7777777/shell-api) | one-api fork turned freemium/single-user aggregator | Go · ~152★ · pushed 2024-11-05 · dormant | +| [RixAPI/Rix-API](https://github.com/RixAPI/Rix-API) | Closed-source New API rewrite; docs-only public repo | JS/Go · ~63★ · pushed 2026-09-15 · product closed-source | +| ✦ [zilliztech/GPTCache](https://github.com/zilliztech/GPTCache) | Seminal semantic-cache library (five-stage taxonomy) | Python · ~8.2k★ · pushed 2026-09-22 · maintenance-mode | +| [codefuse-ai/ModelCache](https://github.com/codefuse-ai/ModelCache) | Ant Group production semantic cache (multi-tenant Redis recall) | Python · ~930★ · pushed 2025-06-30 · dormant | +| [redis/redis-vl-python](https://github.com/redis/redis-vl-python) | Official RedisVL SemanticCache extension | Python · ~430★ · pushed 2026-11-04 · active | +| ✦ [messkan/prompt-cache](https://github.com/messkan/prompt-cache) | Go semantic cache proxy with gray-zone judge | Go · ~425★ · pushed 2026-08-18 · quiet | +| [upstash/semantic-cache](https://github.com/upstash/semantic-cache) | Upstash fuzzy KV on Vector (clean API design; cloud-tied) | TypeScript · ~300★ · pushed 2024-11-21 · dormant | +| [kushals256/jevcache](https://github.com/kushals256/jevcache) | MorrowCache: local semantic cache for OpenAI-compat APIs | TypeScript · ~12★ · pushed 2026-11-04 · young/active | +| ✦ [os-tack/ostk-cache](https://github.com/os-tack/ostk-cache) | Rust prompt-cache hit-rate engineering proxy | Rust · ~10★ · pushed 2026-06-12 · dormant; private deps | +| [rohanarun/computer-use-cache](https://github.com/rohanarun/computer-use-cache) | Replay cache for computer-use/agent trajectories | JavaScript · ~47★ · pushed 2026-09-20 · active | +| [LMCache/LMCache](https://github.com/LMCache/LMCache) | KV-cache management layer for vLLM/SGLang | Python/C++ · ~12.3k★ · pushed 2026-11-04 · very active | +| [kvcache-ai/Mooncake](https://github.com/kvcache-ai/Mooncake) | Kimi's disaggregated KVCache platform (FAST '25) | C++ · ~7k★ · pushed 2026-11-04 · very active | +| [protectai/llm-guard](https://github.com/protectai/llm-guard) | Security toolkit: input/output scanner taxonomy (archived) | Python · ~3.2k★ · archived 2026-07 · canonical-dead | +| [guardrails-ai/guardrails](https://github.com/guardrails-ai/guardrails) | Input/Output guards w/ Hub validators, re-ask repair loop | Python · ~7.6k★ · pushed 2026-11-03 · active | +| [NVIDIA-NeMo/Guardrails](https://github.com/NVIDIA-NeMo/Guardrails) | Colang programmable guardrails (flows, rails) | Python · ~7.4k★ · pushed 2026-11-04 · very active | +| [protectai/rebuff](https://github.com/protectai/rebuff) | Self-hardening injection detector (archived; canary pattern) | TypeScript · ~1.5k★ · archived 2024-08 · dead, instructive | +| [deadbits/vigil-llm](https://github.com/deadbits/vigil-llm) | Ensemble scanner library + REST API (YARA, vector DB, canaries) | Python · ~509★ · pushed 2024-01-31 · dead, instructive | +| [data-privacy-stack/presidio](https://github.com/data-privacy-stack/presidio) | De-facto PII engine (analyzer/anonymizer/image/structured) | Python · ~11.7k★ · pushed 2026-11-02 · community-governed | +| [StackOneHQ/defender](https://github.com/StackOneHQ/defender) | Injection defense at tool-result ingestion (22 MB ONNX CPU model) | TypeScript · ~132★ · pushed 2026-11-04 · very active | +| [InditexTech/cerbia](https://github.com/InditexTech/cerbia) | Security gates with multi-layer de-obfuscation before scanning | Python · ~46★ · pushed 2026-11-04 · active | +| [meta-llama/PurpleLlama](https://github.com/meta-llama/PurpleLlama) | LlamaFirewall: PromptGuard 2, CodeShield, AlignmentCheck | Python · ~4.5k★ · pushed 2026-10-30 · active | +| [occludra/gateway](https://github.com/occludra/gateway) | OpenAI-SDK security proxy chaining Presidio+secrets+injection, fail-closed | Python · ~49★ · pushed 2026-08-16 · young | +| [AgentOps-AI/agentops](https://github.com/AgentOps-AI/agentops) | Agent telemetry SDK: session replays, cost tracking | Python · ~5.9k★ · pushed 2026-06-25 · semi-active | +| [samanhappy/mcphub](https://github.com/samanhappy/mcphub) | MCP hub: aggregated SSE/Streamable/stdio endpoints + pgvector routing | TypeScript · ~2.6k★ · pushed 2026-11-03 · active | +| [metatool-ai/metamcp](https://github.com/metatool-ai/metamcp) | MCP meta-server: namespacing, middleware, multi-tenancy | TypeScript · ~2.8k★ · pushed 2026-06-22 · moderately active | +| [sparfenyuk/mcp-proxy](https://github.com/sparfenyuk/mcp-proxy) | De-facto MCP transport bridge (stdio↔SSE/Streamable HTTP) | Python · ~2.8k★ · pushed 2026-11-04 · active | +| [1mcp-app/agent](https://github.com/1mcp-app/agent) | Unified MCP runtime/hub with progressive tool discovery | TypeScript · ~527★ · pushed 2026-11-03 · active | +| [burugo/one-mcp](https://github.com/burugo/one-mcp) | MCP reverse-proxy platform + Anthropic Skills export | Go · ~422★ · pushed 2026-09-06 · maintained | +| [smart-mcp-proxy/mcpproxy-go](https://github.com/smart-mcp-proxy/mcpproxy-go) | Meta-tool MCP proxy with tool quarantine + SARIF scanners | Go · ~407★ · pushed 2026-11-03 · active | +| [ip2a/mcpstore](https://github.com/ip2a/mcpstore) | Rust MCP manager: control plane + data plane, per-agent scoping | Rust · ~433★ · pushed 2026-11-04 · active | +| ✦ [MikkoParkkola/mcp-gateway](https://github.com/MikkoParkkola/mcp-gateway) | Meta-tool MCP gateway with security primitives (PolyForm NC) | Rust · ~85★ · pushed 2026-11-02 · very active | +| [open-webui/mcpo](https://github.com/open-webui/mcpo) | Inverse proxy: MCP servers as OpenAPI/REST endpoints | Python · ~4.4k★ · pushed 2026-05-17 · maintained | +| [mudler/LocalAI](https://github.com/mudler/LocalAI) | Self-hosted OpenAI/Anthropic-compatible AI engine | Go · 50.2k★ · pushed 2026-11-04 · flagship | +| [mostlygeek/llama-swap](https://github.com/mostlygeek/llama-swap) | Zero-dep model hot-swap proxy for local servers | Go · 6.18k★ · pushed 2026-11-04 · very active | +| [theroyallab/tabbyAPI](https://github.com/theroyallab/tabbyAPI) | ExLlamaV3 OpenAI-compatible backend (hobby-grade) | Python · 1.51k★ · pushed 2026-11-04 · active | +| [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python) | Ubiquitous OpenAI-compatible llama.cpp shim (slowing) | Python · 10.7k★ · pushed 2026-10-01 · slowed | +| [oobabooga/textgen](https://github.com/oobabooga/textgen) | Major llama.cpp frontend, now dual-protocol local server | Python · 47.7k★ · pushed 2026-08-17 · active | +| [janhq/jan](https://github.com/janhq/jan) | Offline ChatGPT replacement w/ Rust core + :1337 OpenAI API | Rust · 45.4k★ · pushed 2026-11-04 · very active | +| [mozilla-ai/llamafile](https://github.com/mozilla-ai/llamafile) | Single-file LLM distribution (Cosmopolitan) | C++ · 26.6k★ · pushed 2026-11-04 · active | +| ✦ [EricLBuehler/mistral.rs](https://github.com/EricLBuehler/mistral.rs) | Rust inference engine, dual-protocol serve | Rust · ~7.8k★ · pushed 2026-11-03 · active | +| [inference-gateway/inference-gateway](https://github.com/inference-gateway/inference-gateway) | Cloud-native Go gateway with MCP + A2A + OPA guardrails | Go · 217★ · pushed 2026-11-04 · daily commits | +| [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) | Single-executable local inference, many wire compatibilities | C++ · 12.1k★ · pushed 2026-11-04 · very active | +| [cuihuan/awesome-ai-gateway](https://github.com/cuihuan/awesome-ai-gateway) | 240+-gateway meta-list: scorecard, benchmarks, handbook, datasets | Markdown · ~150★ · pushed 2026-11-04 · active | +| [ai-gateways/awesome-ai-gateways](https://github.com/ai-gateways/awesome-ai-gateways) | Vendor-neutral taxonomy incl. aisix and llmgateway yields | Markdown · ~29★ · pushed 2026-11-03 · active | +| [howardpen9/awesome-ai-api-proxy](https://github.com/howardpen9/awesome-ai-api-proxy) | Relay-station gray market catalog + weekly pricing intelligence | HTML/data · ~102★ · pushed 2026-11-01 · maintained | +| [tensorchord/Awesome-LLMOps](https://github.com/tensorchord/Awesome-LLMOps) | Highest-traffic LLMOps list (bitrouter, shim yields) | Markdown · ~6k★ · pushed 2026-10-16 · active | +| [yenanjing/awesome-model-routing](https://github.com/yenanjing/awesome-model-routing) | Routing-first collection (AIClient2API, ClawRouter yields) | Markdown · ~35★ · pushed 2026-11-03 · active | +| [talhaanwarch/awesome-llmops-platforms](https://github.com/talhaanwarch/awesome-llmops-platforms) | Self-hostable-only catalog, weekly CI refresh, honesty badges | Markdown · ~39★ · pushed 2026-10-26 · active | +| [sihyeong/Awesome-LLM-Inference-Engine](https://github.com/sihyeong/Awesome-LLM-Inference-Engine) | Engine taxonomy incl. gateway section (ferro-labs yield) | Markdown · ~250★ · pushed 2026-10-30 · active | +| [12britz/awesome-ai-gateways](https://github.com/12britz/awesome-ai-gateways) | Capability-organized checklist (semantic cache, cost, guardrails) | Markdown · ~12★ · pushed 2026-07-14 · stable checklist | +| [Router-One/awesome-llm-gateway](https://github.com/Router-One/awesome-llm-gateway) | Managed-router tier + cascade research pointers | Markdown · 0★ · pushed 2026-10-12 · vendor-curated | +| [Ar9av/awesome-agent-control-plane](https://github.com/Ar9av/awesome-agent-control-plane) | Lifecycle framing: identity, policy-at-action, blast radius | Markdown · ~4★ · pushed 2026-08-07 · small | + +Corrections welcome via the usual research-note flow; the registry +(`repos-registry.json`) is the machine-readable counterpart of this survey. diff --git a/docs/research/repo-intelligence.md b/docs/research/repo-intelligence.md new file mode 100644 index 0000000..172b9eb --- /dev/null +++ b/docs/research/repo-intelligence.md @@ -0,0 +1,2695 @@ +# Repository intelligence — fourth research wave (November 2026) + +Method: a fourth research wave, run 2026-11-04, harvesting **63 +repositories** in two tiers. Twenty-three were researched in full along four +angles — **code** (capabilities read from clones), **issues** (user-reported +pain points), **pulls** (where each project is investing), and **gaps** +(verified absences on the llm-proxy side) — and forty more received +**issue-tracker sweeps** (recent issues and PRs mined for pain points and +directions). Tier assignment follows the harvest's data, not a judgment of +importance: the 23 full-angle repositories are exactly those with a code +clone in this wave — 22 under `research-repos/` (`ls research-repos/ | wc -l` +→ 22, run this session) plus mistral.rs at `/tmp/intel-mistral/mistral.rs` — +while the 40 sweep repositories were harvested through their issue/PR +trackers only, via the snapshots at +`research-issues/__/{issues,pulls}.json`. Every claim below +carries the harvest's evidence: a GitHub URL or a clone path. Fully-qualified +clone paths start with `research-repos/__/`; paths written +clone-relative (`src/…`, `crates/…`, `backend/…`, `packages/…`, …) refer to +that section's repository clone under the same root, and mistral.rs paths +are relative to `/tmp/intel-mistral/mistral.rs` @ 52b3c49. Maintenance +statistics (merge counts, issue tallies, dates, star counts) are reads of +those same `research-issues/` snapshots and clone refs captured by the +harvest; they carry no per-claim URL beyond that provenance. +Nothing was built or run; llm-proxy-side statements come from the wave's +provided inventory plus spot-checks verified this session at llm-proxy +commit `6bcf1ef4e4d32fc0eae69b528f0e10cfea3e433b` (`git rev-parse HEAD`, +2026-11-04): no request-path retry outside +`crates/llm-proxy-provider/src/discovery.rs:145`; flat `TimeoutLayer` at +`crates/llm-proxy-server/src/routes/mod.rs:64-67`; `MAX_BODY_BYTES` 32 MiB +at `routes/mod.rs:37`; no `prompt_tokens_details` / `cached_tokens` anywhere +in `crates/` or `apps/`; core usage already carries `cache_creation`/ +`cache_read` at `crates/llm-proxy-protocol/src/core.rs:639-641`; and the +rate limiter keys by caller key id when authenticated, client IP otherwise +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:173-176`) — per-key +RPM already exists, matching the compendium's narrowed gap #3. + +Companion file: [`compensation-backlog.json`](compensation-backlog.json) — +the machine-readable, priority-sorted list of features/fixes this record +implies for llm-proxy. Earlier waves: the gap register (#1–#20) lives in +[`competitive-landscape-compendium.md`](competitive-landscape-compendium.md); +wave-3 niche deep dives in [`niche-repos.md`](niche-repos.md). Wave 4 is +deliberately **not** folded into +[`repos-registry.json`](repos-registry.json): the registry remains the +three-wave capability-row artifact, and this wave's machine-readable output +is the compensation backlog (candidates mapped to crates rather than +capability rows). + +From record to backlog: a Part-I gap or sweep finding becomes a compensation +candidate when the need recurs across three or more harvested repositories +(the recurring-needs threshold below) or is a single-repo capability with +direct user validation and no cheaper substitute. Deliberate exclusions: +ThinkWatch's minimum-generation-speed stream cutting +(`research-repos/ThinkWatchProject__ThinkWatch/crates/gateway/src/proxy/min_speed.rs:1-88`) +presupposes streaming tokens/sec telemetry llm-proxy does not yet collect — +it is a follow-on to the phase-timeout and usage-estimation candidates, not +an independent one; mistral.rs's stateful Responses machinery and +session-portability API (`mistralrs-server-core/src/cached_responses.rs:14-27`, +`handlers.rs:634-691` per the harvest) are design inputs to the Responses +client-endpoint candidate — server-side state is a separate decision once +the endpoint exists; engine-local surfaces (model load/unload, KV-cache +management, GPU residency) belong to inference servers, not a translation +proxy, and stay recorded as context only. + +## Contents + +- Part I — Full four-angle harvests (23 repositories) +- Part II — Issue-tracker sweeps (40 repositories) +- Pain-point index — theme → repos and issues +- Direction index — where the niche ecosystem is heading +- Recurring unmet needs — patterns in 3+ repositories + +--- + +## Part I — Full four-angle harvests + +### EricLBuehler/mistral.rs + +**Identity · Language · Form factor.** Rust workspace (edition 2021, MSRV +1.95, 15+ crates) built on candle: a local LLM inference engine whose +`mistralrs serve` binary doubles as an OpenAI- and Anthropic-compatible API +server with an embedded web UI. + +**Maintenance.** Very active — HEAD 52b3c49b dated 2026-11-02 ("per-request +context limits for coding agents"), PRs merged daily. + +**Capabilities (code).** Three client dialects on one server (OpenAI Chat, +OpenAI Responses, Anthropic Messages) with count_tokens implemented by +round-tripping the real loaded tokenizer +(`/tmp/intel-mistral/mistral.rs/mistralrs-server-core/src/anthropic.rs:1834,1938,2006`); +per-request serving context = min(model context, paged-KV capacity, +`--max-model-len`) with a typed `ContextLengthExceeded` mapped per dialect +(`mistralrs-server-core/src/handler_core.rs:27,144`); one `observe_http` +middleware exporting Prometheus + TTFT/inter-token-latency histograms, +request-id injection, and SSE usage deferred until the body stream ends +(`mistralrs-server-core/src/metrics.rs:45-98,214-331`); client-disconnect +cancellation wired end-to-end into the scheduler +(`mistralrs-core/src/scheduler/default_scheduler.rs:364-370`); runtime model +lifecycle over HTTP plus confined hot LoRA swap (`mistralrs-server-core/src/handlers.rs:419-529`); +MCP in both directions (`mistralrs-server-core/src/mcp_server.rs:13-35`); +human-in-the-loop tool approvals over SSE (`mistralrs-server-core/src/approvals.rs:53-130`); +OS-level subprocess sandbox (Landlock/seccomp/Seatbelt, `mistralrs-sandbox/src/lib.rs:1-40`); +OpenAI-compatible `/v1/files` and versioned `/v1/skills` +(`mistralrs-server-core/src/skills.rs:47-282`). + +**Pain points (issues).** Default `0.0.0.0:1234` with no auth exposed the +whole control surface; five recent GHSAs were all reachable unauthenticated +(https://github.com/EricLBuehler/mistral.rs/issues/2490); tool-call parsing +is the most bug-dense area (https://github.com/EricLBuehler/mistral.rs/issues/2466, +/issues/2520, /issues/2528); `thinking.budget_tokens` accepted but never +enforced (https://github.com/EricLBuehler/mistral.rs/issues/2522); no +per-request prefix-cache isolation — `CacheSalt` exists but is never +constructed (https://github.com/EricLBuehler/mistral.rs/issues/2523); +auto KV sizing starves admission on small GPUs with a 503 that never names +memory pressure (https://github.com/EricLBuehler/mistral.rs/issues/2502). + +**Directions (pulls).** The server is being tuned as a local backend for +coding agents: per-request context limits with native per-API overflow +errors and `max_model_len` on `/v1/models` +(https://github.com/EricLBuehler/mistral.rs/pull/2538); per-request +`cache_salt` cache isolation open (https://github.com/EricLBuehler/mistral.rs/pull/2524); +periodic defensive-security passes (https://github.com/EricLBuehler/mistral.rs/pull/2447); +Apple Silicon and quantization as the performance bets +(https://github.com/EricLBuehler/mistral.rs/pull/2418, /pull/2512). + +**Gaps vs llm-proxy.** Context-limit preflight is a cheap win for llm-proxy, +which surfaces `context_length` on `/v1/models` +(`crates/llm-proxy-server/src/routes/models.rs:35,142`) but never +preflight-checks requests (gap evidence: mistral.rs PR #2538 + +`mistralrs-core/src/engine/add_request.rs:523-537` @ 52b3c49); also absent +from llm-proxy: legacy `/v1/completions`, `/v1/skills`, session-portability +API, stateful Responses machinery (background tasks + retrieval/cancel + +`previous_response_id` chaining), served OpenAPI spec, live model-ops over +HTTP, and a client chat UI (`mistralrs-server-core/src/route_registry.rs:27-120`). + +### MikkoParkkola/mcp-gateway + +**Identity · Language · Form factor.** Rust single-binary MCP gateway +(edition 2024, MSRV 1.95, v4.0.0, ~302k lines) exposing a compact ~20-tool +`gateway_*` meta-surface over MCP backends plus 125 prebuilt REST +capabilities. PolyForm-NC with a commercial tier. + +**Maintenance.** Extremely active and effectively solo — 69 of the last 100 +PRs merged on 2026-11-04 alone; nearly all 394 true issues are owner-filed +severity-tagged review findings. + +**Capabilities (code).** Per-backend `Failsafe` composing circuit breaker, +governor token bucket, backon retry, and latency health — with 429s excluded +from failure accounting (`research-repos/MikkoParkkola__mcp-gateway/src/failsafe/mod.rs`); +three-layer kill switch: operator disable meta-tool, per-backend sliding-window +error budgets, per-capability budgets with cooldown auto-recovery +(`src/kill_switch/mod.rs`); pre-invoke `BudgetEnforcer` in micro-USD atomics +with UTC-day CAS reset (`src/cost_accounting/enforcer.rs`); structural-diff +hot reload that refuses listener changes (`src/config_reload/mod.rs`); +canonical-JSON idempotency with hard resource fences +(`src/idempotency/admission.rs`); credential injection with forced overwrite +and audit (`src/secret_injection.rs`); TF-IDF tool discovery behind the +meta-surface (`src/semantic_search/mod.rs`). + +**Pain points (issues).** Rate limiter and breaker keyed by API-key display +name so renamed/shared keys share buckets +(https://github.com/MikkoParkkola/mcp-gateway/issues/3950); stdio pipe +backpressure wedged the whole gateway +(https://github.com/MikkoParkkola/mcp-gateway/issues/2531); the idempotent +cache answered before policy so a firewalled tool still served from cache +(https://github.com/MikkoParkkola/mcp-gateway/issues/2445); +credential-echo redaction kept missing stored-answer paths +(https://github.com/MikkoParkkola/mcp-gateway/pull/4552). + +**Directions (pulls).** Zero-restart policy reload under per-request pinned +config generations (https://github.com/MikkoParkkola/mcp-gateway/pull/4508); +rotation-safe auth state — rate-limit windows keyed to principal not digest +(https://github.com/MikkoParkkola/mcp-gateway/pull/4501); credential-echo +scrubbing as a core boundary over every answer path +(https://github.com/MikkoParkkola/mcp-gateway/pull/4561); append-only +transparency log (https://github.com/MikkoParkkola/mcp-gateway/pull/4598). + +**Gaps vs llm-proxy.** Always-on DNS-rebinding/Origin guard (CWE-346 fix) +versus llm-proxy's opt-in-only CORS and absent Host validation +(`src/gateway/router/origin_guard.rs:316-400` vs +`crates/llm-proxy-server/src/routes/mod.rs:111-119`); cleartext-credential +refusal at config load (`src/config/mod.rs:1144-1190`); per-API-key tool +scoping with globs (`examples/per-client-tool-scopes.yaml:17-52`); `doctor` +diagnostics with fix/verify/rollback commands (`src/commands/doctor.rs:47-105`); +client mTLS (`src/mtls/mod.rs`); hot reload with auth-weakening refusal +semantics (`src/config_reload/mod.rs:93-135`); idempotency replay where +llm-proxy's dedup only 409-rejects concurrent duplicates +(`src/idempotency/admission.rs`). + +### NVIDIA-NeMo/Switchyard + +**Identity · Language · Form factor.** Apache-2.0 Rust workspace (edition +2024, MSRV 1.96.1, v0.3.0) whose core is `libsy`, an embeddable +model-routing library; the OpenAI/Anthropic/Responses-compatible server is +explicitly a local reference implementation, with Python/Go/TS/WASM +bindings. + +**Maintenance.** Very active — 65 of 100 harvested PRs merged 2026-10-23 → +2026-11-03 (~6/day), NVIDIA-engineer-driven. + +**Capabilities (code).** Host-driven Step-protocol routing decoupled from +transport, panic-contained and cancellation-safe +(`research-repos/NVIDIA-NeMo__Switchyard/crates/libsy/src/core/algorithm.rs:613-716`); +per-backend retry budget covering transport failures, timeouts, 408/429/5xx +with capped Retry-After and one deadline spanning attempts and stream reads, +first decoded event checked inside the retry boundary +(`crates/libsy-llm-client/src/client.rs:400-528,886-944`); ordered fallback +candidates with per-candidate rewrites, advancing only on gated reasons +(`crates/libsy/src/core/algorithm.rs:257-323`); per-route capability gating +pre-dispatch (`crates/switchyard-server/src/capabilities.rs:12-62`); durable +JSONL routing log + `GET /v1/routing/session-stats` +(`crates/switchyard-server/src/routing_log.rs:27-51,325-371`); dual-source +cost accounting with `CostSource` tags +(`crates/switchyard-costs/src/lib.rs:46-123`); Prometheus export via an +opentelemetry_prometheus reader with per-metric histogram buckets plus +routing-specific instruments (`classifier_fail_open`, `retry_recovered`, +`client_disconnected` kept separate from served responses) +(`crates/switchyard-server/src/metrics.rs:24-33,50-77,119-165`); credential +redaction of +headers/JSON/SSE (`crates/switchyard-server/src/redaction.rs:16-70`); +loss-aware `TranslationPolicy` with structured diagnostics +(`crates/switchyard-translation/README.md:91-123`). + +**Pain points (issues).** An OpenAI upstream can return HTTP 200 and reject +the stream in its first SSE frame; treated as success, no fallback +(https://github.com/NVIDIA-NeMo/Switchyard/issues/1132); codecs silently +drop request fields with no diagnostics channel on the stream path +(https://github.com/NVIDIA-NeMo/Switchyard/issues/1215); slow first byte +(~414 s) trips client/proxy timeouts — early response headers plus an +Anthropic `event: ping` keepalive requested, none exists +(https://github.com/NVIDIA-NeMo/Switchyard/issues/1168). + +**Directions (pulls).** Library-first positioning +(https://github.com/NVIDIA-NeMo/Switchyard/pull/1155); in-stream pre-answer +fallback for 200-then-error with a held prelude +(https://github.com/NVIDIA-NeMo/Switchyard/pull/1133); cross-wire reasoning +normalization with explicit loss diagnostics +(https://github.com/NVIDIA-NeMo/Switchyard/pull/1216); session-affinity +stability including preserving the prompt-cache-reusable prefix +(https://github.com/NVIDIA-NeMo/Switchyard/pull/1213); decision models +spreading through every algorithm +(https://github.com/NVIDIA-NeMo/Switchyard/pull/1195). + +**Gaps vs llm-proxy.** Execution-signal stage routing from tool-result +history (`crates/libsy/src/algorithms/util/tool_signals.rs:34-120`); LLM-judge +task-classifier routing (`crates/switchyard-server/README.md:176-190`); +learned prefill routing (`crates/prefill-router/src/`); verification-gated +routing with shadow mode and kill switch (`crates/libsy/src/algorithms/vgr.rs`); +first-party evaluation/soak harness (`benchmark/README.md:8-19`); +provider-credential redaction at the output boundary +(`crates/switchyard-runner/src/provider_key_redactor.rs:19-43`); per-target +request augmentation (system prompt prepend, `extra_body` merge) +(`crates/switchyard-server/README.md:106-107`). + +### ThinkWatchProject/ThinkWatch + +**Identity · Language · Form factor.** Rust workspace (6 crates: auth, +common, gateway, mcp-gateway, server, test-support) plus a React console — +a self-hosted AI API + MCP gateway for organizations over PostgreSQL, +Redis, and ClickHouse. BUSL-1.1. + +**Maintenance.** Extremely active — v3.11.0 committed 2026-11-03, 93 of the +last 100 PRs merged; organic user signal thin (only 5 true issues ever, two +of them vendor noise). + +**Capabilities (code).** Strategy-driven weighted routing with Redis-shared +EWMA health and CAS-only breaker state transitions +(`research-repos/ThinkWatchProject__ThinkWatch/crates/gateway/src/strategy.rs:1-120`, +`src/health.rs:1-119`); deadline-budget retry with precise classification — +transient = 5xx/529/connection/stream-dead-before-output, never 429/other +4xx/after-content — with SSE `: keep-alive` frames while waiting +(`crates/gateway/src/proxy/retry.rs:1-130`); sealed-history provider pinning +for Responses `store:false` (30-day Redis, plain-language refusal when the +provider has no route, `crates/gateway/src/proxy/sealed.rs:1-92`); +per-provider minimum generation speed that cuts dribbling streams with 503 +and drops affinity (`crates/gateway/src/proxy/min_speed.rs:1-88`); Redis Lua +bucketed sliding windows for requests *and* tokens with admit/record split +and fail-open/closed switch (`crates/common/src/limits/sliding.rs:1-46`); +exact-match response cache keyed on the whole request body whose hits still +debit limits (`crates/gateway/src/cache.rs:1-110`); usage estimation when +upstream usage never arrives (`crates/gateway/src/usage_estimate.rs:1-15`); +early-cancel 499 audit rows via a drop guard armed at auth +(`crates/gateway/src/proxy/early_cancel.rs:1-24`, merged as the pre-stream +cancellation rows of https://github.com/ThinkWatchProject/ThinkWatch/pull/54 +and the drop-guard hardening of +https://github.com/ThinkWatchProject/ThinkWatch/pull/62); self-healing +protocol relearn persisted per route +(`crates/gateway/src/proxy/protocol_relearn.rs:1-24`). + +**Pain points (issues).** OIDC multi-audience ID tokens rejected +out-of-the-box (https://github.com/ThinkWatchProject/ThinkWatch/issues/10); +third-party MCP-server admission scanning declined not-planned +(https://github.com/ThinkWatchProject/ThinkWatch/issues/113). + +**Directions (pulls).** Failover accounting decided by upstream error class +so one caller's bad request cannot open breakers +(https://github.com/ThinkWatchProject/ThinkWatch/pull/38); transient retry +across all candidate routes under a 120 s ceiling +(https://github.com/ThinkWatchProject/ThinkWatch/pull/122); per-bucket +pricing with cache-read/write weights +(https://github.com/ThinkWatchProject/ThinkWatch/pull/40); limits hardening +with lineage-surviving key counters and computed `Retry-After` + +`x-should-retry: false` (https://github.com/ThinkWatchProject/ThinkWatch/pull/81); +Gemini client surface plus Responses over WebSocket through one pipeline +(https://github.com/ThinkWatchProject/ThinkWatch/pull/43). + +**Gaps vs llm-proxy.** Minimum-generation-speed stream cutting (not in the +register); org identity — OIDC/TOTP/RBAC with deny-wins policy union +(`crates/auth/src/oidc.rs:1-50`, `src/rbac.rs:16-33`, `src/totp.rs:7-23`); +stream-safe reversible redaction with a placeholder ledger and scan cache +(`crates/gateway/src/redaction.rs:1-79`); tool-call inspection against an +upstream-MITM threat model (`crates/gateway/src/tool_inspection.rs:1-40`); +client-facing `GET /v1/usage` for the caller's own key +(`crates/server/src/handlers/key_usage.rs:1-24`); SIEM forwarding with a +durable outbox (`crates/common/src/audit/outbox.rs:1-35`). + +### Wei-Shaw/sub2api + +**Identity · Language · Form factor.** Go 1.27 monolith (Gin + Ent + +PostgreSQL/Redis + Vue 3 admin) pooling upstream AI subscription/OAuth +accounts (Claude, Codex, Gemini, Grok, Antigravity, CN providers) and +redistributing quota via billed API keys. 1,448 non-test Go files. + +**Maintenance.** Extremely active — HEAD 2026-11-04; the issue tracker is +high-volume and code-level (34 true issues in five days at harvest). + +**Capabilities (code).** Request-path failover loop with bounded account +switches, failed-account exclusion, livelock guards, and client-gone +cancellation semantics (`research-repos/Wei-Shaw__sub2api/backend/internal/handler/failover_loop.go:33-267`); +sticky sessions in Redis (1 h) with re-billing of input tokens as cache +reads when a switch abandons a warm prompt cache — a cost-correctness +detail unique to multi-account pools (`backend/internal/service/gateway_service.go:462-484`); +per-account quota-window pausing from real upstream reset snapshots +(`backend/internal/service/account_scheduling_threshold_eval.go:22-60`); +per-platform OAuth token providers with refresh skew +(`backend/internal/service/claude_token_provider.go:13-73`); first-output +timeout staging for streaming failover — SSE buffered 64 KiB→temp file +until the first output event so a hung upstream can be failed over before +any byte is committed (`backend/internal/service/openai_first_output_timeout.go:23-55`); +in-flight balance reservation from max-output-token estimates +(`backend/internal/handler/gateway_inflight_reservation.go:13-70`); channel +monitor v2 synthetic probes with error-taxonomy rollups +(`backend/internal/service/channel_monitor_v2.go:322-621`); a management MCP +server exposing the platform's own REST admin/user API as tools over +stateless Streamable HTTP, built as four immutable pre-configured servers +(user/admin scope × read-only/write) so no credentials persist across HTTP +requests (`backend/internal/managementmcp/server.go:19-60`; landed via +https://github.com/Wei-Shaw/sub2api/pull/8389). + +**Pain points (issues).** Request-level upstream 403s misclassified as +account faults and failover-replayed until every account is disabled +(https://github.com/Wei-Shaw/sub2api/issues/8442); cooldown scope/duration +mismatched to the upstream signal +(https://github.com/Wei-Shaw/sub2api/issues/8439, /issues/8502, /issues/8479); +abnormal stream terminations masked as normal completion — synthetic +`end_turn` + `[DONE]` silently drop tool calls +(https://github.com/Wei-Shaw/sub2api/issues/8505); WebSocket failover +replayed the conversation's first request, stripping a running agent's +context (https://github.com/Wei-Shaw/sub2api/issues/8513); cross-protocol +conversion loses session IDs and agent semantics +(https://github.com/Wei-Shaw/sub2api/issues/8514, /issues/8461); key-to-group +and model scoping not enforced end-to-end — keys bound to one group +scheduled to other groups' accounts, and account-level model names fail to +constrain passthrough (https://github.com/Wei-Shaw/sub2api/issues/8480, +/issues/8492, /issues/8506); account connectivity testing does not scale — +one account can expose hundreds of models testable only one-by-one, with a +fixed prompt and no thinking-effort control +(https://github.com/Wei-Shaw/sub2api/issues/8454, /issues/8494). + +**Directions (pulls).** Quota-exhaustion semantics — final non-retryable +usage-limit errors carrying the real reset time +(https://github.com/Wei-Shaw/sub2api/pull/7239); prompt-cache engineering by +mimicking first-party client fingerprints — cached tokens 0 → 34,304 by +turn 4 (https://github.com/Wei-Shaw/sub2api/pull/8464); full pricing matrix +at model-add time (https://github.com/Wei-Shaw/sub2api/pull/8364); durable +outbox reworked after a production incident +(https://github.com/Wei-Shaw/sub2api/pull/8468); least-privilege admin keys +(https://github.com/Wei-Shaw/sub2api/pull/8476). + +**Gaps vs llm-proxy.** Background OAuth refresh pool +(`backend/internal/service/token_refresh_service.go:16-34`); upstream +client-identity emulation (`claude_code_version_sync_service.go:8-27`); +rule-based alerting engine (`ops_alert_evaluator_service.go:15-65`); egress +proxy pool with cycle-safe fallback (`proxy_fallback.go:7-40`); Redis leader +election for single-flight background jobs (`leader_lock.go:9-30`). + +### api7/aisix + +**Identity · Language · Form factor.** Apache-2.0 Rust AI gateway from the +original APISIX creators — 18-crate workspace (edition 2021, MSRV 1.93, axum +0.7), one static binary; OSS data plane beneath commercial AISIX +Enterprise/Cloud control planes; declarative `resources.yaml` (SIGHUP) or +etcd for clusters. + +**Maintenance.** Exceptionally active — 92 of 100 harvested PRs merged +2026-10-16 → 2026-11-04; 183 e2e scenario files against real gateway +processes. + +**Capabilities (code).** Six LB strategies incl. least-cost/least-latency +(TTFT EWMA) with retry budgets and configurable `fallback_on_statuses` +(`research-repos/api7__aisix/crates/aisix-core/src/models/routing.rs:46-63,301-332`); +two-phase rate limiting — RPM check-and-increment up front, TPM committed +after the response lands (`crates/aisix-ratelimit/src/lib.rs:1-16`); +ensemble models (fan-out + judge synthesis) (`crates/aisix-proxy/src/ensemble.rs:1-35`); +semantic routing over cached example embeddings +(`crates/aisix-proxy/src/semantic.rs:1-15`); guardrail chains incl. +quickjs-sandboxed custom scripts with a FAULT verdict distinguishable from +content blocks (`crates/aisix-guardrails/src/custom.rs:1-30`); MCP gateway +across revisions + OpenAPI-as-tools (`crates/aisix-mcp/src/gateway.rs:5-33`); +exact + semantic response cache and Anthropic `auto_prompt_caching` +breakpoint injection (`crates/aisix-cache/src/lib.rs:1-22`, +`crates/aisix-provider-anthropic/src/wire.rs:1322`); inbound OIDC/JWT with +JWKS rotation mapped onto API keys (`crates/aisix-proxy/src/jwt.rs:1-45`); +BPE usage-estimation fallback flagged `usage_estimated` +(`crates/aisix-proxy/src/token_estimate.rs:1-16`); ArcSwap lock-free config +snapshots (`crates/aisix-core/src/snapshot.rs:1-27`); resolver-level SSRF +guard with DNS pinning (`crates/aisix-gateway/src/ssrf.rs:1-14`). + +**Pain points (issues).** The single largest defect class is scan-vs-forward +asymmetry — guardrails scan one text but forward another, silently +(https://github.com/api7/aisix/issues/1016, /issues/696, /issues/1026, /issues/1103, /issues/1106); +rate limits and policies silently unenforced on non-chat endpoints +(https://github.com/api7/aisix/issues/775, /issues/697, /issues/715); usage +blind spots on secondary endpoints (https://github.com/api7/aisix/issues/748, /issues/699, /issues/688); +an external researcher filed three P2 policy-bypass bugs in one day — +Host-header route shadowing, `forward_client_headers` globs leaking +credentials, dual MCP-name parsers (https://github.com/api7/aisix/issues/1401, /issues/1403, /issues/1404); +credential rotation lagging ~59 minutes behind key replacement +(https://github.com/api7/aisix/issues/779); Retry-After up to 86,000 s +uncapped (https://github.com/api7/aisix/issues/1157). + +**Directions (pulls).** MCP per-user OAuth with keyless sign-in +(https://github.com/api7/aisix/pull/1394); native protocol fidelity beating +hub-and-spoke translation on Bedrock/Vertex +(https://github.com/api7/aisix/pull/1318); budget enforcement pushed to the +gateway so caps survive control-plane outages +(https://github.com/api7/aisix/pull/1366); OIDC `tag_claims` driving +chargeback attribution (https://github.com/api7/aisix/pull/1407); CI-enforced +per-route metric coverage (https://github.com/api7/aisix/pull/1367); +stateless session stickiness via tagged `encrypted_content` +(https://github.com/api7/aisix/pull/1332). + +**Gaps vs llm-proxy.** Inbound OIDC/JWT caller auth; ensemble and semantic +routing; A2A agent gateway (`crates/aisix-proxy/src/a2a.rs:1-13`); etcd +multi-replica config; object-store NDJSON telemetry sink with +content-addressed keys (`crates/aisix-obs/src/sink/object_store.rs:1-18`); +explicit passthrough routes that still detect the LLM envelope +(`crates/aisix-proxy/src/passthrough_route.rs:1-22`); PROXY protocol v1/v2 +(`crates/aisix-server/src/proxy_protocol.rs:1-12`); async video +submit→poll→credential-safe-fetch lifecycle (`crates/aisix-proxy/src/videos.rs:1-20`). + +### katanemo/plano + +**Identity · Language · Form factor.** Envoy-based AI-native data plane for +agentic apps: Rust sidecar (brightstaff) + proxy-wasm filters (llm_gateway, +prompt_gateway) + shared crates, driven by a Python `planoai` CLI that +renders Envoy config; hot path runs in sandboxed Wasm inside Envoy. + +**Maintenance.** Active — clone at release 0.4.39 (2026-10-24), 60 merged +PRs May–Oct 2026. + +**Capabilities (code).** Token-budget rate limiting keyed by a configurable +header selector (`research-repos/katanemo__plano/crates/common/src/ratelimit.rs:99-161`); +cost/latency-aware model ranking from live feeds (models.dev/DigitalOcean +pricing, Prometheus latency) (`crates/brightstaff/src/router/model_metrics.rs:84-232`); +session-affinity pinning with a routing budget — warm-cache loss priced, +paid switches capped at `max_switch_spend_pct` of a never-switch baseline +(`crates/brightstaff/src/handlers/llm/session_router.rs:176-216,333-430`); +automatic prompt-cache marker injection per (provider × model family × +upstream API), threshold-guarded and idempotent against client markers +(`crates/brightstaff/src/handlers/llm/prompt_caching.rs:25-70`); Envoy +RING_HASH prefix affinity for KV-cache locality +(`config/envoy.template.yaml:276-281,1005-1008`); SSE framing with +line-holdback, 1 MiB cap, and UTF-8 boundary safety +(`crates/hermesllm/src/apis/streaming_shapes/sse_chunk_processor.rs:9-57`); +Agentic Signals behavioral-quality analyzer +(`crates/brightstaff/src/signals/mod.rs:1-19`). + +**Pain points (issues).** Output-filter guardrails stateless per SSE chunk +and fail open — a secret split across two events passes +(https://github.com/katanemo/plano/issues/1036); multibyte UTF-8 split at +chunk boundaries silently truncated Anthropic streams in production +(https://github.com/katanemo/plano/issues/980); request-derived data can +panic the entire Envoy WASM VM (https://github.com/katanemo/plano/issues/999, +/issues/1030); session traces unattributable because session id comes only +from a Plano-internal header (https://github.com/katanemo/plano/issues/979); +pricing-feed degradation silently weakens the routing budget +(https://github.com/katanemo/plano/issues/992); switch-cost math ignores +cross-model tokenizer variance (https://github.com/katanemo/plano/issues/991). + +**Directions (pulls).** The cost-aware switching arc +(https://github.com/katanemo/plano/pull/982, /pull/1016); server-side turn +semantics for coding agents (`route_on_user_only` replays the prior decision +on tool-result turns, https://github.com/katanemo/plano/pull/1015); +Responses SSE byte-level framing hardening +(https://github.com/katanemo/plano/pull/988); usage-metering completeness — +`Accept-Encoding: identity` forced and `stream_options.include_usage` +injected so usage parses (https://github.com/katanemo/plano/pull/1043). + +**Gaps vs llm-proxy.** Agent orchestration listeners +(`crates/brightstaff/src/handlers/agents/selector.rs:21-26`); Agentic +Signals OTel taxonomy; per-session switch-spend budget; cache-marker +auto-injection; replica-level prefix-hash affinity; shared Redis session +state; live external metrics feeds; stateful Responses emulation with +PostgreSQL persistence (`crates/brightstaff/src/state/postgresql.rs:10-16`); +PostHog product-analytics exporter +(`crates/brightstaff/src/tracing/posthog_exporter.rs`). + +### mozilla-ai/otari + +**Identity · Language · Form factor.** Python 3.13 FastAPI modular monolith +(gateway + management API + React dashboard in one process) delegating to +any-llm for 40+ providers; PostgreSQL in compose, SQLite for dev, 142 +alembic migrations; Apache-2.0 open core with a hosted otari.ai edition. + +**Maintenance.** Extremely active — v0.36.0 → v0.38.0 released +2026-11-03/04; 73 of the last 100 PRs merged within three days. + +**Capabilities (code).** Routing compiled to an ordered plan where every +dropped candidate carries a machine-readable reason +(`research-repos/mozilla-ai__otari/src/gateway/services/routing/compiler.py:118-150`); +attempt walker with retryable classification, per-attempt credential +re-binding, and tool-loop lock-in (fallback stops after the first assistant +message because provider-specific tool ids cannot be replayed) +(`src/gateway/api/routes/_attempts.py:235-314,414-474`); pluggable routers +under a hard 5 s deadline that fall back to the default target +(`src/gateway/services/routing/decide.py:49-52`); exact sliding-window log +rate limiting with cost-carrying entries admitted then settled to true cost +(`src/gateway/rate_limit.py:63-124`); reserve-then-settle budgets in BIGINT +with `BudgetReservation` rows swept back on every axis +(`src/gateway/models/budgets.py:200-296`); Idempotency-Key claims before +budget reservation, replays stored encrypted with request-shaping headers +in the fingerprint (`src/gateway/api/routes/_idempotency.py:50-79`); +provider-neutral cost math reconciling OpenAI vs Anthropic cache-token +conventions with `require_pricing` refusal +(`src/gateway/core/metered_pricing.py:17-35`); format-neutral SSE plumbing +with keepalives and settlement decoupled from streaming +(`src/gateway/streaming.py:34-238`). + +**Pain points (issues).** Guardrail in block mode with `on_unavailable: +block` served the request when the external guardrail returned an +inconclusive verdict (https://github.com/mozilla-ai/otari/issues/2788); +fallback candidates are not interchangeable — Bedrock 400s a field the +first candidate accepted (https://github.com/mozilla-ai/otari/issues/2790); +every usage/budget read API requires the master key hosted tenants never +have (https://github.com/mozilla-ai/otari/issues/2863 and +/issues/2792 — apps must poll one request at a time or discover limits by +triggering refusals); the code-execution +tool is billed but surfaces no cost (https://github.com/mozilla-ai/otari/issues/2864); +models.dev catalog half-used — model type guessed from names, mislabeling +96 transcription models as chat (https://github.com/mozilla-ai/otari/issues/2782). + +**Directions (pulls).** models.dev as canonical catalog, persisted and +refusal-on-unreadable (https://github.com/mozilla-ai/otari/pull/2771, /pull/2787); +pricing schema converging on models.dev's exact unit vocabulary +(https://github.com/mozilla-ai/otari/pull/2812, /pull/2820); tool calls as +first-class billable units (https://github.com/mozilla-ai/otari/pull/2831); +model-free tool endpoints (`/api/v1/code/execute`, `/api/v1/fetch`) +(https://github.com/mozilla-ai/otari/pull/2803); learned kNN routers removed +for privacy after teaching stored tenant prompts in plaintext +(https://github.com/mozilla-ai/otari/pull/2829); estimated usage row when a +client abandons a stream (https://github.com/mozilla-ai/otari/pull/2869). + +**Gaps vs llm-proxy.** Agent Guardrails laptop-side CLI enforcing +repo-committed gates at PreToolUse/Stop (`docs/agent-guardrails.md:3-91`); +workload-identity federation — OIDC identity tokens + jwt-bearer exchange +compatible with Claude Code federation env vars (`docs/workload-identity.md:21-48`); +external usage ingestion (`docs/external-usage.md:23-37`); OTLP ingest +booking tokens from exported traces (`docs/external-usage.md:86-116`); +models.dev default pricing with timestamp-effective rates +(`src/gateway/services/pricing_refresh_service.py:52-53`); replica-shared +Redis rate-limit store (`src/gateway/adapters/rate_limit_store_adapter.py:27-63`). + +### musistudio/claude-code-router + +**Identity · Language · Form factor.** TypeScript/Node pnpm monorepo +(@musistudio/claude-code-router v3.1.3): Electron desktop app, CLI, and +Docker — a local gateway/control-plane for coding-agent CLIs (Claude Code, +Codex, Grok, Kimi, OpenCode, Pi, ZCode) with a web management UI; the HTTP +core is an external plugin runtime supervised as a child process. + +**Maintenance.** Active but backlogged — last merge 2026-10-10 with 55 open +PRs queued; the harvest window shows deep, well-investigated bug reports +against 3.1.1–3.1.3. + +**Capabilities (code).** Config compiled up front with per-rule +diagnostics — a bad rule goes inactive while the rest keep serving +(`research-repos/musistudio__claude-code-router/packages/core/src/routing/config-compiler.ts:46-70`); +fallback as a precomputed execution plan (bounded same-model retry or +ordered model chain) driven by a status-code classifier with retry-after +clamped to 60 s (`packages/core/src/routing/execution-plan.ts:24-45`, +`failure-classifier.ts:10-31`); user-supplied JS routing rules in a +sandboxed worker with code generation disabled and byte budgets +(`packages/core/src/routing/route-script-worker.ts:12-58`); upstream +credential pools with per-credential cooldown and spillover +(`packages/core/src/providers/credential-pool.ts:16-65`); client API keys +with expiry, constant-time comparison, and pre-request reservations across +rpm/rph/rpd/tpm/tph/tpd windows estimated from the request body +(`packages/core/src/gateway/auth/api-key-authorizer.ts:81-162`); five client +protocol surfaces incl. Gemini (`packages/core/src/routing/protocol-endpoints.ts:3-21`); +SQLite usage ledger with cache-token splits and virtual-model attribution +(`packages/core/src/usage/store.ts:40-72`); stream-experience metering +(TTFS/TTFT, inter-event gap p95) (`packages/core/src/observability/stream-experience.ts:27-90`). + +**Pain points (issues).** Fallback chains silently didn't execute — an +ingress header overrode the per-attempt body on every hop +(https://github.com/musistudio/claude-code-router/issues/1850); fallback +outcomes misreported on every surface — dashboards showed 65% success when +clients saw 98.4% (https://github.com/musistudio/claude-code-router/issues/1870, /issues/1875, /issues/1927); +the Node process died under real Claude Code traffic (2–20 MB bodies) via +unhandled error events and log-spool OOMs (https://github.com/musistudio/claude-code-router/issues/1846, /issues/1867); +translation dropped `max_tokens` and vendor `thinking` into upstream 400s +(https://github.com/musistudio/claude-code-router/issues/1843, /issues/1937); +a truncated upstream stream surfaced as a bare socket destroy with no +terminal event (https://github.com/musistudio/claude-code-router/issues/1917); +Codex OAuth import structurally broken by scope demands and per-profile +refresh-token reuse (https://github.com/musistudio/claude-code-router/issues/1941, /issues/1879); +no automatic same-provider credential failover — users must model every API +key as a separate provider, and the filed feature request asks for pool +rotation on 429/401/403/5xx/timeout with Retry-After awareness and +quarantine of persistently unauthorized keys +(https://github.com/musistudio/claude-code-router/issues/1923). + +**Directions (pulls).** Multi-attempt diagnosability — per-attempt +`[stage|status]` summaries appended to `error.message` because Claude Code +renders only that (https://github.com/musistudio/claude-code-router/pull/1773); +prompt-cache affinity headers on Codex legs +(https://github.com/musistudio/claude-code-router/pull/1866); `/compact` +tool_result preservation (https://github.com/musistudio/claude-code-router/pull/1824); +pricing keyed to the routed physical model +(https://github.com/musistudio/claude-code-router/pull/1790); request-log +prune ordering fixed after 13 GB of orphaned bodies +(https://github.com/musistudio/claude-code-router/pull/1931). + +**Gaps vs llm-proxy.** Declarative + scriptable routing policy engine +(`docs/.../routing.md:64-135`); agent-aware subagent model selection via a +`` tag; Fusion capability-composition virtual models +(vision pre-digestion, hosted search); ToolHub lazy MCP tool loading +(`packages/core/src/mcp/toolhub-mcp.ts:185-352`); interactive OAuth login +import from local agent CLIs (`docs/.../providers.md:8-78`); bundled +generated price/context catalog (`packages/core/models.json`); agent launch +profiles that write the client's own config (`docs/.../profiles.md:10-37`). + +### raine/claude-code-proxy + +**Identity · Language · Form factor.** Single-crate Rust binary (edition +2024, axum 0.8, ~17.5k LoC, v0.1.48) presenting an Anthropic-Messages +endpoint and translating for consumer-subscription backends: Codex/ChatGPT, +Kimi, Grok, Cursor Agent, OpenCode Go. One Provider trait with per-provider +modules rather than a hub-and-spoke core. + +**Maintenance.** Very active — weekly-ish releases (v0.1.48 on 2026-11-01), +62 of 100 harvested PRs merged Jul–Oct 2026. + +**Capabilities (code).** Per-session provider affinity from validated +session headers in a bounded LRU (`research-repos/raine__claude-code-proxy/src/session.rs:49-115`); +stateless reasoning replay — upstream encrypted reasoning base64-packed +into a synthetic `ccp:codex:v1:` thinking signature the client echoes back +(`src/providers/codex/translate/reasoning_signature.rs:26-60`); dual-transport +upstream with a pooled Responses-over-WebSocket protocol plus automatic HTTP +degradation (`src/providers/codex/websocket.rs:40-58`); budget-aware retry +on 429/5xx with Retry-After honored and in-band SSE errors also classified +(`src/retry.rs:4-41`); full consumer OAuth — PKCE browser login, device +code, single-flight refresh, keychain-encrypted token files +(`src/providers/codex/auth/manager.rs:16-105`); live ratatui monitor plus a +loopback JSON `/monitor` endpoint for remote dashboards +(`src/monitor.rs:71-136`, `src/monitor/remote.rs:13-52`); opt-in traffic +capture with 15-key redaction (`src/traffic.rs:44-105`); auxiliary-request +rerouting (Claude Code's background security-review requests detected and +routed to a cheap model) (`src/server.rs:52-108`); subscription quota +telemetry parsed out-of-band into the usage block (`src/providers/codex/client.rs:1787-1796`). + +**Pain points (issues).** No per-backend tool-schema sanitization — +Codex/Grok hard-400 on Claude Code tool fields +(https://github.com/raine/claude-code-proxy/issues/142, /issues/109); +in-memory session state mis-keyed — subagents sharing a parent session id +destroyed the main conversation's state (74% 409 rate measured) +(https://github.com/raine/claude-code-proxy/issues/163, /issues/131); +exhausted 503 without `x-should-retry: false` stacked Claude Code's retry +loop on the proxy's own — one turn hung ~30 minutes +(https://github.com/raine/claude-code-proxy/issues/174); the model registry +is hardcoded so every new model needs a release +(https://github.com/raine/claude-code-proxy/issues/198, /issues/169); +subscription upstreams change protocol details without notice and break the +proxy same-day (https://github.com/raine/claude-code-proxy/issues/137, /issues/180). + +**Directions (pulls).** "Semantic commit point" unification so retries only +replay before visible output (https://github.com/raine/claude-code-proxy/pull/103); +`x-should-retry: false` terminality signals (https://github.com/raine/claude-code-proxy/pull/175); +quota/reset-clock surfacing into the Anthropic usage block +(https://github.com/raine/claude-code-proxy/pull/149, /pull/139); runtime +model catalog from the vendor CLI's own cache +(https://github.com/raine/claude-code-proxy/pull/130); 64 MiB body limit +because image-heavy sessions re-send full history every turn +(https://github.com/raine/claude-code-proxy/pull/152); `SSL_CERT_FILE` / +native trust roots (https://github.com/raine/claude-code-proxy/pull/143). + +**Gaps vs llm-proxy.** OAuth subscription login; server-side compaction +with a retained-token budget (`src/providers/codex/compaction.rs:16-59`); +incremental turn continuation via `previous_response_id` + semantic prefix +validation (`src/providers/codex/continuation.rs:8-62`); encrypted-reasoning +replay (llm-proxy's `Thinking.signature` is passthrough-only — a grep for +`encrypted_content` across llm-proxy crates returns nothing); upstream +WebSocket transport; live attachable monitor; XML tool-call synthesis for +models without native function calling (`src/providers/cursor/tool_use_xml.rs:1-50`); +corporate TLS trust-store support (llm-proxy builds reqwest with bundled +roots only, `crates/llm-proxy-provider/Cargo.toml:23`). + +### router-for-me/CLIProxyAPI + +**Identity · Language · Form factor.** Go 1.26 single-binary LLM gateway +(module v8, v8.0.54) fronting CLI-subscription OAuth accounts (Codex, +Claude Code, Gemini/Antigravity, Grok, Kimi, Meta, Devin) plus API-key +providers behind OpenAI/Anthropic/Gemini-compatible endpoints; ships an +embedding SDK, c-shared plugin host, bubbletea TUI, and web panel. + +**Maintenance.** Extremely active — 22 PRs merged 2026-10-27 → 2026-11-04; +issues closed within a day. + +**Capabilities (code).** Multi-strategy credential scheduler (round-robin, +fill-first, smooth WRR, plugin strategies) with cooldown queues +(`research-repos/router-for-me__CLIProxyAPI/sdk/cliproxy/auth/scheduler.go:21-24`); +request-path retry/failover conductor with per-credential error rules +(stop / stop-and-cooldown / continue) (`sdk/cliproxy/auth/conductor_execution.go:123-183`); +session-sticky routing keyed from Claude/Codex session headers, +`prompt_cache_key`, and conversation IDs, with subagent inheritance for +KV-cache locality (`sdk/cliproxy/auth/session_cache.go:31-45`); +in-band overload failover via stream bootstrap buffering — handshake/heartbeat +frames held within a 48-frame/1 MiB budget so a `server_is_overloaded` +rejection inside an HTTP 200 stream can still be failed over before +downstream headers commit (`internal/runtime/executor/codex_executor_stream.go:142-200`); +single-port protocol multiplexer (Redis RESP SUBSCRIBE for live usage) +(`internal/api/protocol_multiplexer.go:125-207`); realtime WebRTC voice +relay (`internal/client/codex/live/media.go:24-58`); uTLS client +impersonation and CLI device profiles (`internal/runtime/executor/helps/utls_client.go:27-31`); +pairwise translator registry rather than hub-and-spoke +(`sdk/translator/registry.go:29-48`); declarative gjson/sjson payload +rewriting (`config.example.yaml:270-336`); debounced hot reload +(`internal/watcher/config_reload.go:1-43`); management API v8 over the whole +config tree (`internal/api/server_management_v8.go:13-69`). + +**Pain points (issues).** Cross-dialect translation loses semantics — +omitted Gemini roles, split parallel tool batches, dropped Devin reasoning +(https://github.com/router-for-me/CLIProxyAPI/issues/7186, /issues/7177, /issues/7228); +coarse error classification takes whole credentials out of service — a +model-scoped 429 cooled an entire Claude credential ~6 days +(https://github.com/router-for-me/CLIProxyAPI/issues/7197, /issues/7241); +OAuth refresh destroys or spins credentials when the store is read-only +(https://github.com/router-for-me/CLIProxyAPI/issues/7214); mid-stream +failures unrecoverable once bytes were forwarded +(https://github.com/router-for-me/CLIProxyAPI/issues/7169); usage conflates +unknown with zero — cache fields absent upstream log as 0% hit rate +(https://github.com/router-for-me/CLIProxyAPI/issues/7157). + +**Directions (pulls).** Quota-reset-aware `soonest-reset` scheduling from +passively observed quota headers (https://github.com/router-for-me/CLIProxyAPI/pull/7246); +final non-retryable usage-limit errors carrying reset times +(https://github.com/router-for-me/CLIProxyAPI/pull/7239); per-credential +connection pooling (https://github.com/router-for-me/CLIProxyAPI/pull/7101); +fine-grained cache-bucket usage accounting (https://github.com/router-for-me/CLIProxyAPI/pull/7074); +embeddability hooks (https://github.com/router-for-me/CLIProxyAPI/pull/7124); +per-key model policies (https://github.com/router-for-me/CLIProxyAPI/pull/7117). + +**Gaps vs llm-proxy.** OAuth login flows for CLI-subscription providers +(`internal/auth/`); shipped credential-pool failover with persisted cooldown +state; session-affinity sticky routing; a management write API (llm-proxy's +admin API is read-only usage); hot reload; client impersonation; a dedicated +reasoning/thinking translation engine converting effort levels to provider +budgets (`internal/thinking/convert.go:8-45`); cache-preserving replay +caches; the payload rules engine; the plugin platform + public SDK. + +### smg-project/smg + +**Identity · Language · Form factor.** Apache-2.0 Rust workspace (26 +members; `model_gateway` binary `smg` plus protocol/parser/router/mesh/wasm +crates and Python/Go bindings; axum 0.8 + tonic 0.14) — an engine-agnostic +model-routing gateway for self-hosted engines (vLLM, SGLang, TensorRT-LLM, +MLX) plus cloud providers; binary, Docker, Helm, PyPI. + +**Maintenance.** Extremely active — HEAD 2026-11-05; 67 of 100 harvested +PRs merged in November 2026; nightly fuzz triage and CI-health bots. + +**Capabilities (code).** Ten LB policies incl. cache-aware routing that +mirrors worker KV-cache radix trees with cache namespaces +(`research-repos/smg-project__smg/model_gateway/src/policies/mod.rs:19-42`, +`src/policies/cache_aware.rs:1-60`); priority-aware admission scheduler with +preemption and distinct rejections (`src/middleware/scheduler/engine.rs:30-50`); +retry + per-endpoint breakers with an attempt ledger charging a worker at +most one failure per request (`crates/external_router/src/retry.rs:96-104`); +per-tenant reserve/settle token limits (`src/rate_limit/mod.rs:1-9`); +21 tool-call parsers and 16 reasoning parsers with auto model detection +(`crates/tool_parser/src/factory.rs:447`); WASM plugin middleware over a +runtime REST API (`model_gateway/src/wasm/route.rs:1-10`); gossip mesh HA +with CRDT state replication and mTLS (`crates/mesh/src/lib.rs:1-24`); +OTel + 340 Prometheus sites (`src/observability/otel_trace.rs:1-4`); +Kubernetes/Consul/Slurm/file discovery (`src/service_discovery/mod.rs:1-32`); +two-level cached tokenizer with chat templates (`crates/tokenizer/src/lib.rs:20-35`). + +**Pain points (issues).** Hardcoded multimodal image-count limits reject +requests the model would serve (https://github.com/smg-project/smg/issues/5044); +a streaming reasoning parser replays the whole already-streamed buffer as +content — clients receive the reply twice at ~1% of long-session messages +(https://github.com/smg-project/smg/issues/5038); the suggested mitigation +knob is dead config for that parser (same issue); nightly fuzz found a +JSON-prefix breach class in the pythonic tool-call parser +(https://github.com/smg-project/smg/issues/5056). + +**Directions (pulls).** Hand-written one-pass SSE serializers replacing +serde on every stream family, byte-pinned against fixtures +(https://github.com/smg-project/smg/pull/4987, /pull/5043, /pull/5031, /pull/5035); +load telemetry moving from polling to push +(https://github.com/smg-project/smg/pull/4971); metrics-correctness pass +closing per-endpoint coverage (https://github.com/smg-project/smg/pull/5017); +MCP tool execution first-class in the Responses path +(https://github.com/smg-project/smg/pull/5045); chat-template byte parity +with the serving engine (https://github.com/smg-project/smg/pull/5052). + +**Gaps vs llm-proxy.** Tool-call/reasoning text parsers for self-hosted +models; KV-state-aware cache routing; WASM runtime-extension API; +infrastructure service discovery; multi-node mesh HA; pluggable chat-history +storage with migrations; priority admission with preemption +(`src/middleware/scheduler/admission.rs:1`); a real tokenizer stack — +llm-proxy's offline tiktoken with a ~4-chars/token fallback silently +mismatches non-OpenAI vocabularies; OIDC/JWKS caller auth +(`crates/auth/src/jwks.rs:1-13`); MCP approval engine with audit ring +(`crates/mcp/src/approval/policy.rs:1-22`). + +### zilliztech/GPTCache + +**Identity · Language · Form factor.** Python library (pip `gptcache`) plus +an optional FastAPI server implementing a modular semantic cache for LLM +responses — pluggable scalar/vector stores, embedders, similarity +evaluators, eviction policies. MIT, from Zilliz. + +**Maintenance.** Caretaker-mode with a Sept 2026 revival — v0.1.44 shipped +Aug 2024, then collaborator xiaofan-luan merged adapters + an LLM-judged +evaluator + a reuse benchmark on 2026-09-22 and opened the v0.2.0 release +PR; 41 of 56 harvested true issues sit open, some unanswered since 2023. + +**Capabilities (code).** A composable `adapt()` pipeline (pre-process → +embed → ANN search → re-rank → post-process → save) where each stage is an +injected callable (`research-repos/zilliztech__GPTCache/gptcache/adapter/adapter.py:12-107`); +every evaluator declares its score range so one user threshold works across +scorers (`gptcache/similarity_evaluation/similarity_evaluation.py:21-41`); +streaming cached without changing the stream contract — hits replay a +synthetic three-chunk SSE sequence (`gptcache/adapter/openai.py:78-106`); +session-scoped isolation with an anti-repeat hit filter +(`gptcache/session.py:48-76`); two-tier eviction with vector-index rebuild +(`gptcache/manager/eviction_manager.py:11-46`); two-store consistency +self-heal under `data_check` (`gptcache/adapter/adapter.py:122-131`); +L1/L2 chaining via `next_cache` (`gptcache/core.py:44,80`); template-aware +cache keys (`gptcache/processor/pre.py:77-135`); LLM-as-verifier hit gate +(`gptcache/processor/post.py:93-205`). + +**Pain points (issues).** The server answered the first question's answer +for every subsequent unrelated question under OpenAI embeddings — ~100% +false hits (https://github.com/zilliztech/GPTCache/issues/612); unsafe YAML +loading gave config-file RCE, reported twice and left open +(https://github.com/zilliztech/GPTCache/issues/676, /issues/687); +`BufferedReader.peek()` hashed only ~8 KiB of files so different files +collide — cross-user disclosure (https://github.com/zilliztech/GPTCache/issues/684); +a timing side channel distinguishes hits from misses at 85.3% accuracy +(https://github.com/zilliztech/GPTCache/issues/652); multi-node deployments +unsupported (https://github.com/zilliztech/GPTCache/issues/666); no +read-only mode (https://github.com/zilliztech/GPTCache/issues/656). + +**Directions (pulls).** The revival: Anthropic Messages + modern OpenAI +adapters plus JEV (https://github.com/zilliztech/GPTCache/pull/701); +cache-hit judgment moving from embedding similarity to an LLM judge asking +five atomic reuse questions at a calibrated 0.70 threshold +(https://github.com/zilliztech/GPTCache/pull/701, +`gptcache/similarity_evaluation/jev.py:186-297`); a frozen 1,536-case +cache-reuse benchmark — JEV 99.7% precision vs 42.2% for the default +reranker at 0.80 (https://github.com/zilliztech/GPTCache/pull/702); +cost-aware eviction research PRs open (W-TinyLFU, GDSF, ARC, CARMA) +(https://github.com/zilliztech/GPTCache/pull/680, /pull/689, /pull/692, /pull/696). + +**Gaps vs llm-proxy.** LLM-judged reuse gating and a public hit-policy +correctness dataset; per-request cache controls on the wire (`cache_skip`, +temperature-derived bypass, `cache_factor`); session anti-repeat policy; +pluggable key preprocessing separating store key from match key; multi-tier +chaining; two-store consistency self-heal; hit-audit accounting with +client-visible token savings +(`research-repos/zilliztech__GPTCache/gptcache/adapter/openai.py:111-122`). + +### eliasbiondo/antirion + +**Identity · Language · Form factor.** Apache-2.0 Rust workspace (~28 +crates, edition 2024, MSRV 1.98, v0.1.1) shipping one binary with an +embedded Next.js admin console; standalone (SQLite) or clustered +(PostgreSQL + Valkey); positioned as "the fastest open-source AI gateway" +with a reproducible benchmark harness vs Bifrost and LiteLLM. + +**Maintenance.** Young and single-author but disciplined — public since +2026-10-17, nightly supply-chain and per-parser fuzzing CI gating every PR; +human reports triaged within days. + +**Capabilities (code).** Routing trees compiled to an arena at config time — +nine node kinds incl. weighted, bandit, least-latency, conditional, +classifier, tier (`research-repos/eliasbiondo__antirion/crates/antirion-router/src/tree/mod.rs:80-152`); +selection state (PeakEWMA latency, success EWMA, in-flight) keyed by stable +identities that survive config swaps (`crates/antirion-router/src/state.rs:1-38`); +request hedging plus a cross-request retry budget (ratio 0.2 over 10 s) +(`crates/antirion-resilience/src/hedging.rs:1-4`, `src/budget.rs:1-32`); +exact integer GCRA mirrored in Lua with a differential test +(`crates/antirion-limits/src/math.rs:1-70`); RFC 8785 JCS canonical cache +keys (`crates/antirion-cache/src/jcs.rs:1-3`); single-flight cache misses +via watch channels (`crates/antirion-cache/src/flights.rs:1-12`); semantic +cache with a negation lexicon so "don't enable X" cannot match "enable X" +(`crates/antirion-cache/src/semantic.rs:1-25`); guardrails with +format-preserving redaction (`crates/antirion-guardrails/src/preserve.rs:1-19`); +MCP gateway bridging both protocol eras with stateless XChaCha20-Poly1305 +session tokens (`crates/antirion-mcp/src/token.rs:1-7`); WASI 0.2 plugin +extensions sha256-pinned (`crates/antirion-ext/src/wasm/mod.rs:1-13`); +three-class egress SSRF catalogue (`crates/antirion-egress/src/ssrf.rs:1-23`); +thread-per-core data plane with a workspace `deny(unsafe_code)` +(`crates/antirion-server/src/threads.rs:1-38`); disk-spooled idempotent +spend ledger — a full spool refuses budgeted requests with 503 rather than +losing money (`crates/antirion-store/src/lib.rs:3`). + +**Pain points (issues).** Nightly cargo-deny/audit failures outpace +remediation — 12 bot issues Oct 21–Nov 4, five open with no triage +(https://github.com/eliasbiondo/antirion/issues/63); a fuzz crash found +Nov 1 sits open and gates the release workflow +(https://github.com/eliasbiondo/antirion/issues/60); retry predicates +cannot branch on response payload values +(https://github.com/eliasbiondo/antirion/issues/47); provider model +listing stops at a hard 20-page cap with no partial-catalog signal +(https://github.com/eliasbiondo/antirion/issues/41). + +**Directions (pulls).** A typed decisions endpoint (`POST /v1/systemone`) +as a new endpoint family where routing defaults to refusing the unknown +operation (https://github.com/eliasbiondo/antirion/pull/48); supply-chain +sweeps as merged maintenance work (https://github.com/eliasbiondo/antirion/pull/34, /pull/44); +build-time gzip embedding of the console (https://github.com/eliasbiondo/antirion/pull/45). + +**Gaps vs llm-proxy.** A2A agent proxy (`crates/antirion-mcp/src/a2a.rs:1-5`); +WASM plugin extensions; dry-run simulation endpoints for routes and limits +(`crates/antirion-admin/src/simulate.rs:1`); shadow-traffic mirroring with +measured divergence (`crates/antirion-router/src/shadow.rs:1-28`); +multi-node cluster mode; enterprise SSO/SAML/SCIM/RBAC; tenant erasure as a +store operation (`crates/antirion-store/src/erasure.rs:1-16`); signed +attested releases (SBOM + Sigstore + SLSA L3, `.github/workflows/release.yml:2-3`); +in-repo fuzzing of every untrusted parser (`fuzz/fuzz_targets/`); Gemini +and Bedrock client dialects (`crates/antirion-pipeline/src/accept.rs:382-391`). + +### deanxv/done-hub + +**Identity · Language · Form factor.** Go 1.26 monolith (Gin + GORM + +embedded React 18) forked from one-hub/one-api — a full commercial LLM +gateway with ~45 upstream adapters incl. dedicated client-reverse-proxy +channels (Claude Code, Codex, Gemini CLI, Antigravity), payments, and an +MCP server; single Docker image, MySQL/PostgreSQL/SQLite. + +**Maintenance.** Active (last commit 2026-10-27); community-PR intake is +weekend-cadence with maintainer-side equivalents landing directly. + +**Capabilities (code).** Failover loop capping retries at available-channel +count, accumulating skip ids, and freezing a failed (channel, model) with a +cooldown preferring the upstream's exact reset time +(`research-repos/deanxv__done-hub/relay/main.go:74-176`, +`model/balancer.go:95-163`); layered admission control — per-token model +allowlists enforced pre-routing (`CheckLimitModel` over +`token.setting.limits.models[]`, `relay/common.go:72-111`), token-group +validation, a per-user-group Redis API limiter, dual user/IP rate buckets, +and CIDR whitelisting (`middleware/distributor.go:22-70`, +`middleware/rate-limit.go:29-113`); Redis sticky sessions keyed per CLI-client +protocol (Claude `metadata.user_id`, Gemini UA+IP, Codex `session_id`) +(`model/balancer.go:194-309`); raw-upstream passthrough preserving unknown +fields and upstream rate-limit headers while final errors are collapsed to +hide upstream identity but preserve 429 for client backoff +(`relay/common.go:416-468,796-877`); SSE `::PING` heartbeats after idle and +post-disconnect drain so usage/billing stays accurate +(`relay/relay_util/heartbeat.go:14-175`); auto channel disabling with +keyword exemptions (middleware + `relay/common.go:764-792`); OAuth account +channels with auto-refresh persisted back to the database +(`providers/claudecode/base.go:56-89`); the gateway itself as an MCP server +with a per-user dashboard tool (`mcp/server.go:14-75`); scheduled price +sync from models.dev (`model/pricing_modelsdev.go:18-108`). + +**Pain points (issues).** Strictly-typed unmarshalling breaks streams when +real clients send object-vs-string fields (https://github.com/deanxv/done-hub/issues/58, /issues/25); +Codex/Responses compatibility churns release-over-release +(https://github.com/deanxv/done-hub/issues/54); upstream +Claude-Code-only providers fingerprint and reject relayed requests — +byte-exact passthrough requested (https://github.com/deanxv/done-hub/issues/16); +cache-token accounting lossy across dialects (https://github.com/deanxv/done-hub/issues/51, /issues/55); +no request/response observability of what was actually sent upstream +(https://github.com/deanxv/done-hub/issues/32); no multi-key channels — one +key per channel forces duplication (https://github.com/deanxv/done-hub/issues/57); +no self-hosted/personal mode with the commercial layer off +(https://github.com/deanxv/done-hub/issues/14). + +**Directions (pulls).** Wire-exact Responses compatibility — removing +`omitempty` on `usage.input_tokens_details.cached_tokens` fixed Codex CLI +parsing (https://github.com/deanxv/done-hub/pull/39); Anthropic cache fields +parsed on OpenAI-compatible channels (https://github.com/deanxv/done-hub/pull/52); +a JSON-Schema→Gemini-Schema structural converter because Antigravity +silently ignores `parametersJsonSchema` (https://github.com/deanxv/done-hub/pull/59); +nested `remove_params` paths for per-channel body surgery +(https://github.com/deanxv/done-hub/pull/44). + +**Gaps vs llm-proxy.** Gemini-native client routes +(`router/relay-router.go:120-127`); consumer-OAuth CLI channels with +impersonation headers (`providers/codex/base.go:28-148`); Gemini 3 +`thoughtSignature` recovery via the official sentinel + one idempotent +retry (`relay/common.go:735-759`, issue #28); cross-dialect cached-token +normalization into one Usage type (`types/common.go:10-135`); channel-level +parameter policy (`providers/base/common.go:608-705`); client-header +passthrough rules (`providers/base/common.go:160-266`); +`context.WithoutCancel` disconnect settlement (`providers/base/common.go:276-301`); +`{model}` URL templating (`providers/openai/base.go:138-189`); no-cache +headers on the admin surface after a CDN-cached authorization bypass +(`middleware/cache.go:8-16`); untranslated passthrough relay for whole +endpoint families (`router/relay-router.go:53-65`). + +### messkan/prompt-cache + +**Identity · Language · Form factor.** Go 1.24 single-binary semantic cache +(Gin + BadgerDB + in-memory ANN) exposing an OpenAI-compatible +`/v1/chat/completions` that serves near-duplicate prompts from an +embedding-similarity cache; OpenAI, Mistral, and Claude+Voyage providers. + +**Maintenance.** Maintainer-driven, moderate — v0.4.0 (2026-04-25), last +merge 2026-08-18; two external PRs (ANN staleness fix, MiniMax provider) +open and uncommented since Jul/Sep 2026. + +**Capabilities (code).** Two-threshold matching — ≥0.70 hit, <0.30 miss, +gray zone deferred to an LLM "semantic judge" +(`research-repos/messkan__prompt-cache/internal/semantic/semantic.go:256-336`); +SSE transparency both ways — misses pipe verbatim while reassembling, +hits re-synthesize a fresh chunk sequence +(`internal/semantic/providers.go:40-111`, `cmd/api/main.go:198-273`); +cross-protocol translation inside the cache provider +(`internal/semantic/providers.go:815-1103`); per-caller namespace +partitioning where namespaced lookups use filtered scans so candidates +never cross a partition (`internal/cachecontext/namespace.go:10-83`); +persistent two-tier cache with an hourly TTL janitor +(`internal/cache/cache.go:76-143`); runtime threshold reconfiguration and +provider hot-swap (`cmd/api/main.go:383-443`); cache warming API +(`cmd/api/main.go:492-560`); retryable upstream client for 429/5xx +(`internal/http/client.go:74-146`); `X-Cache`/`X-Similarity-Score` response +headers + Prometheus (`internal/metrics/metrics.go:86-186`); multimodal +cache-key hygiene via media digests (`cmd/api/content.go:40-105`). + +**Pain points (issues).** Cache invalidation is TTL-only so RAG users get +stale answers when the corpus changes (https://github.com/messkan/prompt-cache/issues/8); +no multi-tenancy — one shared token, and the README itself warns "semantic +similarity is not an authorization mechanism" (https://github.com/messkan/prompt-cache/issues/9); +an adopter asked when v1.0.0 ships before committing (unanswered ~3 months) +(https://github.com/messkan/prompt-cache/issues/16); OpenShift non-root +execution broken at release time (https://github.com/messkan/prompt-cache/issues/10). + +**Directions (pulls).** Namespace isolation landed claims-first — docs +disclosure before the mechanism, then tests +(https://github.com/messkan/prompt-cache/pull/13, /pull/14, /pull/15); +multimodal content parts with media digests in the cache key +(https://github.com/messkan/prompt-cache/pull/17); container hardening for +arbitrary-UID security contexts (https://github.com/messkan/prompt-cache/pull/12). + +**Gaps vs llm-proxy.** A two-threshold semantic cache with an LLM gray-zone +verifier; persistent TTL cache with LRU; cache-aware streaming playback; +multimodal cache-key derivation; per-tenant partitioning of shared state; +mutating admin API (`PATCH /v1/config`); cache lifecycle admin +(list/delete/warm); Prometheus exposition; request-path HTTP retry. + +### EeroEternal/xlb + +**Identity · Language · Form factor.** Single-crate Rust (edition 2021, +tokio + httparse + rustls, ~4.8k LoC) streaming-first thin-L7/L4 load +balancer that fronts AI gateways (LiteLLM, New API, xlite, AISIX) — it +forwards bytes, not tokens, with zero payload inspection; prebuilt musl +tarballs and Docker images. + +**Maintenance.** Very young and dormant — all 10 PRs authored and merged +2026-10-14 → 2026-10-19 (v0.0.1–v0.0.9), then silent; solo maintainer; zero +user-filed issues (all tracker signal is the maintainer's own PR bodies). + +**Capabilities (code).** Framing-only forwarding that pools completed +streaming POSTs at proven message boundaries +(`research-repos/EeroEternal__xlb/src/proxy/l7.rs:1-13`, +`src/pool.rs:60-147`); phase-shaped timeout hierarchy — connect / +first_byte (TTFB) / idle (per-direction silence) / total (3600 s default) / +fin, cross-validated (`src/config.rs:73-111`); pre-first-byte status +rerouting with the request buffer dropped the moment response headers are +written (`src/proxy/l7.rs:117-163`, `src/proxy/http1.rs:846-923`); +non-idempotent retry boundary — POST/PUT/PATCH reroute only on unambiguous +pre-execution refusals, `respect_idempotency_key` restores the full list +(`src/config.rs:253-298`); one wall-clock retry budget spanning connect +retries, stale-socket failovers, and status reroutes (`src/config.rs:241-248`); +automatic L4 fallback via protocol sniffing (`src/proxy/l7.rs:61-106`); +active+passive health with hysteresis and authenticated probes +(`src/health.rs:58-100`); client-abort propagation during silent prefill +(`src/proxy/http1.rs:133-228`); proxy-added-latency TTFB-delta metric +(`src/metrics.rs:10-60`); validated LIFO pool with transparent stale-socket +redial incl. POSTs (`src/pool.rs:62-97`). + +**Pain points (from the maintainer's PR bodies — no user issues exist).** +Non-idempotent POSTs replayed on in-doubt 500/502/504 risk double billing +by default (https://github.com/EeroEternal/xlb/pull/10); retries had no +total ceiling until `retry.timeout` (https://github.com/EeroEternal/xlb/pull/10); +client aborts during silent prefill held upstream sockets for the full +timeout (https://github.com/EeroEternal/xlb/pull/6); client-sent +`Connection: close` poisoned backend pooling (https://github.com/EeroEternal/xlb/pull/7); +credential-like request ids landed in correlation logs until auto-redaction +(https://github.com/EeroEternal/xlb/pull/10). + +**Directions (pulls).** Timeout phases reshaped around LLM stream +lifecycles (https://github.com/EeroEternal/xlb/pull/2); per-backend +concurrency caps with bounded queueing (https://github.com/EeroEternal/xlb/pull/3); +data-plane circuit breaking via passive dial-failure ejection +(https://github.com/EeroEternal/xlb/pull/4); privacy-preserving multi-attempt +tracing (https://github.com/EeroEternal/xlb/pull/10); zero-dependency +distribution (https://github.com/EeroEternal/xlb/pull/5). + +**Gaps vs llm-proxy.** Timeout hierarchy (llm-proxy has one flat 300 s +per-route `TimeoutLayer` at `crates/llm-proxy-server/src/routes/mod.rs:64-67` +plus a single per-chunk idle watchdog applied identically to prefill wait +and reasoning pauses at `crates/llm-proxy-provider/src/transport.rs:71-81`); +idempotency-aware replay boundary; wall-clock retry ceiling; +active+passive health; per-backend concurrency caps; checkout-validated +connection pooling with POST replay; OS TCP keepalive on upstream sockets +(llm-proxy sets keepalive only on accepted client sockets); listener +admission control. + +### yolorouter/yolorouter + +**Identity · Language · Form factor.** Go 1.25 single-binary gateway +(gleam SQLite or PostgreSQL) with an embedded Vue 3 console; distinctive +fact→decision kernel — request-path logic is a decision table guarded by +AST-based structural tests. + +**Maintenance.** Fast and steady — ~11 releases 0.1.9 (Aug 2026) → 0.2.9 +(2026-10-27); development lands as direct commits; PRs are an +outside-contribution channel with a 0% merge rate so far. + +**Capabilities (code).** Key pool with per-provider rotation and per-key +benching — a plain 429 benches a key for its Retry-After window clamped to +[1 s, 10 min], benched keys demoted not excluded, soonest-to-recover first +(`research-repos/yolorouter__yolorouter/internal/gateway/keypool.go`); +three-state circuit breaker where admission/accounting travel as a +generation token so a pre-trip result cannot mis-book half-open +(`internal/gateway/circuit/circuit.go`); per-model scheduling modes with +per-caller-key sticky bindings that keep upstream prompt caches warm +(`internal/gateway/binding.go`); seven independent timeout phases (connect, +header, first-byte, body-idle, attempt, request, TLS) cross-validated +(`internal/config/config.go:42-79`); protocol-from-path ingress over five +surfaces incl. native Gemini (`internal/gateway/negotiate.go`); +framework-aware tool-output compression chains (`internal/compress/compress.go`); +vision fallback — images described by a vision model through a loopback +call, spliced back with surgical JSON patching +(`internal/capability/visionfallback/describe.go`); per-key access control +with model allowlist, RPM/TPM, concurrency, cumulative budget in micros, +and saturating token arithmetic (`internal/model/api_key.go`); learned +rate-limit evidence parsed from exactly three header dialects with a +corroboration-capped 24 h ceiling (`internal/gateway/rate_limit_evidence.go:1-27`); +config-time model-mapping verification against the real upstream +(`internal/service/modeladmin/probe_queue.go:15-27`); in-place self-update +with checksum and rollback (`internal/selfupdate/selfupdate.go`). + +**Pain points (issues).** Egress proxying was impossible (hardcoded nil +Proxy) — shipped in v0.2.8 as global + per-provider overrides +(https://github.com/yolorouter/yolorouter/issues/8); no model capability +metadata — operators learned a claimed 128K context was fake by eating the +upstream error (https://github.com/yolorouter/yolorouter/issues/9); +upstream keys were write-only with no reveal/backup path +(https://github.com/yolorouter/yolorouter/issues/10); bulk import forced +pricing decisions before capability data existed +(https://github.com/yolorouter/yolorouter/issues/13); no model-centric +pricing comparison or bulk adjust (https://github.com/yolorouter/yolorouter/issues/14). + +**Directions (pulls).** Windows one-click install reimplemented in-repo +after community PRs were declined (https://github.com/yolorouter/yolorouter/pull/1, /pull/2); +the 0.2.9 release is entirely pricing-surface work (per-model price +summaries, unpriced filter, transactional bulk adjust with dry-run — +`research-repos/yolorouter__yolorouter/CHANGELOG.md:27`). + +**Gaps vs llm-proxy.** Vision fallback; tool-output compression; +per-key system-prompt injection (`internal/capability/systemprompt/inject.go:14-40`); +seven-phase timeouts; learned per-key rate-limit evidence; Gemini client +ingress (`internal/router/router.go:611-617`); OAuth2/OIDC console SSO +(`internal/service/oauth/oauth_login_service.go:30-40`); config-time +mapping probes; agent-session reconstruction over the request log +(`internal/agentturns/backfill.go:1-13`); SSRF-safe egress with redirects +disabled (`internal/gateway/upstream.go:12-13`). + +### Chleba/ollamaMQ + +**Identity · Language · Form factor.** Rust single-crate binary (edition +2024, axum 0.8 + tokio, ~11k LoC, v0.5.1) with a built-in ratatui dashboard +— a queueing, load-balancing proxy in front of multiple Ollama, LM Studio, +and OpenAI-compatible backends; everything runs through one central +scheduler. + +**Maintenance.** Active — HEAD 2026-10-30; all four PRs in its history are +external, including a substantial Sept 2026 hardening pass; exactly one +user-filed issue ever. + +**Capabilities (code).** Per-user fair-share queuing — FIFO per X-User-ID, +turn order by a usage score halving every 5 minutes, charged at dispatch +regardless of outcome so failing clients keep no priority +(`research-repos/Chleba__ollamaMQ/src/dispatcher.rs:735-786`); +model-residency-aware backend tiers (prefer loaded, then cold, evicting +last) (`src/dispatcher.rs:1122-1137`); bounded aggregate queue memory — +512 MiB default across all queues with an always-admit-when-empty rule +(`src/dispatcher.rs:2337-2410`); aggregated multi-backend `/v1/models` +merging live listings (`src/dispatcher.rs:360-535`); concurrent health +probing with negative capability memory — endpoints a backend rejects are +skipped until re-probe (`src/dispatcher.rs:1218-1282`); model load/unload +control plane (`src/control.rs:855-1060`); fail-fast satisfiability +analysis for unservable requests (`src/dispatcher.rs:1771-1860`); metadata +reads never queue (`src/dispatcher.rs:565-570`); non-blocking JSONL request +log on a bounded channel (`src/reqlog.rs:20-104`). + +**Pain points (issues).** Docker deployment broken out of the box — the +entrypoint passed a flag the CLI didn't accept, and users were fully +blocked (https://github.com/Chleba/ollamaMQ/issues/3). + +**Directions (pulls).** The merged hardening pass: admission control, +fair-share redesign, mutex-poisoning recovery, concurrent probes +(https://github.com/Chleba/ollamaMQ/pull/4); an optional embedded web +dashboard with durable blocklists open (https://github.com/Chleba/ollamaMQ/pull/5). + +**Gaps vs llm-proxy.** Multi-user queueing with decayed fair-share; +interchangeable backend pools with residency awareness; aggregate in-flight +memory budgeting; fail-fast satisfiability; model lifecycle control; +aggregated cross-backend listings; negative capability memory; the +Ollama-native dialect on both sides (`src/main.rs:357-396`); per-model +concurrency caps keyed on normalized names (`src/config.rs:166-187`); +interactive live-ops surfaces (`src/tui.rs:455-640`). + +### ghuntley/preflight + +**Identity · Language · Form factor.** Rust single-crate loopback proxy +(edition 2024, axum 0.8, ~3.3k LoC) that inspects OpenAI-compatible requests +and attachments for secrets before forwarding to ghuntley/underclass; +chains a pair of proxies — preflight owns secret hygiene, underclass owns +routing. + +**Maintenance.** Young and automation-heavy but quiet — single squashed +commit era from 2026-09-20, last master commit that date; the entire +tracker is three bot PRs (Renovate declined, two Dependabot bumps open); +zero human-filed issues. + +**Capabilities (code).** Three-mode secret enforcement (redact / no-go / +advisory) where enforcing modes reject findings without a safe replacement +(`research-repos/ghuntley__preflight/src/main.rs:23-30,405-426`); +Gitleaks-derived rules embedded via `include_str!` behind an explicit +enable-profile with byte-safe regex rewrites (`src/scanner.rs:70-141`); +exact SHA-256 "carnet" allowlist — no regex allowlists, no path exclusions, +no inline suppression comments (`src/main.rs:149-163`); attachment +inspection in a Bubblewrap-sandboxed worker with rlimits and OCR, driving +Poppler/QPDF/ExifTool/ZBar/Tesseract through typed argv +(`src/attachments.rs:221-280`); verify-after-redact that fails closed if +any finding survives (`src/attachments.rs:139-194`); content-addressed +inspection cache keyed on credential scope + rule profile + toolchain +(`src/cache.rs:152-191`); SSRF-hardened resolver with DNS pinning and +refused redirects (`src/resolver.rs:61-104`); ingress-owned cross-proxy +request correlation (`src/correlation.rs:30-47`); SIGHUP reload that keeps +the previous runtime on failure and refuses restart-requiring changes +(`src/main.rs:120-127,296-305`); anti-smuggling JSON traversal rejecting +duplicate keys and reconstructing concatenated text across content +boundaries (`src/document.rs:11-73`). + +**Pain points (from dependency state — no user issues exist).** Security- +relevant bumps (rustls 0.23.45→.46) sit unmerged on a 45-day-stale master +(https://github.com/ghuntley/preflight/pull/3); the PDF inspection path +pins lopdf four minors behind the line adding decryption +(https://github.com/ghuntley/preflight/pull/2). + +**Directions (pulls).** Real updates bypass PRs: a weekly scheduled workflow +revalidates and commits a refreshed vendored Gitleaks database straight to +main (`.github/workflows/rules.yml:44-64`). + +**Gaps vs llm-proxy.** Request-path credential DLP; attachment OCR redaction +with pixel-coordinate mapping; sandboxed out-of-process processing of +untrusted content; SSRF-hardened fetching of client-referenced content; a +request-derived verdict cache; SIGHUP reload with restart-required +refusals; automated provenance-pinned refresh of a vendored data dependency +(`docs/adr/0006-direct-validated-rules-updates.md`); hardened NixOS deploy +with a QEMU VM test (`nixos/tests/preflight.nix`); differential +detection-parity testing against the reference implementation +(`tests/differential.rs:14-60`). + +### NoahStepheno/completion-to-response + +**Identity · Language · Form factor.** Go stdlib-only single binary (~1.1k +LoC, zero dependencies) serving an OpenAI Responses API endpoint and +translating to any Chat Completions upstream — the reverse direction of +most proxies, so Codex CLI and Claude Code can drive +Chat-Completions-only backends (Zhipu, DeepSeek, Ollama, vLLM, Azure). + +**Maintenance.** Terminal — one merged external PR (2026-05-17, also the +last push); archived read-only 2026-08-16; its single true issue (429 +retry) has zero replies. + +**Capabilities (code).** A stateful `StreamConverter` synthesizing the full +Responses streaming event ladder (created → in_progress → output_item → +content_part → deltas → done → completed) from chat deltas +(`research-repos/NoahStepheno__completion-to-response/internal/transform/transform.go:368-609`); +streaming tool-call arguments reassembled per tool-call index +(`internal/transform/transform.go:455-497`); consecutive `function_call` +items grouped into ONE assistant message with a `tool_calls` array, and +mid-conversation system/developer messages hoisted to the top so the +assistant(tool_calls)→tool sequence is never broken +(`internal/transform/transform.go:87-139,143-179`); DeepSeek +`reasoning_content` serialized WITHOUT `omitempty` so empty reasoning still +reaches the backend (`internal/types/completion.go:29`); `finish_reason` → +Responses status mapped identically on both paths +(`internal/transform/transform.go:250-302`); a companion Node gateway +adding model-name→backend routing (`codex-gateway.js:26-96`). + +**Pain points (issues).** A single upstream 429 kills the entire Codex +session — no retry, and the repo is archived so it will never be fixed +(https://github.com/NoahStepheno/completion-to-response/issues/2); +multi-turn thinking-mode and parallel-tool-call correctness all arrived in +the one merged PR (https://github.com/NoahStepheno/completion-to-response/pull/1). + +**Directions (pulls).** The single merged PR is the whole direction: +per-vendor extension fields must round-trip even when empty, grouping rules +for parallel tool calls, message-ordering repair, and a turnkey +Codex-on-DeepSeek kit (https://github.com/NoahStepheno/completion-to-response/pull/1). + +**Gaps vs llm-proxy.** Codex-proven tool-call grouping and ordering +semantics for a Responses client surface; Responses SSE event synthesis +(llm-proxy's client-facing event machine is Anthropic-shaped); +`text.format.json_schema` → `response_format` structured-output translation +(a grep over llm-proxy `crates/` for `response_format|json_schema` returns +no non-test hits); model-field front-door routing at a single base URL. + +### os-tack/ostk-cache + +**Identity · Language · Form factor.** Rust (edition 2024) loopback-only +"L1.5" context-anchoring caching proxy for Anthropic `/v1/messages` plus an +OpenAI `/v1/responses` adapter; three binaries (proxy, hooks installer, +stats reporter) over a tokio-free core crate; depends on private +os-tack/haystack crates via authenticated git-deps. + +**Maintenance.** Stale — v0.4.0 tagged 2026-06-12, last push that date +(~5 months before this wave); its only tracker entry ever is one +unmerged docs PR. + +**Capabilities (code).** Strips Claude Code's per-turn volatile `cch=;` +billing token from every string so byte-for-byte prefix caching actually +hits — reported cache_read 0 → 963K tokens/turn, ~$2.50 → ~$0.29 +(`research-repos/os-tack__ostk-cache/src/billing_strip.rs:1-57`); +progressive soft-cap reduction tiers (eject oversized tool_result bodies → +prune oldest tool pairs → drop unreferenced tool definitions → structured +413 payload) (`src/rebuild.rs:591-701`); overflow answered locally with +Anthropic's exact `prompt is too long: N tokens > M maximum` shape — +binary-verified to trigger Claude Code's reactive auto-compaction +(`src/usage_truth.rs:1-79`); an SSE usage rewriter scaling `input_tokens` +so the client's proactive auto-compact re-arms (`src/usage_truth.rs:103-160`); +WARM/DEAD cache-write lane state machine picking 5m vs 1h breakpoints from +observed wake cadence (`src/write_policy.rs:1-54`); config provenance — +every resolved value carries a Source and `--print-config` shows it +(`src/config.rs:86-118`); bounded async HTTP capture subsystem +(`crates/ostk-cache-core/src/capture.rs:104-178`); self-loop defenses — +own-address refusal plus an `x-ostk-cache-hop` header +(`crates/ostk-cache-core/src/capture.rs:29-54`); inline-content→file-handle +rewriting (`src/rewrite_middleware.rs:58-109`). + +**Pain points (issues/pulls).** The sole tracker entry — a member's +CONTEXT_PROJECTION.md with Mermaid diagrams — has sat unmerged ~6 months +(https://github.com/os-tack/ostk-cache/pull/1); bus factor one with private +git-deps blocking outside builds. + +**Directions (pulls).** Shipped direction is empirical validation of its +own premise: a `PASSTHROUGH=1` mode for A/B-measuring the amplification +ratio, with the docs openly questioning whether rewrites help +(`research-repos/os-tack__ostk-cache/docs/PASSTHROUGH.md`). + +**Gaps vs llm-proxy.** Cache-busting byte normalization (llm-proxy does no +request-body normalization); soft-cap reduction before the 32 MiB wall +(llm-proxy rejects flat at `crates/llm-proxy-server/src/routes/mod.rs:37`); +client-compaction steering via dialect-exact errors and usage rewrites; +stateful cache-write policy; agent-lifecycle hook ingestion +(`src/main.rs:173-260`); cache-effectiveness analytics (amp ratios, +hit-rate per session — llm-proxy stores raw cache columns at +`crates/llm-proxy-storage/src/sqlite.rs:118-119` but computes no ratios); +opt-in wire capture with redaction; self-loop prevention; config +provenance. + +### agiwhitelist/tokdiet + +**Identity · Language · Form factor.** TypeScript/Node (≥20, MIT, v0.1.0) +loopback streaming reverse proxy + SQLite store + SSE dashboard that meters +tokens/USD and actively compacts agent context for Anthropic, OpenAI, and +Gemini traffic; ships as an npm bin plus a Claude Code metering-hook +plugin. + +**Maintenance.** Dormant — created 2026-06-16, last push 2026-06-18 (a +3-day arc ending in launch marketing); zero PRs ever; its only issue is +unanswered. + +**Capabilities (code).** Recoverable context compaction — old tool results +above a floor replaced by head+salient-lines+tail markers with the full +block persisted under a content-addressed id +(`research-repos/agiwhitelist__tokdiet/src/compactor/elision.ts:36-77`); +page-fault auto-recovery — if the answer echoes a paged-out id or complains, +the proxy restores the blobs and silently re-sends once +(`src/pagefault.ts:23-80`); a shadow-eval quality guard — sampled compacted +requests re-run uncompacted, scored heuristically or by a cheap LLM judge, +rolling degradation trips safe-mode per strategy with persisted per-repo +priors (`src/quality/guard.ts:83-136`); cache-prefix-aware mutation — every +strategy refuses to rewrite past the last Anthropic cache_control +breakpoint, savings re-derived from authoritative recounts +(`src/providers.ts:318-369`); thinking/signature blocks excluded from all +mutation (`src/providers.ts:46-63`); fail-open passthrough as the cardinal +rule (`src/proxy.ts:452-492`); tiered session/day/repo-month budgets whose +breach action can be force-compaction rather than refusal +(`src/budget.ts:53-105`); honest proxy-overhead accounting +(`src/proxy.ts:593-614`). + +**Pain points (issues).** Models absent from pricing.json are silently +priced $0.00 with no flag — an unpriced month looks like a free month +(https://github.com/agiwhitelist/tokdiet/issues/1); longest-prefix fallback +prices next-generation ids at previous-generation rates — `claude-opus-4-5-*` +inherits the `claude-opus-4` key's rate (`src/pricing.ts:48-60`, per the +same report); the bundled catalog is frozen at the prior model generation +(the same issue: pricing.json dated 2026-06-16, models one generation +behind, refreshable only by hand). + +**Directions (commits — no PRs exist).** The core bet is "context as +virtual memory" with measured compaction quality: a 66-task A/B suite and +honest two-model results shipped alongside the features +(`https://github.com/agiwhitelist/tokdiet/commits/main`). + +**Gaps vs llm-proxy.** Request-path context compaction; shadow-eval +quality guard for any transformation llm-proxy performs; recoverable +elision; cache-prefix mutation safety; fail-open passthrough discipline; +Gemini client endpoint (`src/providers.ts:567-647`); session/repo/source +attribution on usage rows (`src/session.ts:20-61`); agent-side hooks; +budget-breach escalation to compaction. + +--- + +## Part II — Issue-tracker sweeps (40 repositories) + +Compact records: one identity line, then the strongest pain points and +directions with evidence. + +### 0xzr/freellmpool + +Python free-LLM aggregator proxy (chat/embeddings/transcription + MCP +toolbox + OpenCode plugins) pooling ~24 keyless providers behind multi-mode +routing. Routing evolved from naive `fast` to `fair` to latency-aware +`spread` after agentic loops caused 429 storms — each turn failing over +through ~25 providers (https://github.com/0xzr/freellmpool/pull/25); a +persistent, privacy-safe per-route/per-provider health store with circuit +breakers fed by Retry-After and rate-limit-reset headers, surviving +restarts (https://github.com/0xzr/freellmpool/pull/97); streaming hardening +— no fake `finish=stop` after mid-stream failure, per-chunk wall-clock +deadline against slow-drip upstreams (https://github.com/0xzr/freellmpool/pull/17); +omitting the OpenAI-contract `created` field broke Vercel AI SDK's date +parsing on every call (https://github.com/0xzr/freellmpool/issues/112); +free-provider catalogs churn — the repo's own sentinel auto-filed eight +drift issues until lifecycle-state detection and a single reusable sentinel +issue landed (https://github.com/0xzr/freellmpool/issues/141); an +async-client redirect bug let validated public URLs redirect requests and +credentials to internal targets (https://github.com/0xzr/freellmpool/issues/87). + +### 1mcp-app/agent + +TypeScript/Node MCP gateway aggregating stdio/SSE/HTTP servers behind one +client endpoint; 86 of 100 recent PRs merged. New Streamable HTTP sessions +timed out for hours while `/health` and `/health/ready` stayed green — +observational health that doesn't prove the server can accept sessions +(https://github.com/1mcp-app/agent/issues/670); a single backend exit +wedged the whole inbound stdio connection; the fix withdraws only the +failed backend's routes (https://github.com/1mcp-app/agent/issues/641); +upstream OAuth initiation failures around RFC 9728 `resource` matching and +incomplete issuer metadata (https://github.com/1mcp-app/agent/issues/653); +tool-result byte budgets with a recognizable `schema_budget_exceeded` error +and spill-to-file instead of hard failure (https://github.com/1mcp-app/agent/pull/726); +per-purpose capture budgets so an over-budget page is skipped without +silently dropping the whole capability list +(https://github.com/1mcp-app/agent/pull/645); a timeout taxonomy where an +execution timeout "never triggers fallback or replay" because the outcome +is uncertain (https://github.com/1mcp-app/agent/issues/709). + +### AgentOps-AI/agentops + +Python LLM-agent observability SDK + OTLP collector + dashboard; pushed to +fully open-source self-hosting in Sept 2025. Token-usage extraction is the +most-repeated defect — providers returning usage as a mapping get all-None +counts (https://github.com/AgentOps-AI/agentops/issues/1432); legacy +`prompt_tokens/completion_tokens` attribute names vs the new +`input_tokens/output_tokens` family silently undercount whichever a +collector sums (https://github.com/AgentOps-AI/agentops/issues/1447); +Anthropic paths drop cache_read/cache_creation entirely so cached sessions +under-report (https://github.com/AgentOps-AI/agentops/issues/1445); cost is +a bare float with no currency — the OTel GenAI proposal adds +amount/currency/source (https://github.com/AgentOps-AI/agentops/issues/1462); +model-price catalogs rot three ways at once +(https://github.com/AgentOps-AI/agentops/issues/1460); stream finalization +overwrites iteration errors to OK so failed streams export as successful +(https://github.com/AgentOps-AI/agentops/issues/1434); a hardcoded fallback +signing secret allowed JWT forgery (https://github.com/AgentOps-AI/agentops/issues/1446); +cost computed at query time from a versioned dictionary in ClickHouse UDFs +(https://github.com/AgentOps-AI/agentops/pull/1249). + +### AmoyLab/Unla + +Go + React self-hosted MCP gateway wrapping MCP servers and OpenAPI/REST +APIs. `initialize` hardcodes protocolVersion 2024-11-05 and never reads the +client's requested version (https://github.com/AmoyLab/Unla/issues/341); +compressed upstream bodies read as text garbled payloads for seven months +until a decode-and-normalize pass landed +(https://github.com/AmoyLab/Unla/issues/234, +https://github.com/AmoyLab/Unla/pull/302); SSRF hardening with DNS pinning +and per-redirect revalidation — and the matching escape hatch after blanket +internal bans broke in-cluster deployments (https://github.com/AmoyLab/Unla/pull/305, +https://github.com/AmoyLab/Unla/issues/299); OTel tracing with optional +body capture (https://github.com/AmoyLab/Unla/pull/252) and a configurable +Prometheus endpoint (https://github.com/AmoyLab/Unla/pull/289); MCP 2026-07-28 +stateless revision support with `_meta` negotiation +(https://github.com/AmoyLab/Unla/pull/324); a gin middleware-order bug +silently stripped admin checks from tenant routes — registration order +matters (https://github.com/AmoyLab/Unla/issues/339). + +### ENTERPILOT/GoModel + +Go self-hostable multi-protocol gateway with virtual models, a dashboard, +MCP serving, and a licensed "Pro" billing tier; 76 of its last 100 PRs +merged within twelve days. Virtual-model `params`/`default_params` presets +(override + fill-if-omitted) answered a LiteLLM-migrant's ask +(https://github.com/ENTERPILOT/GoModel/issues/1333, +https://github.com/ENTERPILOT/GoModel/pull/1353); updating a provider's +credentials unregistered it and wiped the discovered inventory across every +replica within ~2 s — the fix is an atomic in-place adapter swap inheriting +the old inventory (https://github.com/ENTERPILOT/GoModel/issues/1308, +https://github.com/ENTERPILOT/GoModel/pull/1338); per-request steering +headers behind an allowlist because per-client control is an abuse vector +(https://github.com/ENTERPILOT/GoModel/issues/1347); native TLS/mTLS with +5 s-stat cert reload for cert-manager symlink swaps +(https://github.com/ENTERPILOT/GoModel/issues/1285, +https://github.com/ENTERPILOT/GoModel/pull/1326); DB-only cluster sync of +admin writes with per-replica writer ids +(https://github.com/ENTERPILOT/GoModel/pull/1295, +https://github.com/ENTERPILOT/GoModel/pull/1296); reseller billing — +per-customer markups, invoice export, `billable: false` BYOK skip +(https://github.com/ENTERPILOT/GoModel/pull/1291, +https://github.com/ENTERPILOT/GoModel/pull/1293); workload identity +federation exchanging platform tokens instead of static keys +(https://github.com/ENTERPILOT/GoModel/pull/1276), plus auth before body +decompression (https://github.com/ENTERPILOT/GoModel/pull/1259). + +### EinStack/glide + +Go open-source LLM gateway; development stalled after v0.1.0-rc.1 (June +2024) with roadmap epics still open. Its merged core was request-path +resilience — token-bucket health/rate tracking with priority fallback +(https://github.com/EinStack/glide/pull/68), least-latency routing +normalized per output token (https://github.com/EinStack/glide/pull/111), +permanent unavailability on unauthorized keys +(https://github.com/EinStack/glide/pull/180); streaming error semantics — +typed error codes with finish_reason on the error schema +(https://github.com/EinStack/glide/pull/203); a whole OTel subsystem built +and defaulted off (https://github.com/EinStack/glide/pull/237, +https://github.com/EinStack/glide/pull/254); unshipped demands: BYO keys +with validation (https://github.com/EinStack/glide/issues/257), authn/authz +epics (https://github.com/EinStack/glide/issues/232), message-logging +opt-out for compliance (https://github.com/EinStack/glide/issues/261), and +the still-open semantic-caching epic +(https://github.com/EinStack/glide/issues/230); a released image panicked +on a missing external spec file until the spec was embedded +(https://github.com/EinStack/glide/issues/209, +https://github.com/EinStack/glide/pull/211). + +### Inebrio/Routerly + +TypeScript/Fastify gateway + React dashboard centered on semantic-intent +routing, OAuth subscription providers, and transparent pass-through. +Usage persisted by whole-file JSON rewrite per request grew unbounded until +the Docker VM was OOM-killed — the acknowledged fix is append-only NDJSON +(https://github.com/Inebrio/Routerly/issues/124); missing cache prices +defaulted cached reads to the full input rate, overstating spend and +tripping budgets early (https://github.com/Inebrio/Routerly/issues/144); +operators want in-flight requests visible in the log — routing decisions +appear only after completion (https://github.com/Inebrio/Routerly/issues/148); +horizontal scaling impossible with in-memory routing state and file-based +config writes (https://github.com/Inebrio/Routerly/issues/86); transparent +pass-through forwards any unhandled path to the upstream with only the +credential swapped — a drop-in base-URL replacement for endpoints the proxy +never modeled (https://github.com/Inebrio/Routerly/pull/106); OAuth +subscription providers (Claude Pro/Max, ChatGPT/Codex) forwarded verbatim +(https://github.com/Inebrio/Routerly/pull/107, later hardened by +https://github.com/Inebrio/Routerly/pull/136); semantic intent routing with +per-intent examples — and the immediate follow-on demand for decision +explainability (https://github.com/Inebrio/Routerly/pull/49, +https://github.com/Inebrio/Routerly/issues/113). + +### LMCache/LMCache + +KV/prefix-cache layer for vLLM/SGLang over ZMQ or gRPC with tiered storage +and fleet coordination; mid-extraction into separate client/server +distributions. Cache identity omitted the model revision so two servers +running one model ID at different `--revision` values shared cache entries +computed by older weights (https://github.com/LMCache/LMCache/issues/6253); +ambient `http_proxy` env vars silently hijacked localhost gRPC for the full +300 s timeout — explicit per-endpoint proxy config preferred +(https://github.com/LMCache/LMCache/issues/6277); one blocked shared-socket +client froze every client in the process until NOBLOCK sends landed +(https://github.com/LMCache/LMCache/issues/6255); the server survived +SIGTERM and ran 45–90 s after failed startup until lifecycle CI +(kill/restart/same-port) was added (https://github.com/LMCache/LMCache/issues/6192); +internal clients built without credentials failed 401 every cycle and +silently degraded two subsystems (https://github.com/LMCache/LMCache/pull/6247); +request-scoped cache policy landing as a tagged identity with +operator-gated defaults (https://github.com/LMCache/LMCache/pull/6199). + +### LostRuins/koboldcpp + +llama.cpp-derived single-binary local inference server with OpenAI/Anthropic +APIs and a vendored stable-diffusion fork. Generation errors returned as +HTTP 200 with `finish_reason:"error"` drove unbounded agent retry storms, +and custom stop-string hits reported as `length` instead of `stop` +(https://github.com/LostRuins/koboldcpp/issues/2481); streamed tool requests +took a fake-stream path emitting no final usage chunk, so agentic clients +saw 0 tokens and auto-compaction never fired +(https://github.com/LostRuins/koboldcpp/issues/2480); Anthropic conformance +closed one field at a time — `message_start` content/stop_reason (without +which the official SDK's `get_final_message()` crashed), stop_sequences, +tool_choice (https://github.com/LostRuins/koboldcpp/pull/2567, /pull/2568, +/pull/2571); a non-function entry in the tools array crashed the handler +(https://github.com/LostRuins/koboldcpp/issues/2516); MCP tool-result images +were rendered into the prompt as ~120k tokens of text on one path +(https://github.com/LostRuins/koboldcpp/pull/2376); a malformed tool call +spun the Python layer at 100% CPU with no abort +(https://github.com/LostRuins/koboldcpp/issues/2500); disconnect-driven +cancellation and a progress endpoint for long generation jobs +(https://github.com/LostRuins/koboldcpp/pull/2318, /pull/2316). + +### MartialBE/one-hub + +Go + React gateway (the fork done-hub derives from); maintenance stalled +after 2026-01-25, with an open "is this still updating?" issue. Users +demanded failover retry on ANY non-success status and error-rate-driven +weight downgrades (https://github.com/MartialBE/one-hub/issues/852, +/issues/921); a client-facing `/v1/responses` is needed because the hub +only converts upstream Responses→Chat — and Responses SSE clients +hard-require wire-exact usage fields, a zero-value `cached_tokens` omitted +via `omitempty` broke Codex (https://github.com/MartialBE/one-hub/issues/868, +https://github.com/MartialBE/one-hub/pull/910); OpenAI→Gemini tool-calling +translation is a live bug farm — `strict` must be stripped, `functionCall` +parts must map back to `tool_calls` (https://github.com/MartialBE/one-hub/issues/918); +Claude streaming lost cache-token usage fields +(https://github.com/MartialBE/one-hub/issues/884); resale operators asked +for per-API-key rate limits because group-level limits throttle every user +sharing the group (https://github.com/MartialBE/one-hub/issues/835); +merged: a GroupManager +primary/backup failover ladder (https://github.com/MartialBE/one-hub/pull/823), +per-token IP whitelists (https://github.com/MartialBE/one-hub/pull/824) and +billing-tag metadata (https://github.com/MartialBE/one-hub/pull/891); a +per-channel request-mutation pipeline is a durable theme — `extra_body` +passthrough, custom-param modes, an open `__delete__` sentinel +(https://github.com/MartialBE/one-hub/pull/900, /pull/873, +/issues/927). + +### NVIDIA-NeMo/Guardrails + +NVIDIA's Python guardrails toolkit pivoting into infrastructure: an +experimental typed proxy launched from YAML manifests with +validate/inspect/schema/serve commands (https://github.com/NVIDIA-NeMo/Guardrails/pull/2468); +LiteLLM Chat and Responses dialects preserving gateway-added extension +fields and encrypted reasoning items (https://github.com/NVIDIA-NeMo/Guardrails/pull/2529); +an HTTP hooks sidecar exposing `/v1/hooks/claude-code` and `/v1/hooks/codex` +PreToolUse endpoints plus a Rust Relay plugin on the same toolchain as +llm-proxy (https://github.com/NVIDIA-NeMo/Guardrails/pull/2473, /pull/2489); +self-check rails sent `temperature`/`max_tokens` that reasoning models +reject — per-model-family parameter normalization must apply to every +internal caller (https://github.com/NVIDIA-NeMo/Guardrails/issues/2539); +per-tool rails need rewrite semantics, not just allow/block +(https://github.com/NVIDIA-NeMo/Guardrails/issues/2497); request-scoped +correlation IDs don't reach rails (https://github.com/NVIDIA-NeMo/Guardrails/issues/2495); +PII masking leaked raw values on overlapping spans and needs stable +numbered placeholders (https://github.com/NVIDIA-NeMo/Guardrails/issues/2469, +/issues/2546); built-in rails failed open on `text=None` while OTEL spans +logged untransformed messages (https://github.com/NVIDIA-NeMo/Guardrails/issues/2478). + +### Nya-Foundation/NyaProxy + +Python gateway with Vue dashboard and embedded config UI pooling provider +keys. A CVSS-9.8 report: with the default empty api_key the web config UI +and dashboard were fully unauthenticated, enabling endpoint rewrites for +SSRF, with redirect-following to private IPs and credentialed wildcard CORS +(https://github.com/Nya-Foundation/NyaProxy/pull/100); multi-window +per-key rate limits — every window enforced independently, key usable only +while all have room (https://github.com/Nya-Foundation/NyaProxy/pull/133); +dashboard write paths with schema validation, rollback, and a control flag +(https://github.com/Nya-Foundation/NyaProxy/pull/134); a held credential +lock was never released when stream teardown raised — finalizers must run +in `finally` with per-finalizer isolation (https://github.com/Nya-Foundation/NyaProxy/pull/117); +locks without expiry deadlocked an entire API for 30 minutes until +self-expiring deadlines landed (https://github.com/Nya-Foundation/NyaProxy/pull/121); +wheel and container smoke tests so a broken ENTRYPOINT can never ship +(https://github.com/Nya-Foundation/NyaProxy/pull/111). + +### RouteWorks/RouterArena + +Benchmark/leaderboard for LLM routers grading per-query model picks on +accuracy and cost. Two of the top four entries send >98% of queries to one +model — the audience demands routing reported as a per-query distribution +with single-model submissions labeled baselines +(https://github.com/RouteWorks/RouterArena/issues/272); models absent from +the price catalog were silently dropped before evaluation (0 of 5,273 +evaluated) and new model slugs blocked submissions +(https://github.com/RouteWorks/RouterArena/issues/193, /issues/206); the +fix rebuilt pricing as a governed subsystem — alias spellings collapsed to +profiles, exact matching, unpriced charged the table maximum instead of $0, +and an offline re-pricing tool that re-costs stored results without +re-grading (https://github.com/RouteWorks/RouterArena/pull/218); a router's +own decision-call usage became a separately charged `router_cost` computed +by the same code as grading (https://github.com/RouteWorks/RouterArena/pull/251); +a wall-clock-only sandbox let one allocation loop OOM the whole runner +until RLIMIT_AS caps landed (https://github.com/RouteWorks/RouterArena/issues/238, +/pull/239); grading now runs base-branch code with write tokens away from +submission code (https://github.com/RouteWorks/RouterArena/pull/254); +stored scores are versioned artifacts of the evaluator and get re-graded +retroactively (https://github.com/RouteWorks/RouterArena/pull/244). + +### StackOneHQ/defender + +TypeScript prompt-injection defense library (`defendToolResult()`) with a +three-tier pipeline and a Python port held to shared fixtures. Default +config was exploitable via `__proto__` mutation and pre-formed fake +`[UD-TRUSTED]` boundary tags (https://github.com/StackOneHQ/defender/issues/46); +tier 3 became a bring-your-own-LLM judge where unusable replies BLOCK +because an attacker can cause them, with fail-closed selectable +(https://github.com/StackOneHQ/defender/pull/106); thresholds are governed — +deciding on P(block) instead of argmax after a corpus change silently moved +recall 96%→61% at a fixed cut (https://github.com/StackOneHQ/defender/pull/74); +two silent-coverage bypasses fixed: deterministic oversize sampling let an +attacker precompute the dropped slot (fixed with boundary-aware chunking), +and index-free drop tracking stripped a field from every array element +(https://github.com/StackOneHQ/defender/pull/96, /pull/118); flattening +structured data to bare values turned `permissionLevel: "create"` into a +block at p≈1.00 until record-oriented review input landed +(https://github.com/StackOneHQ/defender/pull/94); cross-implementation +behavior pinned by public conformance fixtures with mutation-testing +verification (https://github.com/StackOneHQ/defender/pull/108, /pull/110). + +### abetlen/llama-cpp-python + +Official Python bindings + OpenAI-compatible server for llama.cpp, on rapid +automated upstream tracking. macOS Metal wheels above 0.3.30 corrupt when +two runners build the same wheel filename into one dist directory +(https://github.com/abetlen/llama-cpp-python/issues/2352); a `**kwargs` sink +silently swallows documented flags (https://github.com/abetlen/llama-cpp-python/pull/2364); +`max_completion_tokens` requests pass `max_tokens=None`, silently dropping +the limit (https://github.com/abetlen/llama-cpp-python/pull/2386); a shared +instance is not thread-safe and aborts the process uncatchably — motivating +out-of-process local runtimes behind a proxy +(https://github.com/abetlen/llama-cpp-python/issues/2392); native-resource +teardown on reload fixed after a null-pointer crash on second load +(https://github.com/abetlen/llama-cpp-python/issues/2342); the disk state +cache ignores its own capacity knob — knobs must demonstrably drive +eviction (https://github.com/abetlen/llama-cpp-python/pull/2400). + +### aws-samples/bedrock-access-gateway + +AWS's Python/FastAPI OpenAI-compatible proxy in front of Bedrock Converse — +officially deprecated 2026-09-04 because Bedrock now natively serves +Anthropic Messages and OpenAI-compatible Chat/Responses endpoints, so a +Converse adapter is largely replaced by pointing existing OpenAI/Anthropic +adapters at the new surfaces (https://github.com/aws-samples/bedrock-access-gateway/pull/264); +per-generation thinking-API churn — Opus 4.7 requires +`thinking.type=adaptive` + `output_config.effort`, with community fix open +since May (https://github.com/aws-samples/bedrock-access-gateway/issues/248); +per-model sampling-parameter denylists grown release over release +(https://github.com/aws-samples/bedrock-access-gateway/pull/238); +tool-use content dropped when truncation coincides (issue #197); +OpenAI-SDK `cache_control` hints never become Bedrock `cachePoint` blocks +(issue #229); catalog correctness — `/v1/models` doubling entries where +only the inference-profile-routed ID is invocable (issue #256); SSRF fix +for server-side fetching of caller media URLs at deprecation time +(https://github.com/aws-samples/bedrock-access-gateway/pull/262). + +### blue-pen5805/llm-proxy-on-cloudflare-workers + +TypeScript proxy on Cloudflare Workers with per-provider key rotation and +AI Gateway integration. Caller-side proxy auth headers were forwarded +upstream and could override provider credentials because caller headers +spread after provider headers — fixed by stripping and making provider +credentials non-overridable (https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/issues/350); +discovery rotates provider keys only on 429, shares one 60 s timeout +across attempts, and logs named credential profiles +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/pull/349); +provider adapter headers are rebuilt per fallback slot so a later key +cannot keep the first slot's auth scheme +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/pull/346); +`Math.random()` key selection replaced with rejection-sampled crypto +randomness after a code-scanning alert +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/pull/315); +selective URL-logging redaction preserving non-sensitive params +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/pull/305); +continuous JS dependency-vulnerability pressure as a standing ops cost +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/issues/344). + +### cita-777/metapi + +TypeScript relay-site aggregator with pooled OAuth accounts, weighted +routing, and site probing; bursty maintenance. Gemini upstreams reject +replayed OpenAI-style tool-call history without `thought_signature` — fixed +on the OpenAI-compat path after an earlier native-path fix +(https://github.com/cita-777/metapi/issues/580); multi-turn `/v1/responses` +proxying breaks on reasoning items with `encrypted_content` but no +`content` (https://github.com/cita-777/metapi/issues/538); credential-health +classification marking keys expired on any 401 or "token"+"invalid" +substring silently drops healthy keys (https://github.com/cita-777/metapi/issues/582); +`ADMIN_IP_ALLOWLIST` reads client IP from `X-Forwarded-For` with no +trusted-proxy check — spoofable (https://github.com/cita-777/metapi/issues/631); +cost reconciliation by latency windows pollutes channel stats under +retries — match on request identity, not latency +(https://github.com/cita-777/metapi/issues/566); per-site model probing +with auto-disable past a latency threshold +(https://github.com/cita-777/metapi/pull/510); proactive OAuth/session +refresh before expiry (https://github.com/cita-777/metapi/pull/430); +per-site concurrency caps, explicit priorities, and channel isolation +during retries landed in one community PR +(https://github.com/cita-777/metapi/pull/609). + +### cuihuan/awesome-ai-gateway + +Curated bilingual gateway list with an evidence methodology — canary checks +against first-party references, dated data snapshots, a CI freshness gate. +Curated data rots faster than re-verification: the freshness gate failed +with seven datasets over the 30-day maximum +(https://github.com/cuihuan/awesome-ai-gateway/issues/146); relay commercial +terms (minimum top-up, credit expiry) became a dataset after user demand +(https://github.com/cuihuan/awesome-ai-gateway/issues/88); the canary +script detects silent model substitution via tokenizer-fingerprint skew and +downgrade flags (https://github.com/cuihuan/awesome-ai-gateway/issues/133); +two independent October submissions are pure failover companions — weighted +round-robin + cooldown + `/health` (https://github.com/cuihuan/awesome-ai-gateway/pull/156); +agentgateway CEL policies failing open on OR'ed matchExpressions +(https://github.com/cuihuan/awesome-ai-gateway/pull/110); routing +explainability (an `x-or-route-reason` header on every response) and +savings dashboards as emerging differentiators +(https://github.com/cuihuan/awesome-ai-gateway/pull/170); a reproducible +translation-overhead benchmark — Bifrost 0.63 ms vs LiteLLM 5.34 ms at c=1 +(https://github.com/cuihuan/awesome-ai-gateway/pull/96); a key-liveness +monitor because balance pages stay green while keys are disabled upstream +(https://github.com/cuihuan/awesome-ai-gateway/pull/101). + +### data-privacy-stack/presidio + +Python PII detection/anonymization framework. Pattern detection is +systematically bypassable by Unicode variants — CJK word boundaries, +no-break-space-grouped card numbers, Arabic-Indic digits +(https://github.com/data-privacy-stack/presidio/issues/2421, /issues/2438, /issues/2377); +the fix pattern is an opt-in engine-level NFKC normalization pass rather +than touching 44 recognizer patterns +(https://github.com/data-privacy-stack/presidio/pull/2397); quadratic span +processing in the redaction hot path fixed as one sorted sweep +(https://github.com/data-privacy-stack/presidio/issues/2370); the HF NER +recognizer fails silent/open — a guardrail must fail closed +(https://github.com/data-privacy-stack/presidio/issues/2441); +REMOVE_INTERSECTIONS conflict resolution left flagged text in the clear +until results were sorted by (start, end) +(https://github.com/data-privacy-stack/presidio/pull/2331); analyzer pods +OOM on large inputs — bounded processing is config, not luck +(https://github.com/data-privacy-stack/presidio/issues/2446); a privacy +tool making outbound calls breaks air-gapped enterprises +(https://github.com/data-privacy-stack/presidio/issues/2447). + +### eclipse-lmos/lmos-router + +Kotlin/Spring routing library of the Eclipse LMOS agent platform (LLM, +embedding, and MVEL rule routers). Multi-provider support was the first ask +(https://github.com/eclipse-lmos/lmos-router/issues/30); LLM-authored +routing output breaking a strict schema resolved queries to no agent — +treat model decisions as untrusted input with tolerant parsing and a +deterministic fallback (https://github.com/eclipse-lmos/lmos-router/issues/45); +an optional routing flavor crashed at runtime because optional backends +weren't classpath-isolated (https://github.com/eclipse-lmos/lmos-router/issues/42); +routing runs on embeddings against agent specs in Qdrant +(https://github.com/eclipse-lmos/lmos-router/pull/254); a deterministic MVEL +rule engine coexists with statistical routing +(https://github.com/eclipse-lmos/lmos-router/pull/303); monthly merged +azure-identity bumps indicate sustained Entra credential support +(https://github.com/eclipse-lmos/lmos-router/pull/307); the 2026 tracker is +99% dependabot — no visible feature work +(https://github.com/eclipse-lmos/lmos-router/pull/211). + +### ericc-ch/copilot-api + +TypeScript/Bun proxy re-exposing GitHub Copilot subscriptions as +OpenAI/Anthropic endpoints for agent CLIs; stalled with a growing pile of +unmerged community fixes. Copilot now tags models with +`supported_endpoints`, and Responses-only models hard-fail on +`/chat/completions` — per-model endpoint-capability routing requested +(https://github.com/ericc-ch/copilot-api/issues/232); reasoning workloads +die at a ~60 s upstream idle timeout before the first token — retry, +keepalives, and abort-on-disconnect requested +(https://github.com/ericc-ch/copilot-api/issues/223; open PR +https://github.com/ericc-ch/copilot-api/pull/278 aborts the upstream fetch +when the client disconnects, to stop burning quota); client beta fields +outpace the upstream and `max_tokens` is rejected on newer models +(https://github.com/ericc-ch/copilot-api/issues/217); users cannot see or +control Copilot "premium interaction" quota burn +(https://github.com/ericc-ch/copilot-api/issues/201); multi-account token +pooling is a top ask (https://github.com/ericc-ch/copilot-api/issues/255); +hiding the real context window in `/v1/models` made Claude Code over-send +and compact too late (https://github.com/ericc-ch/copilot-api/issues/210); +usage passthrough so ccusage-class tools work +(https://github.com/ericc-ch/copilot-api/pull/86). + +### ferro-labs/ai-gateway + +Go LLM gateway ("ferrogw") with strategy routing across eight surfaces, a +plugin guardrail framework, an MCP tool loop, an embedded dashboard, and +weekly minor releases. The built-in response cache is in-memory only, so +multi-replica deployments get cold misses per pod +(https://github.com/ferro-labs/ai-gateway/issues/399); a crashed stdio MCP +server is detected but never respawned (blocked on the official Go MCP SDK) +(https://github.com/ferro-labs/ai-gateway/issues/361); MCP tool-call error +classification requested as a NEW Prometheus series rather than a relabel, +because adding a label to a live metric resets it +(https://github.com/ferro-labs/ai-gateway/issues/362); the dashboard keeps +its admin token in sessionStorage — HttpOnly cookie + CSRF requested +(https://github.com/ferro-labs/ai-gateway/issues/427); cost keyed on a +routing alias recorded $0 until canonical identity was resolved at provider +selection and carried to both cost paths +(https://github.com/ferro-labs/ai-gateway/issues/396); guardrail config +mistakes loaded as silent no-ops until validation began failing the load +with the offending key named (https://github.com/ferro-labs/ai-gateway/issues/447); +one strategy ranker with per-attempt timeouts, 429 cooldowns honoring +Retry-After, typed context-length failover, and pool modes that advance +only on failover-safe failures (https://github.com/ferro-labs/ai-gateway/pull/425); +a guardrail suite with one observable lifecycle event per match that never +carries the matched text (https://github.com/ferro-labs/ai-gateway/pull/439). + +### guardrails-ai/guardrails + +Python LLM-output validation framework (validators, structured-output +schemas, reask loops, OTel, a server mode). Truthiness coercion silently +corrupts structured output — legitimate 0/False/""/[] dropped, five fix PRs +unmerged (https://github.com/guardrails-ai/guardrails/issues/1603); two +parallel validator runtimes (sync vs async) return different outputs with +no warning (https://github.com/guardrails-ai/guardrails/issues/1633); +streaming-validation fidelity is a three-cluster pain — per-chunk instead +of per-line parsing, a literal validationPassed per chunk, and wire fields +hand-copied on the server path +(https://github.com/guardrails-ai/guardrails/pull/1617, /pull/1667, /pull/1606); +failure-policy defaults disagree per surface and degrade permissively — +one policy default plus hard errors on unknown names is the fix shape +(https://github.com/guardrails-ai/guardrails/issues/1656); OTel attributes +that are Python reprs break backends and leak prompts +(https://github.com/guardrails-ai/guardrails/issues/1631); SQLite trace +writes ran a retention DELETE per write until throttled +(https://github.com/guardrails-ai/guardrails/pull/1628); the +validation-result contract moved to a separately versioned types package — +the same bet llm-proxy makes with its protocol crate +(https://github.com/guardrails-ai/guardrails/pull/1571). + +### inference-gateway/inference-gateway + +Very active Go gateway multiplexing chat/responses/image/TTS/decisions plus +MCP client+server and A2A across dozens of providers, with OPA guardrails, +OIDC/mTLS, OTel, and a Kubernetes operator. The gateway itself is an MCP +server on `POST /mcp` speaking the stateless 2026-07-28 revision, tools +namespaced and run through the existing middleware +(https://github.com/inference-gateway/inference-gateway/pull/709); A2A as a +pure stateless relay with trace context propagated +(https://github.com/inference-gateway/inference-gateway/pull/756); new +OpenAI endpoint families shipped in days as byte-for-byte relays behind env +flags (https://github.com/inference-gateway/inference-gateway/pull/827); +mutual TLS in both directions, with a self-loop short-circuit for calls that +return through the gateway's own listener +(https://github.com/inference-gateway/inference-gateway/issues/822); +deprecated models hidden from `/v1/models` by syncing models.dev status +(https://github.com/inference-gateway/inference-gateway/issues/793); the +provider's Retry-After forwarded on upstream errors so clients learn the +real backoff signal (https://github.com/inference-gateway/inference-gateway/pull/752); +config/doc drift was the most frequent issue class (~12 of 100) +(https://github.com/inference-gateway/inference-gateway/issues/735). + +### janhq/jan + +Tauri/Rust + TypeScript local-first desktop assistant with an OpenAI-compatible +Local API and an agent runtime. The Anthropic→OpenAI converter mapped +context-window truncation to `finish_reason: "stop"` — truncation reported +as completion (https://github.com/janhq/jan/pull/9515); loopback Ollama +requests 403'd after Local-API hardening (https://github.com/janhq/jan/issues/9585); +MCP tool results with native image content pass through but are never +rendered (https://github.com/janhq/jan/issues/9547); security hardening +refuses non-loopback binds without an API key and stops `*` trusted-hosts +from accepting every Host header — DNS-rebinding defense +(https://github.com/janhq/jan/pull/9505, /pull/9507); tool dispatch became +consent-scoped with revalidated identities and a consent mirror that can +only narrow (https://github.com/janhq/jan/pull/9548); agent memory +snapshotted once per session so the system-prompt prefix stays +byte-identical — a changed prompt had been appending a ~10K-token copy per +run and destroying the provider cache +(https://github.com/janhq/jan/pull/9497). + +### kvcache-ai/Mooncake + +KVCache-centric disaggregated-serving data plane behind vLLM/SGLang. Async +transfer completion is poll-only although the engine already knows — an +eventfd-first completion RFC (https://github.com/kvcache-ai/Mooncake/issues/5164); +free-form "session" strings let a stale writer corrupt state — fenced +engine-owned sessions with abort propagation proposed +(https://github.com/kvcache-ai/Mooncake/issues/5182); under RelWithDebInfo, +struct_pack silently drops compatible fields between client and Master — a +release-only wire-compatibility drop (https://github.com/kvcache-ai/Mooncake/issues/5230); +lease expiry enforced inconsistently across replica types +(https://github.com/kvcache-ai/Mooncake/issues/5196); one week of merged PRs +fixed boolean env vars silently ignored, one env name meaning opposite +things in two transports, and a zero-thread RPC pool that hung every pooled +call while health checks stayed green +(https://github.com/kvcache-ai/Mooncake/pull/5125, /pull/5127, /pull/5189). + +### lidge-jun/opencodex + +TypeScript/Bun LLM proxy + Tauri dashboard fronting Codex and Claude Code +clients onto dozens of providers with account pools and quota metering; +releases 2.96.0→2.98.0 within four days. A pre-output 503 +`server_is_overloaded` was returned to the client instead of hopping to the +next target — a decline before any output must be classified safe-to-replay +(https://github.com/lidge-jun/opencodex/issues/8107); upstreams that refuse +oversized input without the `context_length_exceeded` code got a generic +retryable 502, so Claude Code retried for minutes instead of compacting — +error classification now reads message text, not provider codes alone +(https://github.com/lidge-jun/opencodex/pull/8128); all client-facing +errors rewritten to proxy-owned constants that may name the provider id but +never provider text (https://github.com/lidge-jun/opencodex/pull/8108); +an Ollama-compatible inbound API reusing the existing admission chain +rather than new code (https://github.com/lidge-jun/opencodex/pull/8099); +Ollama's `prompt_eval_cached_count` mapped so cache-read stops reporting 0 +(https://github.com/lidge-jun/opencodex/pull/8124); discovery timeouts +sized per source, single-flighted, with an `ok: true, discovering: true` +state distinct from failure (https://github.com/lidge-jun/opencodex/pull/8125); +headroom-aware account placement and HMAC-keyed durable conversation +affinity that discards all rows after an unclean shutdown +(https://github.com/lidge-jun/opencodex/pull/8092, /pull/8098); two open +classification bugs: an explicit body-stall 504 re-classified as 502, and +consumer backpressure recorded as upstream inactivity +(https://github.com/lidge-jun/opencodex/issues/8157, /issues/8159). + +### meta-llama/PurpleLlama + +Meta's umbrella for LLM cybersecurity tooling (CyberSecEval, CodeShield, +LlamaFirewall). A benchmark judging empty replies and "FAIL TO QUERY" +sentinels as accept gave a failing model the best refusal rate — bucket +transport failures separately from real responses +(https://github.com/meta-llama/PurpleLlama/issues/267); judge output valid +but off-schema crashes the run — schema validation with a neutral fallback, +not just syntax handling (https://github.com/meta-llama/PurpleLlama/pull/252); +frozen embeddings separate OOD injections at AUC≈0.999 while the native +head recalls ~23% — calibration is the lever, not the encoder +(https://github.com/meta-llama/PurpleLlama/issues/257); LlamaFirewall's +async wrapper hardcoded ALLOW/score-0 and discarded every scanner's real +verdict — sync and async paths must share one aggregation +(https://github.com/meta-llama/PurpleLlama/pull/185); the harness smuggles +protocol selection through model-name suffixes because the client is +hardwired to Chat Completions — per-model protocol choice is the expressed +need (https://github.com/meta-llama/PurpleLlama/pull/189); tamper-evident +signed, hash-chained results requested (https://github.com/meta-llama/PurpleLlama/issues/211). + +### metatool-ai/metamcp + +TypeScript MCP gateway aggregating servers into namespaced endpoints with +OAuth, API keys, and an admin UI. Pooled upstream sessions are the dominant +pain: stale session IDs after idle expiry, per-namespace staleness +multiplied, a leaked idle session per refresh cycle, and pool exhaustion +surfacing as a misleading "Unknown tool" +(https://github.com/metatool-ai/metamcp/issues/294, /issues/324, /issues/304, /issues/318); +the proxy injected schema-declared optional parameters with empty defaults +so strict servers rejected correctly-formed calls +(https://github.com/metatool-ai/metamcp/issues/365); StreamableHTTP never +enabled JSON responses, and `tools.list_changed` is never advertised so +clients hold stale tool lists (https://github.com/metatool-ai/metamcp/issues/331, /issues/314); +OAuth token exchange read `client_id` from the body only, failing every +RFC 6749 `client_secret_basic` client 100% of the time +(https://github.com/metatool-ai/metamcp/issues/342); multi-user trust +boundary wounds: a tool gate that fails open, unscoped admin queries, and +CVE-2026-79538 (CVSS 9.8) command spawn + SSRF from request input +(https://github.com/metatool-ai/metamcp/issues/364, /issues/375, /issues/381); +a June batch built pooled-session resilience worth copying — re-init on 404 +per the spec, disjoint "session lost" vs "transport lost" envelopes feeding +one recovery path, and TOCTOU fixes for leaking stdio children +(https://github.com/metatool-ai/metamcp/pull/283, /pull/293, /pull/273). + +### mostlygeek/llama-swap + +Go model-swapping reverse proxy for local backends (llama-server, vLLM, +ComfyUI, audio), growing into a GPU-capacity-aware serving platform with +SQLite activity/capture stores. A pooled connection raced llama-server's +post-SSE close (`Keep-Alive` advertised but connection dying) — fixed with +a per-model `disableKeepAlives` option +(https://github.com/mostlygeek/llama-swap/issues/1205); forwarding the +browser's `Origin` header on the server-to-server hop made Anthropic 401 — +strip browser-only headers when the proxy terminates CORS +(https://github.com/mostlygeek/llama-swap/issues/1214); concurrency-limit +rejections returned as 429 so clients armed hour-long backoff ladders for a +90-second wait — capacity shedding must be 503 +(https://github.com/mostlygeek/llama-swap/issues/1256); `/v1/models` +omitted `context_length` for never-loaded models, forcing clients to guess +— probe parsing made tolerant of upstream schema drift via field-wise reads +(https://github.com/mostlygeek/llama-swap/issues/1281); a hard-coded 250 MB +body limit silently broke multi-GB uploads between releases +(https://github.com/mostlygeek/llama-swap/issues/1245); per-backend token +semantics differ (llama.cpp excludes cached tokens from prompt counts, +vLLM includes them) — normalize per upstream +(https://github.com/mostlygeek/llama-swap/issues/1218); usage parsing +extracted into pluggable per-upstream extractors dispatched off an upstream +hint (https://github.com/mostlygeek/llama-swap/pull/1292); request/response +captures moved into a separate SQLite DB pruned asynchronously by total +compressed size, preserving request totals across pruning +(https://github.com/mostlygeek/llama-swap/pull/1294). + +### mozilla-ai/llamafile + +Single-file cosmopolitan llama.cpp distribution with an OpenAI/Anthropic-compatible +server. GPU/backend detection is the dominant unresolved bucket — AMD iGPUs +ignored, pre-Pascal laptops failing both backends +(https://github.com/mozilla-ai/llamafile/issues/965, /issues/1048); new +architectures wait on vendored-upstream bumps +(https://github.com/mozilla-ai/llamafile/issues/1075); no build provenance +ties release assets to a controlled build +(https://github.com/mozilla-ai/llamafile/issues/1059); HTTPS rides a frozen +2021 Mbed TLS fork (https://github.com/mozilla-ai/llamafile/issues/1012); +the default combined mode skips the sandbox entirely +(https://github.com/mozilla-ai/llamafile/issues/1017); Unix-socket listening +silently failed for 5.5 months — the fix binds the socket before installing +the sandbox (https://github.com/mozilla-ai/llamafile/issues/971); agentic +layering: documented built-in tools plus an Agentfile CLI with opt-in +web tools and guardrail callbacks (https://github.com/mozilla-ai/llamafile/pull/1060). + +### mudler/LocalAI + +Mature Go self-hosted OpenAI-compatible inference server turning into an +agentic platform. The prompt cache works but the API never returns +`usage.prompt_tokens_details.cached_tokens`, so gateways in front see every +request as uncached (https://github.com/mudler/LocalAI/issues/13006); +Responses clients send MCP tools as a single `{"type":"namespace"}` entry +that the converter drops — flattened to `ns_{n}___{name}` and split back +(https://github.com/mudler/LocalAI/pull/13098); TTS rejecting +`response_format=mp3` forced every mp3 client to run its own ffmpeg proxy — +format negotiation or an explicit 415 is the drop-in difference +(https://github.com/mudler/LocalAI/issues/13094); an in-app gallery +simulator asked so contributors can validate catalog definitions offline +(https://github.com/mudler/LocalAI/issues/13040); frontends sharing one +metadata DB can silently diverge to different blob backends — a stored +backend-identity fingerprint checked loudly at startup +(https://github.com/mudler/LocalAI/issues/13047); local signup trusted a +configured admin email without verification +(https://github.com/mudler/LocalAI/pull/12993); a capability-gated add-on +platform where a designated owner holds revisioned authority over egress +allowlists and settings, failing closed, re-authorizing every request +(https://github.com/mudler/LocalAI/pull/13033); durable, capped agent memory +with propose/confirm settlement (https://github.com/mudler/LocalAI/pull/13064). + +### oobabooga/textgen + +Python/Gradio local-LLM web UI with OpenAI/Anthropic-compatible APIs. The +API returns a non-standard `reasoning_content` field that mainstream +clients silently drop, yielding empty content — strict spec conformity +requested (https://github.com/oobabooga/textgen/issues/7601); premature +`LengthFinishReasonError` and mid-session `RemoteProtocolError` deaths +(https://github.com/oobabooga/textgen/issues/7585); bare tool-call parsing +resolves names by order-dependent prefix match, so `search_web` silently +executes `search` — longest-match resolution proposed twice +(https://github.com/oobabooga/textgen/issues/7666); image-heavy +conversations die at the transport's payload limit — opt-in image +downscaling before forwarding proposed +(https://github.com/oobabooga/textgen/issues/7603); per-request thinking +toggles and prior-turn reasoning stripping unresolved +(https://github.com/oobabooga/textgen/issues/7650); a loader endpoint +accepted-and-ignored an `args` dict and returned 200 — never +accept-and-ignore (https://github.com/oobabooga/textgen/issues/7577); +path-traversal fixes for API-supplied template names +(https://github.com/oobabooga/textgen/pull/7562); tool-call argument +parsing hardened for quotes and newlines — the recurring failure mode of +streamed tool args (https://github.com/oobabooga/textgen/pull/7477). + +### open-webui/mcpo + +Python/FastAPI "MCP-to-OpenAPI" proxy; hard maintenance stall — zero merges +in ~9 months while 15 later PRs (incl. security fixes) queue. A transient +502 left a broken async teardown pinning one CPU core at 100% for ~14 +hours — containing upstream session failure per-connection is the hard part +(https://github.com/open-webui/mcpo/issues/302); MCP JSON-Schemas are not +valid OpenAPI schemas and the converter keeps crashing — enums dropped, +$ref broken, tuple items crashing startup +(https://github.com/open-webui/mcpo/issues/296, /issues/245, /issues/244); +static OAuth client metadata unusable because DCR is always performed — +Google has no DCR (https://github.com/open-webui/mcpo/issues/272); embedded +resources unhandled (https://github.com/open-webui/mcpo/issues/270); a +shipped option bricks startup — config surface needs round-trip validation +(https://github.com/open-webui/mcpo/issues/267); no per-server health +visibility (https://github.com/open-webui/mcpo/issues/225); the merged arc +is upstream-session resilience — reconnect/retry on ClosedResourceError, +tolerate some down upstreams, symlink-safe hot reload +(https://github.com/open-webui/mcpo/pull/266, /pull/194, /pull/212); a +constant-time auth comparison fix sits unmerged +(https://github.com/open-webui/mcpo/pull/317). + +### protectai/llm-guard + +Python composable LLM security scanners; archived 2026-07-08. The Secrets +scanner LEAKS plaintext when redact_mode=all and a prompt contains more +than one secret — offset drift; the fix stayed unmerged +(https://github.com/protectai/llm-guard/issues/305); deployment weight — +8 GB of model downloads when one scanner (~750 MB) is wanted +(https://github.com/protectai/llm-guard/issues/306); offline/air-gapped +operation broken by scanner model loading +(https://github.com/protectai/llm-guard/issues/347); false-positive +management never solved — no allowlist shipped +(https://github.com/protectai/llm-guard/issues/331); rigid dependency pins +froze downstream security patches +(https://github.com/protectai/llm-guard/issues/342); a known config +path-traversal in the API service carried unpatched to archival +(https://github.com/protectai/llm-guard/pull/323); unmerged community PRs +show demand had moved to agent-era scanning — tool-call audit, malicious +URLs, agent escape vectors — an entire niche left unowned at archival +(https://github.com/protectai/llm-guard/pull/327, /pull/350). + +### redis/redis-vl-python + +Redis's official Python client for vector search and agentic RAG, with a +semantic cache and an MCP server. Index-existence detection string-matches +error wording and keeps regressing per server flavor — classify by reply +type (https://github.com/redis/redis-vl-python/issues/773); `NullHandler` +on library loggers opts out of the stderr fallback so data-loss warnings +are never shown (https://github.com/redis/redis-vl-python/issues/786); +`to_dict()` lower-cases an enum that `from_dict()` rejects — self-produced +config must round-trip (https://github.com/redis/redis-vl-python/issues/808); +raw `execute_command` bypassed protocol response callbacks so the fallback +never ran (https://github.com/redis/redis-vl-python/issues/714); MCP +security arc: tool profiles AND-scoped to a tenant claim from the caller's +verified token, failing closed when the scope gate can't reach its config, +with the Host/Origin guard extended to every HTTP transport +(https://github.com/redis/redis-vl-python/pull/745, /pull/716, /pull/746); +TTL-fused cache round trips cut a 10-hit mget from 4,166 µs to 1,173 µs +(https://github.com/redis/redis-vl-python/pull/764); an `otel` extra with +per-operation spans and hit/miss counters — "every deployment writes its +own counters" (https://github.com/redis/redis-vl-python/pull/814). + +### samanhappy/mcphub + +TypeScript MCP gateway (hono + better-auth + dashboard), extremely fast +maintenance — 95 of 100 PRs merged within three weeks, issues closed in +0–4 days. Config-edit blast radius was the worst production pain — every +admin action re-spawned every non-connected stdio upstream; the fix scopes +invalidation to the changed server +(https://github.com/samanhappy/mcphub/issues/1448); re-tokenizing static +tool definitions on every dashboard poll saturated the event loop — cache +counts by digest (https://github.com/samanhappy/mcphub/issues/1408); +forward-auth JWTs via JWKS requested for identity-aware-proxy fronting +(https://github.com/samanhappy/mcphub/issues/1456); allow-list-only scoping +broke silently as servers were added — "all-except" exclude scoping +shipped, with exclusions filtered from listings +(https://github.com/samanhappy/mcphub/issues/1402); 99-tool governance +needed bulk selection and one batch endpoint +(https://github.com/samanhappy/mcphub/issues/1371); keys hashed at rest +with prefix-only display and per-user upstream credential slots injected +into env/args/URLs (https://github.com/samanhappy/mcphub/pull/1387, /pull/1313); +protocol negotiation per upstream with change propagation to both session +eras (https://github.com/samanhappy/mcphub/pull/1416); PostgreSQL replica +mode with config revisions and If-Match concurrency +(https://github.com/samanhappy/mcphub/pull/1431). + +### smart-mcp-proxy/mcpproxy-go + +Go local MCP gateway with agent tokens, authz profiles, and an offline +security scanner; 96 merged PRs in two weeks. Secret references (`${file:}`, +`${env:}`) expand in every config string with no allow-list — an +arbitrary-file-read primitive via a header +(https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1721); Docker stdio +upstreams put resolved secrets on `-e KEY=VALUE` argv, exposing them in +`/proc` (https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1809); a +bbolt read transaction held across a channel send hung every authenticated +request 40+ minutes while `/healthz` stayed 1 ms +(https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1742); per-server +init timeouts silently capped at 30 s by a parent context +(https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1772); real-client +interop tested against shipped binaries — Codex and Claude Desktop user +agents matched with a strict trailing-tag grammar +(https://github.com/smart-mcp-proxy/mcpproxy-go/pull/1799); scanning became +the default trust mode, backed by signed Ed25519 sequence-versioned rule +bundles with downgrade protection +(https://github.com/smart-mcp-proxy/mcpproxy-go/pull/1740, /pull/1701); +authz profiles with refusal attribution and `access suggest` proposing +reviewed patches that never auto-widen +(https://github.com/smart-mcp-proxy/mcpproxy-go/pull/1718); per-token +call-rate limits at one admission seam +(https://github.com/smart-mcp-proxy/mcpproxy-go/pull/1746). + +### sparfenyuk/mcp-proxy + +Python MCP transport bridge (stdio/SSE/Streamable HTTP, multi-server +configs), actively hardened by a solo maintainer. Multi-key scoped metadata +requested — inbound auth is a single shared token +(https://github.com/sparfenyuk/mcp-proxy/issues/201); a per-call +authorization hook requested because putting a stdio server on the network +is all-or-nothing (https://github.com/sparfenyuk/mcp-proxy/issues/270); +per-server circuit breakers requested (https://github.com/sparfenyuk/mcp-proxy/issues/202); +upstream-aware `/health` requested — `/status` always returns 200 +(https://github.com/sparfenyuk/mcp-proxy/issues/197); one failing server +crashed the whole proxy until failures were isolated per server +(https://github.com/sparfenyuk/mcp-proxy/issues/198); the proxy kept +answering 200 after the wrapped stdio child died, so supervisors never +restarted it — exit non-zero when the child exits +(https://github.com/sparfenyuk/mcp-proxy/issues/247); the dominant merged +theme is radical protocol transparency — forward every optional field both +ways (https://github.com/sparfenyuk/mcp-proxy/pull/254, /pull/226, /pull/244, /pull/189, /pull/242, /pull/272, /pull/298); +and bounded lifecycle waits that never replay side effects +(https://github.com/sparfenyuk/mcp-proxy/pull/285, /pull/277, /pull/241). + +--- + +## Pain-point index — theme → repos and issues + +Cross-repo user pain, strongest first. Each theme lists the repos and the +representative issue/PR evidence. + +**1. Retry/failover misclassification takes healthy capacity offline.** +sub2api — a request-level 403 disables every Gemini account in sequence +(https://github.com/Wei-Shaw/sub2api/issues/8442); family-wide cooldowns and +ignored Retry-Afters (https://github.com/Wei-Shaw/sub2api/issues/8439); +CLIProxyAPI — a model-scoped 429 benches a whole credential ~6 days +(https://github.com/router-for-me/CLIProxyAPI/issues/7197); metapi — any 401 +or "token"+"invalid" substring marks keys expired +(https://github.com/cita-777/metapi/issues/582); one-hub — no retry on +504/524 (https://github.com/MartialBE/one-hub/issues/852); opencodex — +pre-output 503 returned instead of hopping +(https://github.com/lidge-jun/opencodex/issues/8107); mcp-gateway — limiter +keyed by display name (https://github.com/MikkoParkkola/mcp-gateway/issues/3950). + +**2. Streaming failure semantics lie to clients.** sub2api — abnormal +termination masked as `end_turn` + `[DONE]` +(https://github.com/Wei-Shaw/sub2api/issues/8505); koboldcpp — HTTP 200 + +`finish_reason:"error"` drove retry storms +(https://github.com/LostRuins/koboldcpp/issues/2481); freellmpool — fake +`finish=stop` after mid-stream failure +(https://github.com/0xzr/freellmpool/pull/17); CCR — truncated stream as a +bare socket destroy (https://github.com/musistudio/claude-code-router/issues/1917); +glide — body-validation failure closes the stream channel +(https://github.com/EinStack/glide/issues/272); jan — truncation mapped to +`stop` (https://github.com/janhq/jan/pull/9515); opencodex — a stall 504 +re-classified as 502 (https://github.com/lidge-jun/opencodex/issues/8157). + +**3. Usage/token accounting has blind spots — especially cached tokens.** +done-hub — cache tokens logged 0 on OpenAI-compatible channels +(https://github.com/deanxv/done-hub/issues/51); one-hub — Claude streaming +loses cache fields (https://github.com/MartialBE/one-hub/issues/884); +AgentOps — Anthropic cache tokens dropped +(https://github.com/AgentOps-AI/agentops/issues/1445); LocalAI — +`cached_tokens` never returned (https://github.com/mudler/LocalAI/issues/13006); +llama-swap — per-backend cached-token semantics differ +(https://github.com/mostlygeek/llama-swap/issues/1218); opencodex — Ollama +cached count dropped (https://github.com/lidge-jun/opencodex/issues/8101); +aisix — secondary endpoints emit no usage events at all +(https://github.com/api7/aisix/issues/748); koboldcpp — streamed tool +requests emit no final usage chunk +(https://github.com/LostRuins/koboldcpp/issues/2480); otari — abandoned +streams billed upstream but free in the ledger +(https://github.com/mozilla-ai/otari/pull/2869). + +**4. Protocol translation drops or mangles client fields into hard 400s.** +CCR — `max_tokens` and `thinking` stripped +(https://github.com/musistudio/claude-code-router/issues/1843); CLIProxyAPI — +Gemini roles, parallel tool batches, Devin reasoning +(https://github.com/router-for-me/CLIProxyAPI/issues/7186); done-hub — +object-vs-string fields break typed unmarshalling +(https://github.com/deanxv/done-hub/issues/58); metapi — Gemini +`thought_signature` replay rejected +(https://github.com/cita-777/metapi/issues/580); one-hub — OpenAI→Gemini +tool translation (https://github.com/MartialBE/one-hub/issues/918); +completion-to-response — empty `reasoning_content` omitted +(https://github.com/NoahStepheno/completion-to-response/pull/1); +bedrock-gateway — per-generation thinking-API churn +(https://github.com/aws-samples/bedrock-access-gateway/issues/248); +oobabooga — non-standard `reasoning_content` breaks clients +(https://github.com/oobabooga/textgen/issues/7601). + +**5. Credential/OAuth lifecycle fragility.** CCR — Codex OAuth import +structurally broken (https://github.com/musistudio/claude-code-router/issues/1941); +CLIProxyAPI — refresh on a read-only store invalidates every credential +after restart (https://github.com/router-for-me/CLIProxyAPI/issues/7214); +sub2api — plan renames and quota-shape changes break account management +(https://github.com/Wei-Shaw/sub2api/issues/8456); 1mcp — OAuth initiation +failures around RFC 9728 matching +(https://github.com/1mcp-app/agent/issues/653); mcpo — static OAuth clients +unusable (https://github.com/open-webui/mcpo/issues/272); metamcp — +`client_secret_basic` clients fail 100% +(https://github.com/metatool-ai/metamcp/issues/342); copilot-api — token +swapping breaks provider analytics +(https://github.com/ericc-ch/copilot-api/issues/228). + +**6. Hardcoded model catalogs lag provider launches.** claude-code-proxy — +every new model needs a release (https://github.com/raine/claude-code-proxy/issues/198); +sub2api — new models missing from sync +(https://github.com/Wei-Shaw/sub2api/issues/8508); inference-gateway — +deprecation dimension added via models.dev +(https://github.com/inference-gateway/inference-gateway/issues/793); +RouterArena — unknown slugs block submissions +(https://github.com/RouteWorks/RouterArena/issues/206); freellmpool — +sentinel auto-filed eight drift issues +(https://github.com/0xzr/freellmpool/issues/141); PurpleLlama — static +vendor lists reject custom endpoints +(https://github.com/meta-llama/PurpleLlama/pull/253). + +**7. Pricing data rots; unpriced traffic looks free.** tokdiet — absent +models silently $0.00 (https://github.com/agiwhitelist/tokdiet/issues/1); +RouterArena — unpriced models silently skipped at evaluation +(https://github.com/RouteWorks/RouterArena/issues/193); AgentOps — invalid +price rows abort collector config +(https://github.com/AgentOps-AI/agentops/issues/1460); Routerly — missing +cache prices default to full input rate +(https://github.com/Inebrio/Routerly/issues/144); awesome-ai-gateway — +snapshot freshness gates failing +(https://github.com/cuihuan/awesome-ai-gateway/issues/146); CCR — 24.6 MB +generated catalog needed to make pricing work out of the box +(`research-repos/musistudio__claude-code-router/packages/core/models.json`). + +**8. Config surface silently no-ops or bricks startup.** mcphub — every +admin action re-spawned every upstream +(https://github.com/samanhappy/mcphub/issues/1448); oobabooga — loader +accepted-and-ignored args with a 200 +(https://github.com/oobabooga/textgen/issues/7577); mcpo — `disabled_tools` +bricks startup (https://github.com/open-webui/mcpo/issues/267); Mooncake — +boolean env vars silently ignored +(https://github.com/kvcache-ai/Mooncake/pull/5125); ferro-labs — guardrail +config mistakes enforced nothing +(https://github.com/ferro-labs/ai-gateway/issues/447); GoModel — credential +updates wipe discovered inventory +(https://github.com/ENTERPILOT/GoModel/issues/1308); CLIProxyAPI — runtime +auth-dir changes leave the watcher behind +(https://github.com/router-for-me/CLIProxyAPI/issues/7160); mcpproxy-go — +per-server timeouts silently capped at 30 s +(https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1772). + +**9. Security boundary failures at the loopback/edge.** metapi — admin IP +allowlist spoofable via X-Forwarded-For +(https://github.com/cita-777/metapi/issues/631); NyaProxy — unauthenticated +dashboard + redirect SSRF + credentialed wildcard CORS +(https://github.com/Nya-Foundation/NyaProxy/pull/100); Unla — middleware +order stripped admin checks (https://github.com/AmoyLab/Unla/issues/339); +metamcp — CVE-2026-79538 command spawn + SSRF +(https://github.com/metatool-ai/metamcp/issues/381); cloudflare-workers — +caller auth headers forwarded and overridable +(https://github.com/blue-pen5805/llm-proxy-on-cloudflare-workers/issues/350); +freellmpool — redirects leak credentials to internal targets +(https://github.com/0xzr/freellmpool/issues/87); mcpproxy-go — secret +references as arbitrary file read +(https://github.com/smart-mcp-proxy/mcpproxy-go/issues/1721); mistral.rs — +0.0.0.0 default with no auth (https://github.com/EricLBuehler/mistral.rs/issues/2490). + +**10. Session/state stickiness lost across failover or restart.** +claude-code-proxy — subagents sharing a parent session id destroy state +(https://github.com/raine/claude-code-proxy/issues/163); sub2api — WS +failover replays the conversation's first request +(https://github.com/Wei-Shaw/sub2api/issues/8513); mcp-gateway — busy +sessions reaped by age not idleness +(https://github.com/MikkoParkkola/mcp-gateway/issues/2567); metamcp — stale +pooled sessions surface as "Unknown tool" +(https://github.com/metatool-ai/metamcp/issues/318); plano — session ids +read only from an internal header +(https://github.com/katanemo/plano/issues/979); CCR — fallback served-model +misattribution (https://github.com/musistudio/claude-code-router/issues/1870). + +**11. Guardrails/evaluators fail open or silently skip coverage.** otari — +inconclusive verdict passes a block-mode guardrail +(https://github.com/mozilla-ai/otari/issues/2788); presidio — HF recognizer +returns empty on error (https://github.com/data-privacy-stack/presidio/issues/2441); +PurpleLlama — async wrapper discards scanner scores +(https://github.com/meta-llama/PurpleLlama/pull/185); guardrails-ai — +permissive defaults differ per surface +(https://github.com/guardrails-ai/guardrails/issues/1656); aisix — +scan-vs-forward asymmetry as the largest defect class +(https://github.com/api7/aisix/issues/1016); tokdiet — pricing failures +invisible in the dashboard (https://github.com/agiwhitelist/tokdiet/issues/1). + +**12. Ops/packaging fragility.** ollamaMQ — Docker entrypoint flag mismatch +(https://github.com/Chleba/ollamaMQ/issues/3); claude-code-proxy — GLIBC +floor jump made releases unrunnable +(https://github.com/raine/claude-code-proxy/issues/144); sub2api — one-click +self-update broken by permissions +(https://github.com/Wei-Shaw/sub2api/issues/8467); llama-cpp-python — +corrupted wheels from dual runners +(https://github.com/abetlen/llama-cpp-python/issues/2352); inference-gateway +— generated example configs unreachable in Docker +(https://github.com/inference-gateway/inference-gateway/issues/735); glide — +released image panicking on a missing external file +(https://github.com/EinStack/glide/issues/209). + +## Direction index — where the niche ecosystem is heading + +**Coding-agent-first servers.** mistral.rs ships per-request context limits +"for coding agents" with a rewritten agent guide +(https://github.com/EricLBuehler/mistral.rs/pull/2538); claude-code-proxy +fingerprint's Claude Code's auxiliary requests for cheaper routing +(https://github.com/raine/claude-code-proxy/pull/72); CCR intercepts the +client's own compaction flow (https://github.com/musistudio/claude-code-router/pull/1824); +plano replays routing decisions on tool-result turns +(https://github.com/katanemo/plano/pull/1015); opencodex sanitizes every +client-facing error to proxy-owned constants +(https://github.com/lidge-jun/opencodex/pull/8108). + +**Quota-aware multi-account scheduling.** CLIProxyAPI's `soonest-reset` +strategy from passively observed quota headers +(https://github.com/router-for-me/CLIProxyAPI/pull/7246) and final +usage-limit errors with reset times +(https://github.com/router-for-me/CLIProxyAPI/pull/7239); sub2api's +quota-window account pausing +(`research-repos/Wei-Shaw__sub2api/backend/internal/service/account_scheduling_threshold_eval.go`); +opencodex's headroom-aware placement +(https://github.com/lidge-jun/opencodex/pull/8092); yolorouter's learned +reset evidence (`research-repos/yolorouter__yolorouter/internal/gateway/rate_limit_evidence.go`). + +**MCP as a universal attach point.** aisix per-user OAuth to upstream MCP +servers (https://github.com/api7/aisix/pull/1394); inference-gateway as a +stateless 2026-07-28 MCP server +(https://github.com/inference-gateway/inference-gateway/pull/709); GoModel +serving provider-hosted MCP with per-key scoping +(https://github.com/ENTERPILOT/GoModel/pull/1272); SMG's MCP tool execution +inside the Responses path (https://github.com/smg-project/smg/pull/5045); +NVIDIA Guardrails' hooks sidecar for Claude Code/Codex +(https://github.com/NVIDIA-NeMo/Guardrails/pull/2473); mcp-gateway's +meta-tool surface with TF-IDF discovery +(`research-repos/MikkoParkkola__mcp-gateway/src/semantic_search/mod.rs`). + +**models.dev as the canonical catalog/pricing source.** otari persisting +the full v2 catalog (https://github.com/mozilla-ai/otari/pull/2771); plano's +pluggable pricing feeds (https://github.com/katanemo/plano/pull/971); +inference-gateway syncing deprecation status +(https://github.com/inference-gateway/inference-gateway/issues/793); +done-hub's scheduled price sync +(`research-repos/deanxv__done-hub/model/pricing_modelsdev.go:18-108`); CCR's +generated merged catalog +(`research-repos/musistudio__claude-code-router/scripts/generate-models-json.mjs:17-32`). + +**Policy/config hot reload under pinned generations.** mcp-gateway's +per-request pinned config generation (ADR-029, +https://github.com/MikkoParkkola/mcp-gateway/pull/4508); CLIProxyAPI's +debounced watcher with request re-dispatch +(`research-repos/router-for-me__CLIProxyAPI/internal/watcher/config_reload.go`); +GoModel's DB-only clustersync in ~2 s +(https://github.com/ENTERPILOT/GoModel/pull/1295); ThinkWatch's ArcSwap + +Redis pub/sub (`research-repos/ThinkWatchProject__ThinkWatch/crates/common/src/dynamic_config.rs`); +mcphub's surgical reload (https://github.com/samanhappy/mcphub/issues/1448). + +**Loss-aware, diagnostics-carrying translation.** Switchyard's +`TranslationPolicy` with structured per-field diagnostics +(`research-repos/NVIDIA-NeMo__Switchyard/crates/switchyard-translation/README.md:91-123`, +issue https://github.com/NVIDIA-NeMo/Switchyard/issues/1215); NVIDIA +Guardrails preserving LiteLLM dialect fields and encrypted reasoning +(https://github.com/NVIDIA-NeMo/Guardrails/pull/2529); sparfenyuk's radical +optional-field transparency (https://github.com/sparfenyuk/mcp-proxy/pull/254); +aisix choosing native protocol fidelity over hub-and-spoke where +translation proved lossy (https://github.com/api7/aisix/pull/1318). + +**Cost governance moving onto the gateway.** otari's tools as billable +units with budget holds (https://github.com/mozilla-ai/otari/pull/2831, +/pull/2839); GoModel's reseller markups and invoice export +(https://github.com/ENTERPILOT/GoModel/pull/1293); mcp-gateway's <0.1 ms +pre-invoke budget enforcer +(`research-repos/MikkoParkkola__mcp-gateway/src/cost_accounting/enforcer.rs`); +Switchyard's dual-source cost with provider-reported precedence +(`research-repos/NVIDIA-NeMo__Switchyard/crates/switchyard-costs/src/lib.rs:46-123`). + +**LLM-judged correctness for caches and transformations.** GPTCache's JEV +five-question reuse gate (https://github.com/zilliztech/GPTCache/pull/701) +plus its frozen reuse benchmark (https://github.com/zilliztech/GPTCache/pull/702); +prompt-cache's gray-zone judge +(`research-repos/messkan__prompt-cache/internal/semantic/semantic.go:256-336`); +tokdiet's shadow-eval compaction guard +(`research-repos/agiwhitelist__tokdiet/src/quality/guard.ts:83-136`). + +**Hardened, attested releases and supply chain.** antirion's SBOM + Sigstore ++ SLSA L3 releases (`research-repos/eliasbiondo__antirion/.github/workflows/release.yml`); +preflight's provenance-pinned weekly rule refresh +(`research-repos/ghuntley__preflight/.github/workflows/rules.yml`); +mistral.rs's verify-release job deriving the expected artifact set after a +release silently published 42 of 143 artifacts +(https://github.com/EricLBuehler/mistral.rs/pull/2407); SMG's resumable +release script (https://github.com/smg-project/smg/pull/4977); NyaProxy's +wheel/container smoke tests (https://github.com/Nya-Foundation/NyaProxy/pull/111). + +**Multi-replica state sharing.** otari's Redis rate-limit store with +clock-skew-safe Lua (`research-repos/mozilla-ai__otari/src/gateway/adapters/rate_limit_store_adapter.py`); +antirion's Valkey cluster mode; SMG's gossip mesh +(`research-repos/smg-project__smg/crates/mesh/src/lib.rs`); GoModel's +per-replica writer ids (https://github.com/ENTERPILOT/GoModel/pull/1296); +plano's Redis session-affinity store +(`research-repos/katanemo__plano/crates/brightstaff/src/session_cache/redis.rs`). + +**Per-key least privilege.** CLIProxyAPI's per-key model policies +(https://github.com/router-for-me/CLIProxyAPI/pull/7117); sub2api's ~60 +selectable admin permissions +(https://github.com/Wei-Shaw/sub2api/pull/8476); mcphub's per-user +credential slots (https://github.com/samanhappy/mcphub/pull/1313); +mcpproxy-go's authz profiles with `access suggest` +(https://github.com/smart-mcp-proxy/mcpproxy-go/pull/1718); redis-vl's +tenant-claim-scoped tool profiles +(https://github.com/redis/redis-vl-python/pull/745). + +## Recurring unmet needs — patterns in 3+ repositories + +Patterns demanded by users of three or more harvested repositories, each a +direct backlog input (see `compensation-backlog.json`). + +**R1. Retry/failover that classifies before it reacts (12 repos).** +ThinkWatch PR #122, Switchyard `client.rs:400-528`, xlb `config.rs:253-298`, +done-hub `relay/main.go:74-176`, CLIProxyAPI conductor, sub2api failover +loop, freellmpool PR #97, glide PR #68, one-hub issue #852, opencodex issue +#8107, smg `retry.rs`, antirion `budget.rs`. The recurring failure is not +absence but misclassification (sub2api #8442, CLIProxyAPI #7197, metapi +#582, opencodex #8107) — cooldowns keyed to model+credential scope and real +reset times, not blanket account bans. + +**R2. Cached-token usage accounting end to end (12 repos).** done-hub PRs +#39/#52, one-hub #884, opencodex #8101, AgentOps #1445, LocalAI #13006, +llama-swap #1218, Routerly #144, otari #2869, Switchyard cache-write split, +aisix `token_estimate.rs`, CLIProxyAPI PR #7074, plano PR #995. llm-proxy's +core usage already carries cache fields fed only from Anthropic dialects +(`crates/llm-proxy-protocol/src/core.rs:639-641,683-685`; no +`prompt_tokens_details` parsing anywhere — verified 2026-11-04). + +**R3. Context-window signaling and preflight (7 repos).** mistral.rs PR +#2538, opencodex PR #8128, copilot-api #210, llama-swap #1281, ostk-cache +`usage_truth.rs`, koboldcpp #2481 (truncation semantics), yolorouter #9 +(capability metadata). Clients branch on the mapped overflow code to decide +compact-vs-retry; llm-proxy already has the offline counter and +`context_length` on `/v1/models` but no preflight. + +**R4. Session attribution and sticky routing (9 repos).** sub2api +`balancer.go:194-309`, CLIProxyAPI `session_cache.go:31-45`, done-hub sticky +sessions, otari session stats + #2792, Switchyard `routing_log.rs`, tokdiet +`session.ts:20-61`, plano session affinity, ThinkWatch sealed pinning +(`sealed.rs:1-92`), SMG cache-aware stickiness. llm-proxy's ledger records +only key id — no session, repo, or client-tool attribution. + +**R5. Streaming idle tolerance: keepalives, phase timeouts, terminal usage +(7 repos).** done-hub `heartbeat.go:14-175`, Switchyard #1168, xlb PR #2, +yolorouter `config.go:42-79`, copilot-api #223, 1mcp #709, koboldcpp #2480. +A single flat timeout cannot express prefill wait vs reasoning pause vs +total stream; silent generation windows also trip client and proxy +timeouts that heartbeat comments prevent. + +**R6. Subscription/OAuth upstream credentials (7 repos).** claude-code-proxy +`auth/manager.rs:18-70`, CLIProxyAPI `internal/auth/`, done-hub OAuth +channels, sub2api token providers, Routerly PR #107, metapi PR #430, +copilot-api #255. The fastest-growing upstream segment is unreachable for +llm-proxy's static bearer keys. + +**R7. Externally synced price catalogs with explicit unpriced state +(6 repos).** otari `pricing_refresh_service.py`, plano PR #971, done-hub +`pricing_modelsdev.go`, CCR `generate-models-json.mjs`, RouterArena PR #218, +tokdiet #1. llm-proxy's hand-typed fixed-point TOML has no refresh story +and no way to distinguish unpriced from free. + +**R8. Per-key scoping, windows, and budgets (8 repos).** yolorouter +`api_key.go`, otari `budgets.py`, mcp-gateway `enforcer.rs`, GoModel PR +#1293, NyaProxy PR #133, one-hub #835 (resale operators asking for +per-API-key rate limits because group-level limits throttle every user +sharing the group, https://github.com/MartialBE/one-hub/issues/835), +ThinkWatch PR #81, aisix conditional policies. llm-proxy rate-limits per +identity — caller key id when authenticated, client IP otherwise +(`crates/llm-proxy-server/src/routes/core_pipeline.rs:173-176`, per-key RPM +already exists per the compendium's narrowed gap #3) — but has no token +(TPM) windows, per-key spend caps, or per-key model scoping, and its keys +grant all-or-nothing model access. + +**R9. Guardrails/inspection that fail closed (6 repos).** otari #2788, +presidio #2441, PurpleLlama PR #185, guardrails-ai #1656, aisix #1016, +defender PR #106. Inconclusive verdicts must land on the closed side, and +sync/async paths must share one aggregation. + +**R10. An admin/config management plane beyond read-only (5 repos).** +CLIProxyAPI management v8, NyaProxy PR #134, mcphub batch endpoints, GoModel +clustersync, prompt-cache `PATCH /v1/config` + provider hot-swap. Operators +expect to change providers/keys/thresholds without a restart; llm-proxy +rejects `server.hot_reload` outright +(`crates/llm-proxy-core/src/provider_config.rs:1531-1538` per the wave-3 +register) and its admin API is read-only. + +**R11. Prompt-cache-aware request handling (6 repos).** ostk-cache +`billing_strip.rs` (cache-busting token strip), sub2api PR #8464 (fingerprint +convergence for cache hits), CCR PR #1866 (cache-key headers), plano +`prompt_caching.rs` (marker injection), tokdiet `providers.ts:318-369` +(mutation guard), jan PR #9497 (byte-stable memory snapshot). None of this +exists in llm-proxy: no normalization, no marker injection, no +mutation-safety boundary. + +**R12. In-flight/observable failure isolation (4 repos).** 1mcp #641 (one +backend's exit must not wedge the gateway), sparfenyuk #198 (a single +failing server crashed the whole proxy until failures were isolated per +server, https://github.com/sparfenyuk/mcp-proxy/issues/198), mcphub #1448, +metamcp pool exhaustion #318. Whatever fails per-upstream must degrade only +that upstream. diff --git a/docs/research/repos-registry.json b/docs/research/repos-registry.json new file mode 100644 index 0000000..16b39d8 --- /dev/null +++ b/docs/research/repos-registry.json @@ -0,0 +1,5808 @@ +{ + "generated": "2026-11-04", + "scope": "Inclusion: one row per product, platform, or specification researched in depth across the three waves (October-November 2026), including the vendored reference checkouts; wave-1/2 theme, sweep, and frontier findings are folded into the products they cover rather than listed separately. The 98 uninstantiated wave-3 candidates of the 121 deduplicated finds are tabled in docs/research/niche-repos.md, not here - hence product rows (73) fall short of the 90 inventory items. Capabilities carry a gap field mapping to the compendium's gap register: a register number 1-20 (';'-separated when several apply), a 'new:' wave-3 candidate gap (timeout-hierarchy, payload-cost-transforms, shadow-eval-guard, oauth-credential-lifecycle, upstream-cancellation, engine-state-routing, admission-queueing, caller-sso-rbac, synced-price-catalogs, hedged-requests), or 'context' for surveyed-but-unranked material (comparison axes, already-scoped future work, parity).", + "products": [ + { + "name": "Agent Router (formerly Envoy AI Gateway)", + "category": "open-source gateway (Envoy Gateway control plane)", + "language": "Go", + "status": "active (GA v1.0.0 2026-06-23; v1.2.0 first release under new name; Agentic AI Foundation)", + "url": "https://theagentrouter.ai/", + "capabilities": [ + { + "area": "auth", + "feature": "BackendSecurityPolicy CRD: API keys, AWS SigV4 via controller-side STS/OIDC exchange, Azure Entra tokens, GCP workload identity federation", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/security/upstream-auth", + "gap": "4" + }, + { + "area": "auth", + "feature": "Per-request credential injection / BYO-key via ext_authz metadata or x-aigw-api-key header", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/security/upstream-auth", + "gap": "context" + }, + { + "area": "auth", + "feature": "Client-facing auth delegated to Envoy Gateway SecurityPolicy (JWT, mTLS, OIDC, basic, API-key, IP allow/denylists, ext-authz)", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/security/", + "gap": "new:caller-sso-rbac" + }, + { + "area": "routing", + "feature": "Priority-ordered backend fallback with BackendTrafficPolicy retries that walk the chain; stream idle timeout with failover before first token", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/traffic/provider-fallback", + "gap": "1" + }, + { + "area": "routing", + "feature": "Model-name virtualization (modelNameOverride across providers; weighted traffic splits; excludeFromModelsEndpoint)", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/traffic/model-name-virtualization", + "gap": "13" + }, + { + "area": "routing", + "feature": "Inference-aware routing via Kubernetes InferencePool + Endpoint Picker on live metrics", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/inference/", + "gap": "new:engine-state-routing" + }, + { + "area": "rate-limiting", + "feature": "Token-usage rate limiting (InputToken/CachedInputToken/OutputToken/TotalToken or CEL cost) into Redis windows; admitted streams never interrupted", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/traffic/usage-based-ratelimiting", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "QuotaPolicy: platform-side token budgets per backend/model with fixed windows, CEL cost expressions, shadow mode, fail-open default", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/traffic/quota-policy", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "Dynamic per-tenant limits from auth metadata (requests_per_unit: 0 suspends a tenant)", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/traffic/usage-based-ratelimiting", + "gap": "3" + }, + { + "area": "mcp", + "feature": "MCP gateway: multi-server tool routing with per-tool CEL authorization, MCP OAuth (PKCE, RFC 8414), SSE merge with Last-Event-ID", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/mcp/", + "gap": "8" + }, + { + "area": "observability", + "feature": "Prometheus metrics on OTel GenAI semantic conventions plus GenAI tracing, AI-metadata access logs, Grafana dashboard", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/observability/metrics", + "gap": "11;12" + }, + { + "area": "protocol", + "feature": "Four client formats under configurable prefixes (OpenAI, Anthropic incl. count_tokens, Cohere rerank, TypeSafe) translated to ~20 providers", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/llm-integrations/supported-endpoints", + "gap": "6" + }, + { + "area": "caching", + "feature": "Provider-agnostic prompt-caching API (unified cache_control abstraction)", + "sourceUrl": "https://theagentrouter.ai/docs/capabilities/llm-integrations/", + "gap": "14" + }, + { + "area": "tooling", + "feature": "aigw run local mode without Docker or Kubernetes", + "sourceUrl": "https://theagentrouter.ai/docs/cli/aigwrun", + "gap": "context" + }, + { + "area": "ops", + "feature": "Two-tier gateway reference architecture (centralized tier one + inference-serving tier two)", + "sourceUrl": "https://theagentrouter.ai/docs/concepts/architecture", + "gap": "context" + }, + { + "area": "ops", + "feature": "Upstream egress via HTTP CONNECT forward proxy (v1.1.0)", + "sourceUrl": "https://github.com/theagentrouter/agent-router/blob/main/release-notes/v1.1.0.md", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "MCP multi-user safety hardening in v1.2: random session encryption seed, OAuth iss exact-match, CEL errors deny, ReferenceGrant for secrets", + "sourceUrl": "https://github.com/theagentrouter/agent-router/blob/main/release-notes/v1.2.0.md", + "gap": "context" + }, + { + "area": "ops", + "feature": "Envoy Gateway LB policies inherited: RoundRobin, LeastRequest, Random, ConsistentHash (Maglev)", + "sourceUrl": "https://gateway.envoyproxy.io/docs/tasks/traffic/load-balancing", + "gap": "13" + } + ] + }, + { + "name": "agentgateway", + "category": "open-source gateway", + "language": "Rust", + "status": "active (v1.7.0, 2026-10-27; Linux Foundation / Agentic AI Foundation)", + "url": "https://agentgateway.dev/", + "capabilities": [ + { + "area": "routing", + "feature": "Virtual models: weighted, failover (eviction groups), CEL conditional routing", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/virtual-models.md", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Retry policy with backend preference (same-backend retries, body-size threshold disables retries)", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/retries.md", + "gap": "1" + }, + { + "area": "auth", + "feature": "Virtual API keys (vk-...) with hash-only storage, model allowlists, UI-rotated values", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/virtual-keys.md", + "gap": "9" + }, + { + "area": "cost", + "feature": "Per-key budgets in USD or tokens with rolling windows; Block (429) or Audit modes; multiple windows per key", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/budget-limits/per-key.md", + "gap": "2" + }, + { + "area": "cost", + "feature": "Model cost catalog: per-token-kind USD pricing (input/output/cacheRead/cacheWrite/reasoning/audio), context tiers, models.dev import", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/costs.md", + "gap": "2" + }, + { + "area": "cost", + "feature": "Invoice-grade cost attribution: Bedrock per-request STS session names into CloudTrail/CUR; Vertex billing labels", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/cost-controls/attribution.md", + "gap": "2" + }, + { + "area": "guardrails", + "feature": "Ordered prompt guards: regex + built-in PII (mask/reject/audit), OpenAI moderation, vendor guards, webhooks, failClosed default", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/prompt-guards/overview.md", + "gap": "7" + }, + { + "area": "mcp", + "feature": "MCP gateway: virtual/multiplexed servers, tool namespacing, per-target CEL conditions", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/integrations/mcp/servers/virtual", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP authorization via CEL RBAC over method invocations plus ExtMCP guardrail processors", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/mcp/guardrails/about.md", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP spec-compatibility: stateless 2026-07-28 by default with cross-version negotiation and encrypted federated cursors", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/mcp/spec-compatibility.md", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP OAuth 2.1 resource-server auth across IdPs (Auth0, authentik, Entra, Keycloak, Okta)", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/mcp/mcp-authn.md", + "gap": "8" + }, + { + "area": "rate-limiting", + "feature": "Local in-memory and remote (Envoy Rate Limit gRPC) limits keyed by CEL expressions; two-phase token enforcement when tokenize:true", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/configuration/resiliency/rate-limits.md", + "gap": "3" + }, + { + "area": "protocol", + "feature": "Multi-family LLM translation in both directions (Chat/Messages/Responses/Gemini/Realtime), ~22 native providers incl. Bedrock, Vertex, GitHub Copilot", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/about.md", + "gap": "6" + }, + { + "area": "protocol", + "feature": "anthropicTokenCount route type (/v1/messages/count_tokens)", + "sourceUrl": "https://docs.solo.io/agentgateway/standalone/latest/documentation/llm/api-types/token-count", + "gap": "6" + }, + { + "area": "observability", + "feature": "OTel GenAI semconv metrics (token usage, TTFT, time-per-output-token) on :15020 plus OTLP traces with cost attributes", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/observability.md", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "Persistent request-log database (SQLite or PostgreSQL) with timing, tokens, realized USD cost, identity, trace ids", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/observability/access-logs/database.md", + "gap": "11;12" + }, + { + "area": "auth", + "feature": "Frontend JWT/OIDC/basic/external authz; backend API keys, JWT signing, OAuth token exchange, cloud credentials via CEL", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/configuration/security/oidc.md", + "gap": "new:caller-sso-rbac;4" + }, + { + "area": "auth", + "feature": "In-process AWS SigV4 with expiry-driven AssumeRole caches and per-request CEL session tags", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/auth/aws.rs", + "gap": "4" + }, + { + "area": "auth", + "feature": "GCP ADC id/access tokens incl. impersonated and GDCH service accounts, per-audience caches", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/auth/gcp.rs", + "gap": "4" + }, + { + "area": "auth", + "feature": "Azure auth: client secret, managed identity, workload identity, environment auto-detection with per-backend caches", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/auth/azure.rs", + "gap": "4" + }, + { + "area": "auth", + "feature": "OAuth client-credential token caching with expiry-aware TTL and safety margin", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/auth/oauth/cache.rs", + "gap": "4" + }, + { + "area": "auth", + "feature": "Per-request JWT signing auth; zero-cache Copilot token loading", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/auth/jwt_sign.rs", + "gap": "4" + }, + { + "area": "auth", + "feature": "Upstream TLS with inline client certificates and SPIFFE SVID rotation", + "sourceUrl": "https://github.com/agentgateway/agentgateway/blob/main/crates/agentgateway/src/http/backendtls.rs", + "gap": "4" + }, + { + "area": "tooling", + "feature": "agctl CLI (experimental): live request trace, config inspection, pprof, catalog import, standalone admin", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/operations/agctl.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Built-in admin UI: key management, cost dashboard, CEL playground, MCP Tool Playground", + "sourceUrl": "https://agentgateway.dev/blog/2026-06-17-agentgateway-v1.3.0", + "gap": "19" + }, + { + "area": "ops", + "feature": "Config hot reload via debounced file watch with validate-then-swap and startup-only exception list; xDS control plane; Helm", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/setup/update", + "gap": "16" + }, + { + "area": "other", + "feature": "A2A agent-to-agent routing with auto-detected agent sessions (x-claude-code-session-id etc.)", + "sourceUrl": "https://github.com/agentgateway/agentgateway", + "gap": "context" + }, + { + "area": "other", + "feature": "Egress control via HTTP CONNECT forward proxy modes (route/deny/dynamic CA)", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/configuration/egress-proxy.md", + "gap": "context" + }, + { + "area": "other", + "feature": "Documented kill-switch controls: revoke agent JWT, zero rate limit for a class, scale gateway to zero", + "sourceUrl": "https://agentgateway.dev/blog/2026-02-21-kill-switch/", + "gap": "context" + }, + { + "area": "protocol", + "feature": "CEL request transformations before/after provider conversion (prompt/field enrichment)", + "sourceUrl": "https://agentgateway.dev/docs/standalone/latest/documentation/llm/transformations.md", + "gap": "context" + } + ] + }, + { + "name": "AISIX (api7/aisix)", + "category": "open-source AI gateway (OSS data plane of AISIX Enterprise/Cloud)", + "language": "Rust", + "status": "active (GA v1.0.0 2026-09-04; v1.8.0 2026-10-29; last push 2026-11-04)", + "url": "https://github.com/api7/aisix", + "capabilities": [ + { + "area": "routing", + "feature": "Six routing strategies (failover, smooth-weighted round-robin, consistent hash, least cost/latency/busy) with priority tiers, retry knobs, cooldowns, background health ejection", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "1;13" + }, + { + "area": "rate-limiting", + "feature": "Per-key/per-model rps/rpm/rph/rpd + tpm/tpd + in-flight concurrency limits with 429 pre-call, Retry-After and X-RateLimit-* headers; Redis (single/cluster/sentinel) shared counters", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/config.example.yaml", + "gap": "3" + }, + { + "area": "auth", + "feature": "Caller API keys stored as SHA-256 hashes with RFC 3339 expiry, disable/enable, glob model allowlists, two-key rotation path", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "9" + }, + { + "area": "auth", + "feature": "OIDC/JWT caller authentication with JWKS verification, claim-to-key mapping, coalesced JWKS refresh with stale-while-refresh", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "new:caller-sso-rbac" + }, + { + "area": "guardrails", + "feature": "Layered guardrails at input/output hooks: keyword/regex/semantic/PII built-ins plus Presidio, Lakera, OpenAI Moderation, Bedrock, Azure, Alibaba, custom scripts; monitor mode", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "7" + }, + { + "area": "caching", + "feature": "Response cache: exact-match on normalized key (temperature compared at 0.001) with memory/Redis backends, TTL, purge generations, Cache-Control bypass; semantic layer on Redis 8+ search", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/config.example.yaml", + "gap": "10;14" + }, + { + "area": "caching", + "feature": "Automatic Anthropic prompt-cache breakpoint insertion", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "14" + }, + { + "area": "mcp", + "feature": "MCP gateway: aggregated /mcp and /mcp/{server} with gateway-held credentials, OpenAPI-to-MCP exposure, stateless 2026-07-28 revision, MCP conformance suite as merge-blocking CI", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "8" + }, + { + "area": "protocol", + "feature": "A2A agent gateway at /a2a/{agent} (JSON-RPC + SSE relaying, card URL rewriting, shared caller-key governance)", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Broad endpoint families (chat, completions, responses, messages+count_tokens, embeddings, rerank, systemone/decisions, images, async video, audio, files/batches/fine-tuning, realtime WS, passthrough)", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "6" + }, + { + "area": "auth", + "feature": "Five native upstream adapter families: openai, anthropic, bedrock (SigV4), vertex (OAuth2), azure-openai (api-key or Entra)", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "4" + }, + { + "area": "observability", + "feature": "Prometheus metrics with request- vs attempt-level counters (successful_fallbacks, TTFT/e2e histograms) on a dedicated listener", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "12;17" + }, + { + "area": "observability", + "feature": "Per-attempt usage events sharing a request_id with attempt_index, cache_status, TTFT, token splits; OTLP GenAI / Datadog / SLS / S3 exporters with bounded queue", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "11;12" + }, + { + "area": "ops", + "feature": "Ops hardening: thread-per-core with SO_REUSEPORT, multi-listener TLS + PROXY protocol v1/v2, real_ip CIDRs, outbound SSRF protection, upstream pool tuning incl. mTLS, graceful drain, <10 MiB idle memory", + "sourceUrl": "https://github.com/api7/aisix/releases", + "gap": "16" + }, + { + "area": "routing", + "feature": "Ensemble models (panel + judge synthesizer) and semantic (embedding) routing incl. jev classifier mode", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "13" + }, + { + "area": "ops", + "feature": "Lock-free hot-path config reads via ArcSwap snapshots with deferred reclamation of replaced snapshots", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/crates/aisix-core/src/snapshot.rs", + "gap": "context" + }, + { + "area": "cost", + "feature": "Budgets and spend caps (org/env/caller/provider/team/member/tag scopes, 429 budget_exceeded, gateway-side decisions since v1.8.0) — Enterprise/Cloud only", + "sourceUrl": "https://github.com/api7/aisix/releases", + "gap": "2" + }, + { + "area": "tooling", + "feature": "Declarative JSON-Schema resources.yaml with offline aisix validate, atomic SIGHUP/etcd hot reload, read-only admin listener (OpenAPI 3 + Scalar UI), 183-scenario e2e suite", + "sourceUrl": "https://raw.githubusercontent.com/api7/aisix/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "Anthropic platform", + "category": "provider API", + "language": "n/a (hosted API)", + "status": "active", + "url": "https://platform.claude.com/docs/en/api/models-list", + "capabilities": [ + { + "area": "lifecycle", + "feature": "Tri-state model lifecycle (active/deprecated/retired) with deprecated_at/retires_at; retired stays listed while inference fails", + "sourceUrl": "https://platform.claude.com/docs/en/api/models-list", + "gap": "18" + }, + { + "area": "caching", + "feature": "cache_control prompt caching: up to 4 breakpoints, 5m (1.25x write) and 1h (2x write) TTLs, 0.1x reads, max_tokens:0 prewarm", + "sourceUrl": "https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching", + "gap": "14" + }, + { + "area": "protocol", + "feature": "context_management server-side edits (clear_tool_uses, clear_thinking, compact) with count_tokens preflight of post-truncation size", + "sourceUrl": "https://platform.claude.com/docs/en/build-with-claude/context-editing", + "gap": "context" + } + ] + }, + { + "name": "Antirion (eliasbiondo)", + "category": "open-source gateway (Rust, embedded console)", + "language": "Rust", + "status": "active but 18 days old (v0.1.1, 2026-10-18; last push 2026-10-31); single maintainer, pre-1.0", + "url": "https://github.com/eliasbiondo/antirion", + "capabilities": [ + { + "area": "ops", + "feature": "Thread-per-core data plane with least-loaded accept lanes and jemalloc/mimalloc", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-server/src/threads.rs", + "gap": "context" + }, + { + "area": "observability", + "feature": "Reproducible open-loop benchmark harness with published raw results vs Bifrost/LiteLLM (no coordinated omission, interleaved reps, validity flags)", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/docs/benchmarks.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Route trees with fallback chains, weighted bandits, least_cost/latency/busy, conditional and classifier pickers, session/prefix affinity, shadow traffic", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-resilience/src/engine/hedge.rs", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Hedged request races: extra attempts every delay up to max_extra while none has answered; losers cancelled and settled with health verdicts", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-resilience/src/engine/hedge.rs", + "gap": "new:hedged-requests" + }, + { + "area": "rate-limiting", + "feature": "Reservation-based limits engine (GCRA/sliding window) with per-attempt holds settled before last byte, degraded store mode with queued-settlement replay, lease-expiry release", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-limits/src/engine.rs", + "gap": "2;3" + }, + { + "area": "cost", + "feature": "Budgets with forecasts, overage grants and approval requests, event feed, refused/downgraded counts, owner notifications — off the request path", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-runtime/src/budgets/mod.rs", + "gap": "2" + }, + { + "area": "guardrails", + "feature": "PII/secrets/regex/moderation/LLM-judge guardrails with format-preserving redaction, block/redact/flag/divert actions, zero-data-retention mode", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-guardrails/src/lib.rs", + "gap": "7" + }, + { + "area": "caching", + "feature": "Exact + semantic response cache with single-flight, epochs, memory slab and Valkey (Lua) backends, console-manageable rules", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-cache/src/semantic.rs", + "gap": "10" + }, + { + "area": "auth", + "feature": "Enterprise identity in-tree: virtual keys, JWT, BYOK, full SAML 2.0 (XML dsig/c14n), OIDC SSO, SCIM, TOTP/WebAuthn, RBAC roles/permissions, audit log with change approval", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-identity/src/saml/mod.rs", + "gap": "new:caller-sso-rbac;9" + }, + { + "area": "mcp", + "feature": "MCP gateway + A2A proxy: virtual servers across protocol eras, encrypted session tokens, tool approval, OpenAPI-as-tools, per-gateway approval choice", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-mcp/src/lib.rs", + "gap": "8" + }, + { + "area": "tooling", + "feature": "WASI 0.2 + webhook plugin hooks, sha256-pinned, per-call stores capped (64 MiB / 5 ms), no files/env/network; OCI-ref modules", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-ext/src/wasm/mod.rs", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTLP metrics/traces with OpenTelemetry GenAI names plus alerting engine with SLOs and Slack/PagerDuty/email/webhook sinks", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-obs/src/metrics/otlp.rs", + "gap": "11;12" + }, + { + "area": "tooling", + "feature": "Embedded Next.js admin console + OpenAPI 3.1 Admin API (ETag/If-Match, Idempotency-Key, ?wait=, config generations with diff/rollback, REDACTED secrets)", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/console/README.md", + "gap": "19" + }, + { + "area": "ops", + "feature": "Zero-config first start, cluster mode on PostgreSQL + Valkey, Helm chart, cosign/SLSA/SBOM signed releases", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/docs/getting-started.md", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Connect-time SSRF validation on egress (URL rule catalogue + per-resolved-address rules, trust classes)", + "sourceUrl": "https://github.com/eliasbiondo/antirion/blob/main/crates/antirion-egress/src/ssrf.rs", + "gap": "context" + }, + { + "area": "tooling", + "feature": "13 cargo-fuzz targets wired into CI, cargo-deny, reproducible builds, JSON-Schema config with antirion schema", + "sourceUrl": "https://github.com/eliasbiondo/antirion/tree/main/fuzz/fuzz_targets", + "gap": "context" + } + ] + }, + { + "name": "Apache APISIX", + "category": "open-source API gateway with AI plugins", + "language": "Lua (OpenResty)", + "status": "active", + "url": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", + "capabilities": [ + { + "area": "protocol", + "feature": "ai-proxy Lua plugin: bidirectional Anthropic-OpenAI translation incl. SSE, Bedrock Converse SigV4, per-protocol deep-merge overrides", + "sourceUrl": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", + "gap": "context" + }, + { + "area": "ops", + "feature": "Admin API over etcd: dynamic route/upstream/consumer CRUD with ttl expiry and plugin hot reload applied to running nodes", + "sourceUrl": "https://apisix.apache.org/docs/apisix/admin-api/", + "gap": "16" + }, + { + "area": "extensibility", + "feature": "General-purpose plugin chassis (kafka-logger/http-logger with LLM labels, proxy-rewrite) plus external plugin runners and Wasm", + "sourceUrl": "https://apisix.apache.org/docs/apisix/plugins/ai-proxy/", + "gap": "context" + } + ] + }, + { + "name": "AWS Bedrock AgentCore Gateway", + "category": "cloud gateway (managed)", + "language": "n/a (managed service)", + "status": "active (GA per AWS docs, October 2026)", + "url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html", + "capabilities": [ + { + "area": "routing", + "feature": "Model-based inference routing across Bedrock/OpenAI/Anthropic with zero-config connectors and target-qualified IDs", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-target-inference-connector.html", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Fallback chains (1-2 ordered targets, status-code classes + CONNECTION_ERROR/FIRST_BYTE_TIMEOUT/UNROUTABLE triggers, firstByteTimeout 50-900,000 ms) with default-on circuit breaker; no fallback after stream start", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules-fallback.html", + "gap": "1" + }, + { + "area": "routing", + "feature": "Up to 20 priority-ordered gateway rules with IAM/path/model conditions, weighted config-bundle and route actions (2-way splits), OTel baggage propagation of resolved bundles", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules.html", + "gap": "1;13" + }, + { + "area": "auth", + "feature": "Inbound auth: IAM SigV4, custom JWT authorizers (OIDC discovery, claim matching), offloaded or none; RFC 6750 challenges + RFC 9728 protected-resource metadata", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-inbound-auth.html", + "gap": "4" + }, + { + "area": "auth", + "feature": "Per-target outbound credential vaulting: token-vault ARNs, API key injection, OAuth2 client-credentials/authorization-code/token-exchange, JWT passthrough, caller IAM via Federated Access Service", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-outbound-auth.html", + "gap": "4" + }, + { + "area": "rate-limiting", + "feature": "Up to 50 dimension-keyed rate limits (target/tool/model/JWT claims/IAM principal) over requests/s, tokens/min (pre-estimate + actuals), and concurrent connections; fail-open, not a security boundary", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rate-limits.html", + "gap": "3" + }, + { + "area": "guardrails", + "feature": "Deny-by-default Cedar policy engine on MCP tools/call with full request context; temporal session-aware policies via x-amzn-bedrock-agentcore-policy-session-id", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/use-gateway-with-policy.html", + "gap": "7;8" + }, + { + "area": "guardrails", + "feature": "Guardrail conditions in policies backed by Bedrock Guardrails (ContentFilter, PromptAttack, SensitiveInformation confidence scores)", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-guardrails-in-policies.html", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "REQUEST/RESPONSE Lambda interceptors (one each) for transformation, custom authorization, model-field rewriting; idempotency required", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-interceptors.html", + "gap": "7" + }, + { + "area": "observability", + "feature": "CloudWatch metrics (latency to first token p50/p90/p99, throttles, target execution time) plus vended logs with bodies and OTel-compatible spans", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-gateway-metrics.html", + "gap": "11;12;17" + }, + { + "area": "mcp", + "feature": "Managed MCP server endpoint aggregating targets: optional stateful sessions with elicitation and sampling, SSE streaming with progress notifications", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-features.html", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Cross-dialect translation (Bedrock/OpenAI/Anthropic) plus native InvokeModel wire format; reserved /mcp, /a2a, /responses, /converse, /.well-known prefixes", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-targets-inference.html", + "gap": "6" + }, + { + "area": "mcp", + "feature": "Semantic tool search over the catalog (x_amz_bedrock_agentcore_search returns ten most relevant tools)", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-features.html", + "gap": "8" + }, + { + "area": "ops", + "feature": "Private networking via VPC Lattice private endpoints for targets and VPC-hosted identity providers", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-vpc-egress.html", + "gap": "context" + }, + { + "area": "cost", + "feature": "Per-attempt invocation billing (failed primary attempts billed) and enforcement-grade token accounting; no USD cost ledger", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-rules-fallback.html", + "gap": "2" + }, + { + "area": "ops", + "feature": "Customer-managed KMS, debug messages toggle, resource tagging, 1-click SaaS tool integrations", + "sourceUrl": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway-features.html", + "gap": "context" + } + ] + }, + { + "name": "Azure AI Foundry (OpenAI models)", + "category": "provider API", + "language": "n/a (managed service)", + "status": "active", + "url": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirements", + "capabilities": [ + { + "area": "lifecycle", + "feature": "Five-stage model lifecycle (Preview/GA/Legacy/Deprecated/Retired; 410 Gone at retirement) with programmatic dates, per-deployment versionUpgradeOption auto-upgrades, and >=60/30-day notices", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirements", + "gap": "18" + } + ] + }, + { + "name": "Azure API Management (GenAI / AI Gateway)", + "category": "cloud gateway", + "language": "n/a (managed service)", + "status": "active (AI Gateway tier and unified model API in preview, 2026)", + "url": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "capabilities": [ + { + "area": "protocol", + "feature": "Unified model API (preview): one OpenAI Chat Completions endpoint + /models over OpenAI/Anthropic backends with auto-translation and model failover", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/unified-model-api", + "gap": "context" + }, + { + "area": "ops", + "feature": "AI Gateway tier (preview): managed runtime with policy cards (content safety, IP filter, token/request rate limit), OTel GenAI telemetry, gateway-held backend credentials", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/ai-gateway-overview", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "llm-token-limit policy: tokens-per-minute (429) plus long-period token quotas over hourly-to-yearly fixed UTC windows (403), any counter-key expression, optional estimate-prompt-tokens pre-rejection", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/llm-token-limit-policy", + "gap": "3" + }, + { + "area": "routing", + "feature": "Backend load balancer (round-robin/weighted/priority/session-aware) with circuit breaker honoring Retry-After", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "1;13" + }, + { + "area": "caching", + "feature": "Semantic caching of completions via embeddings in Azure Managed Redis or any RediSearch-compatible cache", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "10" + }, + { + "area": "observability", + "feature": "Token metrics with custom dimensions and prompt/completion logging to Azure Monitor with a token-consumption dashboard", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "llm-content-safety prompt moderation policy backed by Azure AI Content Safety", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "7" + }, + { + "area": "mcp", + "feature": "MCP server exposure: REST APIs exported as MCP servers (operations become tools) or passthrough of existing MCP servers under Streamable HTTP, with policies applied", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/mcp-server-overview", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Multi-schema LLM API ingestion (OpenAI Chat/Responses, Anthropic Messages, Vertex AI) plus A2A agent API management", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "6" + }, + { + "area": "auth", + "feature": "Managed-identity backend auth and credential-manager OAuth for clients; per-application runtime access keys on the AI Gateway tier", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "context" + }, + { + "area": "tooling", + "feature": "AI gateway embedded in Microsoft Foundry (preview): token quotas on deployments, agent/tool registry, MCP tool registration", + "sourceUrl": "https://learn.microsoft.com/en-us/azure/api-management/genai-gateway-capabilities", + "gap": "context" + } + ] + }, + { + "name": "Bifrost (maximhq)", + "category": "open-source gateway", + "language": "Go", + "status": "active (transports v2.3.2, 2026-10-20; enterprise tier gates some features)", + "url": "https://github.com/maximhq/bifrost", + "capabilities": [ + { + "area": "auth", + "feature": "sk-bf-* virtual keys: expiry, is_active kill switch, deny-by-default provider/model scoping (RE2 regex), key_ids pinning to stored upstream keys", + "sourceUrl": "https://docs.getbifrost.ai/features/governance/virtual-keys", + "gap": "9" + }, + { + "area": "auth", + "feature": "Dashboard/admin authentication with setup token, route whitelisting, env/vault credential references", + "sourceUrl": "https://docs.getbifrost.ai/quickstart/gateway/setting-up-auth.md", + "gap": "9" + }, + { + "area": "cost", + "feature": "Hierarchical USD budgets (customer/team/virtual-key/provider-config) with rolling 1m-1Y or UTC-calendar/fiscal-quarter resets, overrides, cumulative enforcement (402)", + "sourceUrl": "https://docs.getbifrost.ai/features/governance/budget-and-limits", + "gap": "2" + }, + { + "area": "rate-limiting", + "feature": "Request and token limits at provider-config/virtual-key/team/customer scopes with typed 401/402/403/429 errors", + "sourceUrl": "https://docs.getbifrost.ai/features/governance/budget-and-limits", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "Per-model limits (dollar budgets and request/token rate limits on model_name globs) at global and virtual-key scope", + "sourceUrl": "https://docs.getbifrost.ai/features/governance/model-limits", + "gap": "3" + }, + { + "area": "routing", + "feature": "Classified retries: failure_class taxonomy (transient/rate_limit/credential/quota/model_access/caller_fault/timeout) driving retry-same-key vs rotate-key with jittered backoff; rotations granted back", + "sourceUrl": "https://docs.getbifrost.ai/features/retries-and-fallbacks", + "gap": "1" + }, + { + "area": "routing", + "feature": "Per-request cross-provider fallback chains with stacking retry budgets; Azure in-stream errors handled via bounded startup-event buffering (64 chunks/256 KiB)", + "sourceUrl": "https://docs.getbifrost.ai/features/fallbacks", + "gap": "1" + }, + { + "area": "routing", + "feature": "Weighted API-key pools per provider with per-key model scoping and request-level key pinning", + "sourceUrl": "https://docs.getbifrost.ai/features/keys-management", + "gap": "5" + }, + { + "area": "routing", + "feature": "Weighted provider selection with auto-generated highest-weight-first fallback chain", + "sourceUrl": "https://docs.getbifrost.ai/features/governance/routing.md", + "gap": "13" + }, + { + "area": "routing", + "feature": "CEL-expression routing rules with scope precedence (VK/Team/Customer/Global), A/B splits, TTFT cutoffs, chained re-evaluation", + "sourceUrl": "https://docs.getbifrost.ai/providers/routing-rules.md", + "gap": "13" + }, + { + "area": "routing", + "feature": "Session affinity: sticky provider+key per session id (auto-adopts Claude Code/Codex/OpenCode headers), success-only bindings, TTL 1h", + "sourceUrl": "https://docs.getbifrost.ai/providers/session-affinity.md", + "gap": "14" + }, + { + "area": "routing", + "feature": "Complexity router: LLM decision model + semantic phrase matching classifying requests into tiers for CEL rules; session tier retention protects cache reuse", + "sourceUrl": "https://docs.getbifrost.ai/features/complexity-router.md", + "gap": "13" + }, + { + "area": "caching", + "feature": "Semantic + direct-hash response cache over Redis/Valkey/Weaviate/Qdrant/Pinecone with per-entry TTL, async writes, identity scoping, streaming replay", + "sourceUrl": "https://docs.getbifrost.ai/features/semantic-caching", + "gap": "10" + }, + { + "area": "observability", + "feature": "Prometheus telemetry incl. bifrost_overhead_latency_microseconds, key-health gauge, rotation counters, TTFT/inter-token histograms", + "sourceUrl": "https://docs.getbifrost.ai/features/telemetry.md", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "llm.call OTel span per request plus per-attempt spans under GenAI conventions; OTLP HTTP/gRPC export", + "sourceUrl": "https://docs.getbifrost.ai/features/otel.md", + "gap": "11" + }, + { + "area": "observability", + "feature": "Built-in logging plugin (SQLite/Postgres) with full bodies, per-attempt attempt_trail, web dashboard with live streaming and token/cost analytics", + "sourceUrl": "https://docs.getbifrost.ai/features/observability/default", + "gap": "11;12" + }, + { + "area": "mcp", + "feature": "MCP gateway: external servers over STDIO/HTTP/SSE re-exposed at /mcp with virtual-key-scoped tool filtering, explicit execution endpoint, sandboxed Python code mode", + "sourceUrl": "https://docs.getbifrost.ai/mcp/overview.md", + "gap": "8" + }, + { + "area": "guardrails", + "feature": "Enterprise guardrails: CEL rules, Gitleaks secrets detection, LLM-as-judge, ~12 external integrations; 446 on block", + "sourceUrl": "https://docs.getbifrost.ai/enterprise/guardrails", + "gap": "7" + }, + { + "area": "tooling", + "feature": "Go shared-object plugin system with hot reload and full lifecycle hooks (pre-auth, per-attempt LLM/MCP, stream chunk); WASM deprecated", + "sourceUrl": "https://docs.getbifrost.ai/plugins/getting-started.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "20+/23+ providers behind one OpenAI-compatible API; request types span chat, Responses (incl. WebSocket), embeddings, speech, transcription, image/video generation", + "sourceUrl": "https://github.com/maximhq/bifrost", + "gap": "6" + } + ] + }, + { + "name": "BricksLLM", + "category": "open-source gateway", + "language": "Go", + "status": "unmaintained since 2025-01-05 (v1.45.2)", + "url": "https://github.com/bricks-cloud/BricksLLM", + "capabilities": [ + { + "area": "auth", + "feature": "Virtual API keys with per-key USD cost caps and rate limits (rateLimitOverTime, costLimitInUsd) — the 2023 pattern pioneer", + "sourceUrl": "https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md", + "gap": "2;9" + }, + { + "area": "routing", + "feature": "Caching, request retries, and provider failover (OpenAI-to-Azure OpenAI cookbook)", + "sourceUrl": "https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md", + "gap": "1" + }, + { + "area": "guardrails", + "feature": "PII detection and masking backed by AWS Comprehend; model/endpoint access control per key", + "sourceUrl": "https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md", + "gap": "7" + }, + { + "area": "observability", + "feature": "Cost/request analytics, Datadog integration, logging with privacy control", + "sourceUrl": "https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md", + "gap": "context" + }, + { + "area": "ops", + "feature": "Split-plane architecture: admin API :8001 (ADMIN_PASS) and OpenAI-compatible proxy :8002, Swagger for both", + "sourceUrl": "https://raw.githubusercontent.com/bricks-cloud/BricksLLM/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "Claude Code Router (musistudio)", + "category": "local agent-gateway / control plane (desktop + CLI)", + "language": "TypeScript (Node/Electron)", + "status": "active (v3.1.3, 2026-10-09; last commit 2026-10-10; 950 open issues / 237 PRs backlog)", + "url": "https://github.com/musistudio/claude-code-router", + "capabilities": [ + { + "area": "routing", + "feature": "Ordered conditional routing rules with request rewriting (header/body dot-path conditions, 9 operators, set/delete/append rewrites, Claude Code detection, subagent auto-routing)", + "sourceUrl": "https://ccrdesk.top/routing/", + "gap": "13" + }, + { + "area": "routing", + "feature": "Sandboxed Node.js route scripts (frozen Worker input, size/time/rewrite caps, circuit breaking after 3 failures in 60s)", + "sourceUrl": "https://github.com/musistudio/claude-code-router/tree/main/packages/core/src/routing", + "gap": "context" + }, + { + "area": "routing", + "feature": "Retry policy + ordered model-chain fallback (retry on 408/409/429/5xx + network errors; Retry-After honored; x-ccr-fallback-model header)", + "sourceUrl": "https://github.com/musistudio/claude-code-router/releases", + "gap": "1" + }, + { + "area": "auth", + "feature": "Upstream credential pools with per-key priority, weight, quota limits, rotation on refresh-401, macOS keychain lookup, balance connectors", + "sourceUrl": "https://ccrdesk.top/guides/provider/", + "gap": "5" + }, + { + "area": "auth", + "feature": "Client access keys with expiration, instant revoke, per-key local quotas across requests/tokens/images x min/hour/day; constant-time comparison", + "sourceUrl": "https://ccrdesk.top/configuration/api-keys/", + "gap": "9;3" + }, + { + "area": "observability", + "feature": "Request logs with resolved route, credential, latency, tokens, cost, full bodies, and agent execution-chain reconstruction", + "sourceUrl": "https://ccrdesk.top/guides/observability/", + "gap": "context" + }, + { + "area": "observability", + "feature": "Usage dashboard (time ranges, cache ratio, heatmap, distributions) plus exportable PNG share cards", + "sourceUrl": "https://ccrdesk.top/configuration/overview/", + "gap": "19" + }, + { + "area": "protocol", + "feature": "Five client-side protocols with any-to-any translation (OpenAI Chat/Responses, Anthropic Messages, Gemini generateContent/interactions) plus auto endpoint detection and presets", + "sourceUrl": "https://github.com/musistudio/claude-code-router/blob/main/packages/core/src/routing/protocol-endpoints.ts", + "gap": "6" + }, + { + "area": "mcp", + "feature": "ToolHub MCP aggregation with lazy tool resolution (tool_hub.resolve retrieval + tool_hub.invoke; stdio/streamable-http/SSE; browser automation; session id forwarded as prompt_cache_key)", + "sourceUrl": "https://ccrdesk.top/toolhub/", + "gap": "8;14" + }, + { + "area": "other", + "feature": "Fusion composed virtual models bundling base + capability models (vision/search/image/video/MCP tools) as ordinary routing entries", + "sourceUrl": "https://ccrdesk.top/fusion/", + "gap": "context" + }, + { + "area": "ops", + "feature": "Desktop MITM proxy mode, /health, admin token, CDN-refreshed model catalog, AgentClaw chat-relay channel integrations", + "sourceUrl": "https://ccrdesk.top/configuration/server/", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Provider presets, deeplink/manifest import, and a connectivity check that sends real requests before saving", + "sourceUrl": "https://ccrdesk.top/guides/provider/", + "gap": "context" + } + ] + }, + { + "name": "claude-code-proxy (raine)", + "category": "local Anthropic-compatible proxy over subscription upstreams", + "language": "Rust", + "status": "active (v0.1.48, 2026-11-01; last commit 2026-10-31)", + "url": "https://github.com/raine/claude-code-proxy", + "capabilities": [ + { + "area": "routing", + "feature": "Request-path retries: 429/500/502/503/504 with 2s-to-30s exponential backoff honoring Retry-After; Codex empty-completion retries; subscription-limit fail-fast", + "sourceUrl": "https://raw.githubusercontent.com/raine/claude-code-proxy/main/src/retry.rs", + "gap": "1" + }, + { + "area": "auth", + "feature": "Full OAuth credential lifecycle: browser PKCE, device-code, browser-polling flows; per-request token refresh; macOS Keychain or permissioned auth.json", + "sourceUrl": "https://claude-code-proxy.raine.dev/reference/command-reference/", + "gap": "new:oauth-credential-lifecycle" + }, + { + "area": "routing", + "feature": "Model-ID-driven routing on a single listener: service-tier suffixes (-fast/-ultrafast), cursor: prefixes, Anthropic-style aliases via aliasProvider, 400 with supported catalog on unknown IDs", + "sourceUrl": "https://claude-code-proxy.raine.dev/using/models-and-routing/", + "gap": "13" + }, + { + "area": "observability", + "feature": "Embedded monitor TUI (attachable over SSH, auto-reconnect, 250 ms polling) plus loopback-only /monitor snapshot: sessions, lifecycles, tokens, throughput", + "sourceUrl": "https://claude-code-proxy.raine.dev/using/monitor-tui/", + "gap": "context" + }, + { + "area": "cost", + "feature": "Subscription-quota usage and reasoning-token surfacing in usage reports (subscription spend, not USD)", + "sourceUrl": "https://raw.githubusercontent.com/raine/claude-code-proxy/main/CHANGELOG.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "Opt-in traffic capture grouped by session, failed-response error captures, redacting rotating JSONL logs", + "sourceUrl": "https://claude-code-proxy.raine.dev/reference/files-and-storage/", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Multi-protocol inbound (messages, opt-in chat/completions and responses) plus five upstream translators incl. WebSocket transport with HTTP fallback", + "sourceUrl": "https://claude-code-proxy.raine.dev/reference/http-api/", + "gap": "6" + }, + { + "area": "routing", + "feature": "In-memory session-state continuity keyed by x-claude-code-session-id (Cursor conversation IDs, optional Codex previous_response_id/server-compaction)", + "sourceUrl": "https://claude-code-proxy.raine.dev/how-it-works/", + "gap": "14" + }, + { + "area": "protocol", + "feature": "Per-provider offline token estimators for count_tokens (o200k_base plus image/encrypted-reasoning/framing estimates)", + "sourceUrl": "https://claude-code-proxy.raine.dev/how-it-works/", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Opt-in image generation/editing endpoints and a changelog-era transcription route (current status unverified)", + "sourceUrl": "https://claude-code-proxy.raine.dev/reference/http-api/", + "gap": "context" + }, + { + "area": "ops", + "feature": "Homebrew/install-script/Nix distribution, 64 MiB request bodies, graceful SIGTERM shutdown", + "sourceUrl": "https://raw.githubusercontent.com/raine/claude-code-proxy/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "CLIProxyAPI (router-for-me)", + "category": "subscription-to-API proxy + embeddable SDK", + "language": "Go", + "status": "extremely active (v8.0.54, 2026-11-04); core use case ToS-gray — pattern reference only", + "url": "https://github.com/router-for-me/CLIProxyAPI", + "capabilities": [ + { + "area": "auth", + "feature": "Per-provider OAuth login flows: browser PKCE, device-code, service-account import, SIWC binding, headless management-API auth URLs", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/cmd/server/main.go", + "gap": "new:oauth-credential-lifecycle" + }, + { + "area": "routing", + "feature": "Multi-account credential pools with three rotation strategies (round-robin with identity-based rotation, smooth weighted, fill-first) plus priority tiers and prefix pinning", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/sdk/cliproxy/auth/selector.go", + "gap": "5" + }, + { + "area": "routing", + "feature": "Credential-level retry rounds with per-round caps, cooldown backoff levels, model-scoped cooling, and custom request-scoped error classification", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml", + "gap": "1;5" + }, + { + "area": "routing", + "feature": "Session-affinity sticky routing for prompt/KV-cache reuse (session headers, prompt_cache_key, Responses conversation IDs, first-message hash; subagents inherit credentials)", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml", + "gap": "14" + }, + { + "area": "auth", + "feature": "Passive upstream quota observation from response headers (Retry-After watermarks, limit flags) kept separate from cooldown state; live quota fetch/reset API", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/sdk/cliproxy/auth/quota_signals.go", + "gap": "5" + }, + { + "area": "protocol", + "feature": "Very wide surface: chat/completions, responses (HTTP + WebSocket + compact), messages + count_tokens, images, video, TTS, /v1/live realtime incl. WebRTC, Gemini native, Codex aliases", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/internal/api/server_routes.go", + "gap": "6" + }, + { + "area": "tooling", + "feature": "v8 Management API: comment-preserving CRUD over the whole YAML tree, auth-file lifecycle, ~80 runtime toggle routes, auto-downloaded web panel", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/docs/management-api-v8.md", + "gap": "19;16" + }, + { + "area": "ops", + "feature": "Cluster mode with distributed credential-lease control plane (JWT/mTLS bootstrap, heartbeats, fences, reclaim grace)", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/docker-compose.cluster.yml", + "gap": "context" + }, + { + "area": "auth", + "feature": "Pluggable credential token stores: file, Postgres, Git repo, S3/MinIO", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/cmd/server/main.go", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Client fingerprint cloaking / CLI impersonation (pinned CLI wire fingerprints, uTLS, CCH signing, cloak modes) — ToS-gray pattern reference", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Payload rewrite rules engine: set-if-missing / always-overwrite / strip by JSON path with model/protocol/header/payload matchers", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml", + "gap": "context" + }, + { + "area": "routing", + "feature": "Stream-bootstrap buffering: hold handshake/heartbeat frames (48-frame/1 MiB budget) so HTTP-200-embedded rejections can fail over before headers commit", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/config.example.yaml", + "gap": "1" + }, + { + "area": "tooling", + "feature": "Embeddable Go SDK plus c-shared dynamic plugin system with a plugin store; TLS listener, zstd/gzip compression, mDNS discovery, pprof, TUI, safe-mode", + "sourceUrl": "https://github.com/router-for-me/CLIProxyAPI/blob/main/docs/sdk-usage.md", + "gap": "context" + } + ] + }, + { + "name": "Cloudflare AI Gateway", + "category": "cloud gateway (managed)", + "language": "n/a (managed service)", + "status": "active (GA; unified REST API and Workers Logs migration, 2026)", + "url": "https://developers.cloudflare.com/ai-gateway/", + "capabilities": [ + { + "area": "protocol", + "feature": "Unified multi-provider endpoint (model: provider/model) across 24+ provider integrations plus custom providers; REST API serving OpenAI chat/completions, Responses, and Anthropic messages schemas", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/usage/rest-api/", + "gap": "6" + }, + { + "area": "caching", + "feature": "Exact-match response cache keyed on SHA-256 of provider+endpoint+model+auth+body; per-request TTL/key/skip headers; HIT/MISS status header", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/caching/", + "gap": "10" + }, + { + "area": "routing", + "feature": "Automatic retries (max 5 attempts, per-request backoff/delay controls, first-byte timeout)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/configuration/request-handling/", + "gap": "1" + }, + { + "area": "routing", + "feature": "Ordered model fallbacks via request body (cf-aig-step reports the serving entry)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/configuration/fallbacks/", + "gap": "1" + }, + { + "area": "routing", + "feature": "Dynamic routing: versioned JSON routes with conditionals, percentage splits, in-route rate/budget limits and fallback", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/", + "gap": "1" + }, + { + "area": "routing", + "feature": "Auto Router (cloudflare/auto): content-aware model selection with session affinity, allowed-model/provider restrictions, routing-reason headers", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/auto-router/", + "gap": "13" + }, + { + "area": "auth", + "feature": "Authenticated gateways with BYOK Secrets-Store aliases (multiple keys per provider, per-request selection, rotation by edit)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/configuration/bring-your-own-keys/", + "gap": "context" + }, + { + "area": "auth", + "feature": "Cloudflare Access identity integration injecting reserved cf.user_id metadata", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/configuration/cloudflare-access/", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Per-gateway rate limiting with fixed or sliding windows (429)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/rate-limiting/", + "gap": "3" + }, + { + "area": "cost", + "feature": "Spend limits (up to 20 rules) scoped by provider/model/metadata with split-by-value buckets, fixed/rolling windows, block or fall-back-to-cheaper-model on breach", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/spend-limits/", + "gap": "2" + }, + { + "area": "cost", + "feature": "Unified Billing prepaid credits across providers (5% fee) and Machine Payments x402 per-request stablecoin settlement (beta)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/unified-billing/", + "gap": "2" + }, + { + "area": "cost", + "feature": "Token-based cost estimation with custom rates incl. per cache-read/write token fields", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/costs/", + "gap": "2" + }, + { + "area": "observability", + "feature": "Request logging with per-request opt-out and metadata-only mode; Workers Logs pricing for new gateways; Logpush export", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/logging/", + "gap": "context" + }, + { + "area": "observability", + "feature": "Dashboard analytics plus GraphQL API (aiGatewayRequestsAdaptiveGroups)", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/analytics/", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTLP trace export under GenAI semantic conventions with trace-id header correlation", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/otel-integration/", + "gap": "11" + }, + { + "area": "observability", + "feature": "User Insights: per-user spend attribution and rolling-p95 anomaly detection", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/user-insights/", + "gap": "context" + }, + { + "area": "observability", + "feature": "Automatic log classification (task + model-fit rating) feeding tasks-by-model and potential-savings views", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/log-classification/", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Guardrails: S1-S13 hazards (Llama Guard 3 8B) + P1 prompt injection (Prompt Guard 2) with per-category flag/block and audit logging", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/guardrails/", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "DLP scanning of requests and responses (incl. tool arguments) with pass/flag/block and shared Cloudflare One detection profiles", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/features/dlp/", + "gap": "7" + }, + { + "area": "tooling", + "feature": "First-class coding-agent integrations (nine agents), Vercel AI SDK provider, Worker bindings (run/patchLog/getLog), createAI unified client", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/integrations/coding-agents/", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Realtime and non-realtime WebSocket APIs relaying provider WS traffic with subprotocol-based auth", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/usage/websockets-api/", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Proxying of provider-native web-search tools plus a first-party Web Search API (beta) and background webhook execution", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/usage/web-search/", + "gap": "context" + }, + { + "area": "ops", + "feature": "Custom gateway domains, custom-provider registry for arbitrary HTTPS providers, zero-setup default gateway, published platform limits", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/configuration/custom-providers/", + "gap": "context" + }, + { + "area": "observability", + "feature": "Audit logs of gateway configuration changes on all plans", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/reference/audit-logs/", + "gap": "context" + }, + { + "area": "observability", + "feature": "Encrypted Logpush: per-log AES-256-GCM keys wrapped with the customer's RSA-4096 public key", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/observability/logging/logpush/", + "gap": "context" + }, + { + "area": "ops", + "feature": "Require provider credentials (byok_only / cf-aig-no-wholesale) blocking silent Unified Billing fallback", + "sourceUrl": "https://developers.cloudflare.com/ai-gateway/changelog/", + "gap": "2" + } + ] + }, + { + "name": "completion-to-response (NoahStepheno)", + "category": "protocol translation proxy (Responses over Chat Completions)", + "language": "Go", + "status": "archived 2026-08-16 (6 commits May 2026; single release v0.1.0)", + "url": "https://github.com/NoahStepheno/completion-to-response", + "capabilities": [ + { + "area": "protocol", + "feature": "OpenAI Responses client endpoint (POST /v1/responses) translated to any Chat Completions backend", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Full Responses SSE event-sequence synthesis from chat chunks (response.created through response.completed with per-event indexes)", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Complete mapping tables: instructions/developer to system, max_output_tokens to max_tokens, finish-reason to status, usage renames, resp_/msg_/fc_ id generation", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Streaming tool-call reconstruction into function_call output items with stable ids across fragments; multi-tool grouping", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform.go", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Reasoning preservation: reasoning_content deltas buffered and attached as summary on function_call items (unreleased main)", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/internal/types/completion.go", + "gap": "6" + }, + { + "area": "auth", + "feature": "Bearer key passthrough with static -key fallback", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go", + "gap": "context" + }, + { + "area": "routing", + "feature": "Multi-backend model-routing sidecar (Node) advertising alias models via /v1/models so Codex lists them", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/codex-gateway.js", + "gap": "context" + }, + { + "area": "ops", + "feature": "Hardened SSE transport (64 KB initial / 1 MB max scanner, [DONE] handling, flusher requirement) and 10 s graceful drain", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go", + "gap": "context" + }, + { + "area": "observability", + "feature": "Logging + panic-recovery middleware with 2048-byte truncated body capture", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/cmd/server/main.go", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Unit tests pinning the synthesized Responses event contract (ordering + per-chunk event counts)", + "sourceUrl": "https://github.com/NoahStepheno/completion-to-response/blob/main/internal/transform/transform_test.go", + "gap": "context" + } + ] + }, + { + "name": "done-hub (deanxv)", + "category": "open-source gateway (one-api lineage)", + "language": "Go (+ React UI)", + "status": "active (v1.23.22, 2026-09-29; last push 2026-10-27)", + "url": "https://github.com/deanxv/done-hub", + "capabilities": [ + { + "area": "cost", + "feature": "Multi-bucket cache-token usage ledger: 5m/1h cache-write buckets, cache-read, OpenAI cache_write_tokens, audio/image/text x input/output, reasoning tokens — each at its own ratio", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/types/common.go", + "gap": "2" + }, + { + "area": "cost", + "feature": "Quota-ledger correctness fixes incl. the cached_tokens omitempty bug (zero values dropped broke strict clients); zero-value serialization tests", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/README.md#L127-L133", + "gap": "context" + }, + { + "area": "cost", + "feature": "Flat Anthropic-style cache fields parsed in OpenAI-compatible usage as fallback billing (conservative 5m ratio)", + "sourceUrl": "https://github.com/deanxv/done-hub/commit/bbb2293e34", + "gap": "context" + }, + { + "area": "cost", + "feature": "Pre-consume estimate plus streaming realtime quota enforcement via Redis INCRBY Lua with TTL (can abort mid-stream); batched DB quota writer with ordered shutdown flush", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/relay/relay_util/quota.go", + "gap": "2" + }, + { + "area": "auth", + "feature": "Agent-CLI OAuth channel types: Claude Code, Codex (version-gated UA, per-model instruction files), Gemini CLI, Antigravity, Vertex AI Express, with token auto-refresh", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/providers/codex/base.go", + "gap": "new:oauth-credential-lifecycle" + }, + { + "area": "routing", + "feature": "Cross-channel retry bounded by live channel count with wall-clock retry timeout collapsing to 503, honest retry_skip/timeout logs, Retry-After-derived cooldowns", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/relay/main.go", + "gap": "1" + }, + { + "area": "routing", + "feature": "Group-based channel distribution with token-pinned groups, backup groups, and per-group billing multipliers", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/middleware/distributor.go", + "gap": "13;2" + }, + { + "area": "auth", + "feature": "API tokens with per-key quota, expiry, unlimited flag, batched used-quota metering", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/model/token.go", + "gap": "9;2" + }, + { + "area": "rate-limiting", + "feature": "Multi-bucket per-user + per-IP rate limiting with CIDR whitelist and a CriticalRateLimit that whitelisting deliberately does not exempt", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/middleware/rate-limit.go", + "gap": "3" + }, + { + "area": "observability", + "feature": "Prometheus metrics with per-channel/model dimensions; request-id and upstream request-id correlation", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/metrics/main.go", + "gap": "12" + }, + { + "area": "mcp", + "feature": "Built-in MCP server (SSE at /mcp/message plus Streamable HTTP) exposing gateway tools", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/mcp/server.go", + "gap": "8" + }, + { + "area": "guardrails", + "feature": "Pluggable content-safety checker framework plus MaxPromptTokens input guard (400 before upstream call)", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/safty/safty.go", + "gap": "7" + }, + { + "area": "protocol", + "feature": "Broad endpoint families across 45+ adapters (chat, completions, embeddings, rerank, realtime, audio, moderations, images, video, search, Midjourney tasks)", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/relay/relay_util/quota.go", + "gap": "context" + }, + { + "area": "cost", + "feature": "Long-context tiered pricing applied incrementally on the streaming path; bulk price import from models.dev", + "sourceUrl": "https://github.com/deanxv/done-hub/blob/main/relay/relay_util/quota.go", + "gap": "new:synced-price-catalogs" + } + ] + }, + { + "name": "F5 AI Gateway", + "category": "enterprise gateway (Kubernetes container)", + "language": "n/a (container product)", + "status": "in transition — original product docs redirect, Python SDK archived 2026-03-18; successor in Early Access (announced 2026-08-18)", + "url": "https://www.f5.com/company/news/press-releases/f5-unleashes-next-generation-agentic-ready-ai-gateway-to-optimize-the-economics-and-governance-of-enterprise-ai-costs", + "capabilities": [ + { + "area": "guardrails", + "feature": "Processor-chain guardrails (input/response stages) that can Modify, Reject, or Annotate; F5 processors for injection, language-id, repetition, system-prompt", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/processors-overview.html", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Data Security processor: deterministic PII detection/redaction/blocking (credit card, SSN, IBAN, phones, IPs, VIN, SQL, mcp JSON) across requests and responses", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/data-security.html", + "gap": "7" + }, + { + "area": "routing", + "feature": "Tag-based dynamic routing and annotation-driven payload enrichment (first-match-wins selectors)", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/processors-overview.html", + "gap": "13" + }, + { + "area": "auth", + "feature": "JWT client authentication with claim/header-based profile selection; mTLS per-service with file/env secret sources", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-policies.html", + "gap": "context" + }, + { + "area": "routing", + "feature": "Per-service retries (default 429/500/502/503/504, Retry-After priority, 1.5x backoff capped 60s) and one named fallback service", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html", + "gap": "1" + }, + { + "area": "protocol", + "feature": "Upstream executors (openai/azure/anthropic/ollama) with schema conversion; OpenAI-subset client schema", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/configure-services.html", + "gap": "context" + }, + { + "area": "observability", + "feature": "Full-fidelity transaction audit export (one JSON per transaction with bodies and processor chains) to stdout or S3-compatible storage", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/run/export-transactions.html", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTLP-over-gRPC traces and metrics incl. token-count and processor-rejection metrics", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/configuration/telemetry.html", + "gap": "11" + }, + { + "area": "guardrails", + "feature": "ML prompt-injection detector with tunable threshold and sliding-window chunking", + "sourceUrl": "https://web.archive.org/web/2025/https://aigateway.clouddocs.f5.com/processors/prompt-injection.html", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "AI Guardrails enforcement with NGINX Gateway Fabric PayloadProcessor (fail-closed on oversized prompts)", + "sourceUrl": "https://docs.nginx.com/nginx-gateway-fabric/how-to/f5-ai-guardrails/", + "gap": "7" + }, + { + "area": "mcp", + "feature": "Announced next-gen (Early Access): Model Gateway + MCP Gateway (registry, per-tool allow/deny, audit trail) + AI Guardrails with OTel/SIEM export", + "sourceUrl": "https://www.f5.com/company/news/press-releases/f5-unleashes-next-generation-agentic-ready-ai-gateway-to-optimize-the-economics-and-governance-of-enterprise-ai-costs", + "gap": "8" + } + ] + }, + { + "name": "Glama", + "category": "hosted LLM + MCP gateway", + "language": "TypeScript (Lightport core OSS; hosted product closed)", + "status": "active (MCP-first; LLM gateway narrowing to curated providers)", + "url": "https://glama.ai/", + "capabilities": [ + { + "area": "protocol", + "feature": "OpenAI-compatible LLM gateway endpoints (chat/completions, models, request polling) at gateway.glama.ai/v1", + "sourceUrl": "https://glama.ai/gateway/docs/api", + "gap": "context" + }, + { + "area": "routing", + "feature": "Provider-prefixed model routing over a 100+ model catalog with per-model pricing/latency metadata", + "sourceUrl": "https://glama.ai/gateway/models", + "gap": "13" + }, + { + "area": "auth", + "feature": "API key management with per-key log-capture toggle; OAuth PKCE one-click connect for end-user clients", + "sourceUrl": "https://glama.ai/gateway/docs/oauth-pkce", + "gap": "context" + }, + { + "area": "caching", + "feature": "Prompt caching via provider-native caches with cached tokens billed at the cheaper rate", + "sourceUrl": "https://glama.ai/gateway/docs/prompt-caching", + "gap": "14" + }, + { + "area": "observability", + "feature": "Request-logging dashboard with payloads, export to JSON/CSV, x-glama-capture-payload override, x-glama-metadata labels with per-label cost breakdown", + "sourceUrl": "https://glama.ai/gateway/docs/request-logging", + "gap": "context" + }, + { + "area": "mcp", + "feature": "MCP Gateway reverse proxy with connection profiles, managed upstream OAuth, per-tool access control, tool-change alerts, full JSON-RPC logging and SIEM export", + "sourceUrl": "https://glama.ai/mcp/gateway", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP hosting (start/stop/suspend, persistent volumes, per-instance tokens) and a 97k-server MCP directory with a public API", + "sourceUrl": "https://glama.ai/mcp/hosting", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP sampling, elicitation, tool progress notifications, audio/image tool responses, normalized reasoning-effort levels", + "sourceUrl": "https://glama.ai/release-notes", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Lightport: open-source translation core of the gateway (77 providers, per-request upstream proxy, Server-Timing stages); retries/caching/rate-limiting explicitly non-goals", + "sourceUrl": "https://github.com/glama-ai/lightport", + "gap": "context" + } + ] + }, + { + "name": "gproxy (LeenHawk)", + "category": "open-source gateway", + "language": "Rust", + "status": "active (pushed 2026-10-13)", + "url": "https://github.com/LeenHawk/gproxy", + "capabilities": [ + { + "area": "tooling", + "feature": "Built-in web admin console (providers, catalogs, routes, rules, users, usage) with a headless build variant", + "sourceUrl": "https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md", + "gap": "19" + }, + { + "area": "routing", + "feature": "Model routes with provider weights and fallback tiers, hot-editable without client changes", + "sourceUrl": "https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md", + "gap": "1;13" + }, + { + "area": "auth", + "feature": "Credential pools (API keys, OAuth tokens, cookies) with health, quotas, refresh, failover, and rate-limit-aware sorting", + "sourceUrl": "https://gproxy.leenhawk.com/guides/permissions/", + "gap": "5" + }, + { + "area": "rate-limiting", + "feature": "Per-user and per-key rate limits (requests or concurrency metrics) with shared cache-backend counters", + "sourceUrl": "https://gproxy.leenhawk.com/guides/permissions/", + "gap": "3" + }, + { + "area": "cost", + "feature": "USD spending budgets on users, keys, organizations, teams with model filters; post-call settlement", + "sourceUrl": "https://gproxy.leenhawk.com/guides/permissions/", + "gap": "2" + }, + { + "area": "auth", + "feature": "Permission rules (provider, model glob, operation) and OAuth gproxy: scopes", + "sourceUrl": "https://gproxy.leenhawk.com/guides/permissions/", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Per-provider request rewrite rules (system-text injection, cache breakpoints, JSON edits, regex, headers)", + "sourceUrl": "https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md", + "gap": "context" + }, + { + "area": "caching", + "feature": "Upstream prompt-cache management: breakpoint rules and magic tokens (default/5m/1h TTLs)", + "sourceUrl": "https://gproxy.leenhawk.com/guides/claude-caching/", + "gap": "14" + }, + { + "area": "observability", + "feature": "Usage/upstream/downstream/audit records with query APIs and a self-service portal; captures with default secret redaction", + "sourceUrl": "https://gproxy.leenhawk.com/guides/observability/", + "gap": "context" + }, + { + "area": "cost", + "feature": "In-engine settlement pricing from price_rules/rates with prompt-size and service-tier ladders; unpriced exchanges flagged", + "sourceUrl": "https://gproxy.leenhawk.com/reference/pricing/", + "gap": "2" + }, + { + "area": "protocol", + "feature": "16+ built-in channels incl. OAuth subscription channels and hyperscalers; OpenAI Responses client-side", + "sourceUrl": "https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md", + "gap": "6" + }, + { + "area": "ops", + "feature": "Desktop/mobile/CLI/container/serverless deployment; GPROXY_MASTER_KEY encryption at rest; self-update; cross-version migrations", + "sourceUrl": "https://raw.githubusercontent.com/LeenHawk/gproxy/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "GPT-Load (tbphp)", + "category": "open-source gateway", + "language": "Go", + "status": "active (v2.0.0-rc line Oct 2026; v1.4.x stable maintained)", + "url": "https://github.com/tbphp/gpt-load", + "capabilities": [ + { + "area": "auth", + "feature": "Two-tier keys: AUTH_KEY management vs scoped AccessKeys (group/model/protocol restrictions, source IP/CIDR, expiry, rotation preserving id/quota)", + "sourceUrl": "https://www.gpt-load.com/docs/access-keys", + "gap": "9;3" + }, + { + "area": "rate-limiting", + "feature": "Per-AccessKey RPM, concurrency caps, and cost quotas (never-resetting totals or periodic windows one minute to one year); soft enforcement", + "sourceUrl": "https://www.gpt-load.com/docs/access-keys", + "gap": "2;3" + }, + { + "area": "routing", + "feature": "Tier-priority weighted scheduling (group x credential weight) under a global down-tier-only retry budget; no switch after committed stream output", + "sourceUrl": "https://www.gpt-load.com/docs/internals/scheduling", + "gap": "1" + }, + { + "area": "routing", + "feature": "Soft session affinity from AccessKey/protocol/first-input prefix (or prompt_cache_key) with LRU 10,000 capacity", + "sourceUrl": "https://www.gpt-load.com/docs/internals/scheduling", + "gap": "14" + }, + { + "area": "auth", + "feature": "Credential cooldown + consecutive-failure blacklist isolation with model-scoped cooling and auto-recovery of API-key credentials", + "sourceUrl": "https://www.gpt-load.com/docs/internals/scheduling", + "gap": "5" + }, + { + "area": "observability", + "feature": "Embedded monitoring UI: health checklist, request log (7-day retention, opt-in body recording), route-check dry run, usage & cost with cache hit rate", + "sourceUrl": "https://www.gpt-load.com/docs/monitor", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Experimental JEV guardrails via external decision service (presets for credential leaks, PII, prompt injection; 403 block; verdict cache)", + "sourceUrl": "https://www.gpt-load.com/docs/advanced/guardrails", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Global RE2 request redaction with reversible per-AccessKey encryption restored in responses, SSE, and Responses WebSocket", + "sourceUrl": "https://www.gpt-load.com/docs/advanced/redaction", + "gap": "7" + }, + { + "area": "protocol", + "feature": "Broad protocol breadth: OpenAI Chat/Responses/Images/Embeddings/Rerank, Mistral-native OCR/audio, Anthropic Messages, Gemini incl. :embedContent; 35 channels with encrypted credentials", + "sourceUrl": "https://raw.githubusercontent.com/tbphp/gpt-load/main/README.md", + "gap": "6" + } + ] + }, + { + "name": "GPTCache (zilliztech)", + "category": "semantic response cache library + optional server", + "language": "Python", + "status": "maintenance-mode / dead-in-practice (final release v0.1.44, 2024-08-01; one burst of commits 2026-09-22, no release)", + "url": "https://github.com/zilliztech/GPTCache", + "capabilities": [ + { + "area": "caching", + "feature": "Five-stage pluggable cache pipeline (cache_enable_func, pre_embedding, embedding, data manager, similarity, post-process) with next_cache tiering", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/core.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Rank-threshold similarity adjudication with 11 interchangeable evaluators (exact, vector distance, ONNX ALBERT, k-reciprocal, Cohere rerank, SBERT cross-encoder, time-window, JEV)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/similarity_evaluation/time.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Temperature-aware cache skip (softmax over skip/don't-skip) and multi-answer post-processors (softmax/first/random)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "LLM-verification gate on cached answers (yes/no check degrades to a miss on no or exception)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/processor/post.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Eviction subsystem: soft-evict then threshold hard-delete, mark-then-sweep, periodic index rebuild; MemoryCacheEviction LRU/FIFO/LFU/RR and distributed RedisCacheEviction", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/manager/eviction/manager.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Three-axis storage matrix: scalar (SQLite/Postgres/MySQL/DynamoDB/...), vector (Milvus/FAISS/pgvector/Qdrant/Weaviate/...), object store (disk/S3)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/manager/vector_data/manager.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Session-scoped caching with pluggable hit filters (refuse repeats within a session)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/session.py", + "gap": "10" + }, + { + "area": "protocol", + "feature": "Drop-in adapters incl. stream capture on miss and SSE synthesis from a cached answer on hit", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/openai.py", + "gap": "10" + }, + { + "area": "ops", + "feature": "FastAPI cache server + Docker image with a language-agnostic get/put API and cache_skip prefix bypass", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache_server/server.py", + "gap": "context" + }, + { + "area": "caching", + "feature": "Multi-tier cache chaining via next_cache (local L1 falling through to shared L2) and search_only probing", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py", + "gap": "10" + }, + { + "area": "caching", + "feature": "Cache-consistency self-heal between scalar and vector stores (embedding compare, forced miss + rewrite on mismatch)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/adapter.py", + "gap": "10" + }, + { + "area": "observability", + "feature": "Per-stage cache telemetry, hit callbacks with scores, and persisted hit reports", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/report.py", + "gap": "context" + }, + { + "area": "tooling", + "feature": "YAML-driven composition and cache-quality benchmark harnesses (hit ratio, latency, recall)", + "sourceUrl": "https://github.com/zilliztech/GPTCache/blob/main/gptcache/adapter/api.py", + "gap": "context" + } + ] + }, + { + "name": "Helicone", + "category": "observability platform + AI gateway", + "language": "TypeScript (OSS Rust ai-gateway side project)", + "status": "active", + "url": "https://docs.helicone.ai/", + "capabilities": [ + { + "area": "routing", + "feature": "Single OpenAI-compatible gateway endpoint for 100+ providers with per-request provider translation", + "sourceUrl": "https://docs.helicone.ai/gateway/overview.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Model-string routing: manual fallback chains, provider exclusions, cost-priority load balancing (BYOK first, cheapest provider first)", + "sourceUrl": "https://docs.helicone.ai/gateway/provider-routing.md", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Automatic failover (429/401/400/408/500+) and opt-in gateway retries with exponential backoff", + "sourceUrl": "https://docs.helicone.ai/gateway/concepts/error-handling.md", + "gap": "1" + }, + { + "area": "auth", + "feature": "Managed provider keys via prepaid credits (pass-through billing) with BYOK fallback and most-actionable-error selection", + "sourceUrl": "https://docs.helicone.ai/gateway/concepts/error-handling.md", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Header-based rate limits (IETF draft format) global/per-user/per-custom-property, request-count or cost-in-cents", + "sourceUrl": "https://docs.helicone.ai/features/advanced-usage/custom-rate-limits.md", + "gap": "3" + }, + { + "area": "caching", + "feature": "Edge response cache with TTL, buckets, seeds, ignore-keys; HIT/MISS headers; cache-savings dashboards", + "sourceUrl": "https://docs.helicone.ai/features/advanced-usage/caching.md", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "LLM Security prompt-injection screening (Meta Prompt Guard 86M; advanced Llama Guard 3.8B), OpenAI models only", + "sourceUrl": "https://docs.helicone.ai/features/advanced-usage/llm-security.md", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Token-limit exception handlers: truncate, middle-out, or fallback-model switch on context overflow", + "sourceUrl": "https://docs.helicone.ai/features/advanced-usage/token-limit-exception-handlers.md", + "gap": "7" + }, + { + "area": "cost", + "feature": "Dual-mode cost calculation (gateway-accurate vs best-effort from a 300+ model pricing repository)", + "sourceUrl": "https://docs.helicone.ai/references/how-we-calculate-cost.md", + "gap": "2" + }, + { + "area": "observability", + "feature": "Alerts on nine metrics with percentile aggregations via email/Slack", + "sourceUrl": "https://docs.helicone.ai/features/alerts.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "Completion webhooks with HMAC-signed payloads (10 KB bodies, S3 URLs for full data), configurable sampling", + "sourceUrl": "https://docs.helicone.ai/features/webhooks.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "omit-logs headers: keep metrics while dropping request/response content", + "sourceUrl": "https://docs.helicone.ai/features/advanced-usage/omit-logs.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "Sessions with hierarchical paths, HQL SQL query language, weekly reports, PostHog export, REST admin API", + "sourceUrl": "https://docs.helicone.ai/llms.txt", + "gap": "context" + }, + { + "area": "mcp", + "feature": "Official MCP server (query_requests, query_sessions) for Claude Desktop/Claude Code/Cursor/Codex", + "sourceUrl": "https://docs.helicone.ai/integrations/tools/mcp.md", + "gap": "8" + }, + { + "area": "tooling", + "feature": "Prompt Management V2 with versioning and gateway deployment (ID-addressed prompts)", + "sourceUrl": "https://www.helicone.ai/changelog", + "gap": "context" + }, + { + "area": "protocol", + "feature": "OpenAI Responses API through the gateway (OpenAI and Anthropic models)", + "sourceUrl": "https://docs.helicone.ai/gateway/concepts/responses-api.md", + "gap": "6" + }, + { + "area": "ops", + "feature": "Self-hostable full platform (Docker Compose/Helm) and an open-source Rust ai-gateway (beta; last release 2025-07-21)", + "sourceUrl": "https://github.com/helicone/ai-gateway", + "gap": "context" + }, + { + "area": "observability", + "feature": "TTFT and p50/p95/p99 latency analytics in dashboards", + "sourceUrl": "https://helicone-helicone-7.mintlify.app/observability/metrics", + "gap": "17" + }, + { + "area": "caching", + "feature": "Provider prompt-caching pass-through with documented cost multipliers (Anthropic/OpenAI/Gemini)", + "sourceUrl": "https://docs.helicone.ai/gateway/concepts/prompt-caching.md", + "gap": "14" + } + ] + }, + { + "name": "Higress", + "category": "open-source API gateway (CNCF sandbox, AI plugins)", + "language": "Wasm plugins in Go/Rust on Envoy+Istio", + "status": "active (v2.2.5, 2026-10-04)", + "url": "https://higress.ai/", + "capabilities": [ + { + "area": "protocol", + "feature": "ai-proxy plugin: ~30 provider types with automatic OpenAI/Claude client detection and zero-config conversion; original protocol passthrough", + "sourceUrl": "https://higress.cn/en/docs/latest/plugins/ai/api-provider/ai-proxy", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Wildcard modelMapping with prefix patterns and multi-provider routes (providers list + activeProviderId)", + "sourceUrl": "https://higress.cn/en/docs/latest/plugins/ai/api-provider/ai-proxy", + "gap": "6" + }, + { + "area": "auth", + "feature": "key-auth consumer credentials with per-route/per-host allow lists, global_auth, X-Mse-Consumer injection; full auth-plugin suite incl. OIDC, OPA, external auth", + "sourceUrl": "https://higress.ai/en/docs/latest/plugins/authentication/key-auth", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "ai-token-ratelimit: tokens per second/minute/hour/day keyed by consumer/header/param/cookie/per-IP over Redis", + "sourceUrl": "https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-token-ratelimit", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "ai-quota: Redis-stored per-consumer token quotas with admin refresh/delta/read endpoints", + "sourceUrl": "https://higress.cn/en/docs/latest/plugins/ai/api-consumer/ai-quota", + "gap": "3" + }, + { + "area": "routing", + "feature": "API-key pool failover: failure/success thresholds, health checks, cooldownDuration restore", + "sourceUrl": "https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-proxy/README_EN.md", + "gap": "1;5" + }, + { + "area": "routing", + "feature": "ai-load-balancer: global least-request, prefix_cache routing, cluster metrics (EWMA first-token latency), cluster_hash consistent hashing", + "sourceUrl": "https://github.com/higress-group/higress/blob/main/plugins/wasm-go/extensions/ai-load-balancer/README_EN.md", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Model-level fallback chains in the console's AI Route Config (e.g. DeepSeek to qwen-turbo on rate limit)", + "sourceUrl": "https://higress.ai/en/docs/ai/scene-guide/multi-proxy", + "gap": "1" + }, + { + "area": "caching", + "feature": "ai-cache: Redis response cache keyed by GJSON paths with streaming SSE replay and skip header", + "sourceUrl": "https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-cache", + "gap": "10" + }, + { + "area": "caching", + "feature": "Semantic cache mode via embedding provider + DashVector vector store (console AI Route Config)", + "sourceUrl": "https://higress.ai/en/docs/ai/scene-guide/semantic-cache", + "gap": "10" + }, + { + "area": "observability", + "feature": "ai-statistics: Prometheus counters for input/output tokens, service duration, first-token duration across route/cluster/model/consumer labels; custom attribute extraction into logs/traces", + "sourceUrl": "https://higress.cn/en/docs/latest/plugins/ai/api-o11y/ai-statistics", + "gap": "11;12" + }, + { + "area": "guardrails", + "feature": "ai-data-masking: bidirectional sensitive-data replacement (built-in library, grok/regex, hash) with optional restore", + "sourceUrl": "https://higress.ai/en/docs/latest/user/plugins/ai/api-consumer/ai-data-masking", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "ai-security-guard: request/response moderation via Aliyun Content Security with deny codes and metrics", + "sourceUrl": "https://higress.ai/en/docs/latest/user/plugins/ai/api-provider/ai-security-guard", + "gap": "7" + }, + { + "area": "mcp", + "feature": "MCP server hosting: ConfigMap database servers, REST-to-MCP conversion, Nacos >=3.0 registry, OpenAPI-to-MCP tool; 2025-03-26 Streamable HTTP + 2026-07-28 stateless support", + "sourceUrl": "https://higress.cn/en/docs/ai/mcp-quick-start/", + "gap": "8" + }, + { + "area": "mcp", + "feature": "mcp-server plugin: REST-to-MCP with per-tool authorization (allowTools intersection with auth-plugin header), security schemes, passthrough credentials", + "sourceUrl": "https://higress.ai/en/docs/ai/mcp-server", + "gap": "8" + }, + { + "area": "mcp", + "feature": "mcp-proxy: front existing MCP backends over SSE/HTTP with legacy/modern/auto protocol probing and toolSet merging", + "sourceUrl": "https://higress.ai/en/docs/ai/mcp-quick-start", + "gap": "8" + }, + { + "area": "tooling", + "feature": "Web console (LLM Provider Management, AI Route Config, AI Dashboard), hgctl CLI incl. agent and higress-ops MCP server for AI-assisted operations", + "sourceUrl": "https://higress.ai/en/docs/ai/quick-start", + "gap": "19" + }, + { + "area": "ops", + "feature": "Millisecond-level hot config via Envoy/Istio xDS; Wasm plugin hot upgrades; Gateway API + Inference Extension conformance", + "sourceUrl": "https://higress.cn/en/docs/latest/overview/what-is-higress/", + "gap": "16" + }, + { + "area": "routing", + "feature": "Content-based model routing via regex, ai-context-limit context-window enforcement, native Bedrock Anthropic and vLLM passthrough (2026 releases)", + "sourceUrl": "https://github.com/higress-group/higress/releases", + "gap": "13" + } + ] + }, + { + "name": "IBM ContextForge (mcp-context-forge)", + "category": "open-source MCP/AI gateway", + "language": "Python 3.12 (FastAPI) + Rust MCP runtime crate", + "status": "active (1.1.0, 2026-10-19; 1.0.0-RC line)", + "url": "https://github.com/IBM/mcp-context-forge", + "capabilities": [ + { + "area": "mcp", + "feature": "MCP registry + gateway core: tools/resources/prompts and virtual servers federated over HTTP/JSON-RPC, WebSocket, SSE, and Streamable HTTP", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/overview/features/", + "gap": "8" + }, + { + "area": "protocol", + "feature": "OpenAI-compatible LLM proxy endpoints (POST /v1/chat/completions with optional SSE, GET /v1/models)", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/api/llm-proxy/", + "gap": "6" + }, + { + "area": "routing", + "feature": "12 provider types with server-side encrypted keys, health checks, and capability-flagged model catalog", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/overview/features/", + "gap": "context" + }, + { + "area": "auth", + "feature": "Auth-on-by-default: JWT (HS/RS/ES 256-512), basic, SSO/OIDC (GitHub/Google/Entra/Okta/Keycloak), dynamic client registration, mTLS client-cert auth", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "Scoped API tokens: SHA-256 hashed with fine-grained scopes (server, permissions, IP CIDRs, time-of-day, quotas), jti revocation and replay detection", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/", + "gap": "9" + }, + { + "area": "auth", + "feature": "Encrypted credential vault: AES-GCM tool credentials, Fernet OAuth secrets, Argon2id passwords, optional HashiCorp Vault per-user resolution", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/", + "gap": "15" + }, + { + "area": "rate-limiting", + "feature": "Tool-invocation rate limiting (token bucket per tool+user, 429 with Retry-After; Redis/database shared state) plus per-user MCP-server limits", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/adr/006-gateway-tool-rate-limiting/", + "gap": "3" + }, + { + "area": "caching", + "feature": "Pluggable cache backend (memory/Redis/database) for tool responses, resource lookups, session and rate-limit state", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/adr/007-pluggable-cache-backend/", + "gap": "context" + }, + { + "area": "cost", + "feature": "ToolOps: per-tool cost calculation from model pricing, per-call breakdowns, cost thresholds, throttling, historical data, budget alerts", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/overview/features/", + "gap": "2" + }, + { + "area": "observability", + "feature": "OTLP tracing to eight backends plus JWT-protected Prometheus metrics with Grafana dashboards", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/manage/observability/", + "gap": "11;12" + }, + { + "area": "guardrails", + "feature": "Guardrail plugins (deny_filter, pii_filter, content_moderation, schema_guard, sql_sanitizer, secrets_detection, rate_limiter, optional OPA/Rego) over JSON-Schema tool validation", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Centralized input validation: length/charset limits, URL scheme allowlists, MIME allowlist, resource caps (100KB/10KB, 413/415), SSRF/TLS hardening, root-URI deny-by-default", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/security-features/", + "gap": "7" + }, + { + "area": "auth", + "feature": "RBAC (Platform Admin/Team Admin/Developer/Viewer) with team-based multitenancy and visibility scopes; llm.invoke/llm.read permissions on LLM endpoints", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/architecture/multitenancy/", + "gap": "new:caller-sso-rbac" + }, + { + "area": "protocol", + "feature": "REST-to-MCP adapter with automatic JSON-Schema extraction; A2A agent-to-agent integration first-class (gRPC/WebSocket upstreams deprecated)", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/overview/features/", + "gap": "8" + }, + { + "area": "tooling", + "feature": "Admin UI (HTMX/Alpine) with LLM provider CRUD, health checks, no-key model testing; config export/import; 300+ config variables", + "sourceUrl": "https://ibm.github.io/mcp-context-forge/latest/api/llm-proxy/", + "gap": "19" + } + ] + }, + { + "name": "kgateway", + "category": "Gateway API control plane (Envoy data plane)", + "language": "Go", + "status": "active (v2.5.0-rc.1, 2026-10-27; CNCF sandbox)", + "url": "https://kgateway.dev/", + "capabilities": [ + { + "area": "ops", + "feature": "Gateway API edge features: JWT/OAuth2/OIDC + ext-proc auth, local + global rate limiting, retries with per-try timeouts and outlier detection, priority failover, transformations, mirroring", + "sourceUrl": "https://kgateway.dev/", + "gap": "context" + }, + { + "area": "mcp", + "feature": "MCP delegated to agentgateway since v2.2 (separate AgentgatewayPolicy/Backend CRDs, Helm chart, controller; kgateway CRD migration required)", + "sourceUrl": "https://kgateway.dev/blog/kgateway-v2.2-release-blog", + "gap": "context" + } + ] + }, + { + "name": "Kong AI Gateway", + "category": "enterprise gateway (plugins on Kong Gateway / Konnect)", + "language": "Lua plugins (PDK) + control plane", + "status": "active (AI Gateway 2.2.0, 2026-09-30)", + "url": "https://developer.konghq.com/ai-gateway/", + "capabilities": [ + { + "area": "routing", + "feature": "AI Proxy Advanced: seven load-balancing algorithms (weighted RR, consistent-hash, least-connections, EWMA lowest-latency, lowest-usage, semantic, priority tiers)", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Cross-provider failover with configurable retries/timeouts, failover_criteria (http_429/http_502/non_idempotent), circuit breaker (max_fails/fail_timeout)", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "1" + }, + { + "area": "protocol", + "feature": "Route types across chat, completions, embeddings, assistants, responses, files, batches, audio, images, video, realtime WebSocket; 17 upstream providers", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Provider-native passthrough via config.llm_format (anthropic/bedrock/cohere/gemini/huggingface) with analytics and cost still applied", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Dynamic model selection via $(headers.*)/$(uri_captures.*)/$(query_params.*) templating, model aliases, and request-level credential override", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "6" + }, + { + "area": "rate-limiting", + "feature": "AI Rate Limiting Advanced: token-kind or cost limits at consumer/consumer-group/IP/header/path/model/provider level, local/cluster/redis, sliding windows, 429 with X-AI-RateLimit headers", + "sourceUrl": "https://developer.konghq.com/plugins/ai-rate-limiting-advanced/", + "gap": "3" + }, + { + "area": "cost", + "feature": "Layered per-request cost model: service_tier x (context-window-factor x input incl. TTL-tiered cache read/write + output), per-modality rates since 2.2.0", + "sourceUrl": "https://developer.konghq.com/ai-gateway/model-cost-management/", + "gap": "2" + }, + { + "area": "caching", + "feature": "AI Semantic Cache over Redis 8+/Valkey/ElastiCache or pgvector with cache-status headers and Cache-Control directive support", + "sourceUrl": "https://developer.konghq.com/plugins/ai-semantic-cache/", + "gap": "10" + }, + { + "area": "caching", + "feature": "AI Prompt Compressor (LLMLingua-2 in a sidecar service) by ratio, target tokens, conditional rules, or tags", + "sourceUrl": "https://developer.konghq.com/plugins/ai-prompt-compressor/", + "gap": "new:payload-cost-transforms" + }, + { + "area": "guardrails", + "feature": "AI Prompt Guard (PCRE allow/deny on user messages incl. invisible-unicode detection) and AI Semantic Prompt/Response Guards via embeddings + vector similarity", + "sourceUrl": "https://developer.konghq.com/plugins/ai-prompt-guard/", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "AI PII Sanitizer: spaCy sidecar detecting 20+ PII classes with placeholder/synthetic redaction and optional value restoration", + "sourceUrl": "https://developer.konghq.com/plugins/ai-sanitizer/", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Managed guardrail integrations (AWS Guardrails, Azure Content Safety, GCP Model Armor, Lakera, NVIDIA NeMo, custom HTTP guardrails) with shared rejection modes", + "sourceUrl": "https://developer.konghq.com/plugins/?category=ai", + "gap": "7" + }, + { + "area": "mcp", + "feature": "AI MCP Proxy: HTTP-MCP bridge in four modes (passthrough, conversion, conversion-only, aggregated listener) with per-tool ACLs; revisions 2025-06-18/2025-11-25/2026-07-28", + "sourceUrl": "https://developer.konghq.com/plugins/ai-mcp-proxy/", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP 2026-07-28 support (server/discover, subscriptions, ttlMs cache hints), Token Vault for short-lived upstream credentials, MCP forward-proxy support", + "sourceUrl": "https://developer.konghq.com/ai-gateway/changelog/", + "gap": "8" + }, + { + "area": "protocol", + "feature": "AI A2A Proxy: transparent Agent2Agent detection on REST and JSON-RPC bindings with SSE passthrough, task-state extraction, and OTel spans", + "sourceUrl": "https://developer.konghq.com/plugins/ai-a2a-proxy/", + "gap": "context" + }, + { + "area": "auth", + "feature": "AI Auth Strategy entity: key-auth and openid-connect inbound auth with consumer/principal mapping", + "sourceUrl": "https://developer.konghq.com/ai-gateway/entities/ai-auth-strategy/", + "gap": "context" + }, + { + "area": "auth", + "feature": "Cloud IAM/OAuth2 auth for Redis-family and vector-store dependencies (AWS/Azure/GCP; client-credentials with async refresh)", + "sourceUrl": "https://developer.konghq.com/plugins/ai-proxy-advanced/", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "AI Consumer Groups with attached AI Policies for tiered token quotas and cost budgets (e.g. Gold 1M tokens/h + $100 vs Bronze 100K + $10)", + "sourceUrl": "https://developer.konghq.com/ai-gateway/entities/ai-consumer-group/", + "gap": "2;3" + }, + { + "area": "observability", + "feature": "ai_metrics Prometheus family: request/cost/token counters (token_type incl. cache-write TTL splits), latency histograms, cache and MCP telemetry, per consumer/model labels", + "sourceUrl": "https://developer.konghq.com/ai-gateway/monitor-ai-llm-metrics/", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "OTLP GenAI export with selectable OpenInference vs OpenLLMetry span formats since 2.2.0", + "sourceUrl": "https://developer.konghq.com/ai-gateway/changelog/", + "gap": "11" + }, + { + "area": "tooling", + "feature": "kongctl declarative CLI (plan/apply/diff/sync) for AI Gateway entities; decK for on-prem; Vaults entity for secret references", + "sourceUrl": "https://developer.konghq.com/kongctl/", + "gap": "context" + }, + { + "area": "ops", + "feature": "FIPS 140-3 and distroless image variants; forward-proxy support across ~14 AI plugins; conditional plugin execution via typed expressions (2.1.0)", + "sourceUrl": "https://developer.konghq.com/ai-gateway/changelog/", + "gap": "context" + }, + { + "area": "tooling", + "feature": "AI Prompt Template plugin ({template://name} substitution with JSON-control sanitization and allow_untemplated_requests:false)", + "sourceUrl": "https://developer.konghq.com/plugins/ai-prompt-template", + "gap": "context" + }, + { + "area": "ops", + "feature": "RSA-signed admin audit logs with TTL purge and record filters", + "sourceUrl": "https://developer.konghq.com/gateway/audit-logs", + "gap": "context" + } + ] + }, + { + "name": "Lakera Guard (Check Point AI Guardrails)", + "category": "guardrails screening API", + "language": "n/a (SaaS; self-hosting options)", + "status": "active (acquired by Check Point, announced 2025-09-16)", + "url": "https://docs.lakera.ai/", + "capabilities": [ + { + "area": "guardrails", + "feature": "Prompt Defense: direct, indirect, and agent-borne injection/jailbreak detection incl. poisoned tool responses, fetched content, and tool descriptions; 100+ languages", + "sourceUrl": "https://docs.lakera.ai/docs/prompt-defense", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Data Leakage Prevention: named PII detectors (emails, phones, cards with Luhn, IBAN, SSN) with block/mask actions and system-prompt leak detection", + "sourceUrl": "https://docs.lakera.ai/docs/data-leakage-prevention", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Content moderation categories with custom topics and L1-L4 confidence thresholds; obfuscated-profanity detection", + "sourceUrl": "https://docs.lakera.ai/docs/content-moderation", + "gap": "7" + }, + { + "area": "protocol", + "feature": "/v2/guard screening API in OpenAI chat format incl. tools-array screening during MCP handshakes; breakdown and payload masking options", + "sourceUrl": "https://docs.lakera.ai/docs/api/guard.md", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Agent Behavior Defense: dangerous-deviation detector and runtime tool allow/deny lists", + "sourceUrl": "https://docs.lakera.ai/docs/defenses", + "gap": "7" + } + ] + }, + { + "name": "Langfuse", + "category": "observability platform (with Rust ai-gateway)", + "language": "TypeScript (ai-gateway in Rust)", + "status": "active (v4)", + "url": "https://langfuse.com/", + "capabilities": [ + { + "area": "protocol", + "feature": "Rust ai-gateway relaying OpenAI + Anthropic byte-preserving (/openai/v1 incl. responses/compact, /anthropic/v1 incl. count_tokens), gateway-key auth via HMAC-signed resolve", + "sourceUrl": "https://github.com/langfuse/langfuse/tree/main/ai-gateway", + "gap": "context" + }, + { + "area": "observability", + "feature": "ai-gateway automatic OTLP usage telemetry batched with retries; UUIDv7 request ids; W3C traceparent; enrichment and coding-agent headers; full vs usage capture modes", + "sourceUrl": "https://github.com/langfuse/langfuse/tree/main/ai-gateway", + "gap": "11" + }, + { + "area": "observability", + "feature": "OTLP/HTTP ingestion (JSON or protobuf) mapping GenAI and OpenInference attributes onto traces/observations", + "sourceUrl": "https://langfuse.com/docs/opentelemetry", + "gap": "11" + }, + { + "area": "observability", + "feature": "Hierarchical per-request traces (spans/generations/events) with async background export and client-side masking hooks", + "sourceUrl": "https://langfuse.com/docs/observability/overview", + "gap": "11" + }, + { + "area": "observability", + "feature": "Threshold monitors with Slack/HMAC-webhook/GitHub Actions delivery; severity states and no-data modes", + "sourceUrl": "https://langfuse.com/docs/observability/features/alerts.md", + "gap": "context" + }, + { + "area": "cost", + "feature": "Ingestion-time cost computation: usage_details/cost_details buckets, regex-matched per-project model definitions with pricing tiers, daily default-price audits", + "sourceUrl": "https://langfuse.com/docs/observability/features/token-and-cost-tracking.md", + "gap": "2" + }, + { + "area": "protocol", + "feature": "Usage normalization contract: non-overlapping buckets, cache-inclusive provider counts normalized, strict OpenAI-schema mapping", + "sourceUrl": "https://langfuse.com/docs/observability/features/token-and-cost-tracking.md", + "gap": "context" + }, + { + "area": "auth", + "feature": "RBAC (Owner/Admin/Member/Viewer/None) with org+project roles; once-shown project keys; org keys for SCIM/API", + "sourceUrl": "https://langfuse.com/docs/administration/rbac.md", + "gap": "new:caller-sso-rbac" + }, + { + "area": "tooling", + "feature": "Prompt management with stale-while-revalidate client caching and outage fallback prompts", + "sourceUrl": "https://langfuse.com/docs/prompt-management/features/caching.md", + "gap": "context" + }, + { + "area": "ops", + "feature": "Self-hosting: web+worker over PostgreSQL/Redis/ClickHouse/S3 with queued S3-first ingestion for spike tolerance", + "sourceUrl": "https://langfuse.com/self-hosting", + "gap": "context" + }, + { + "area": "observability", + "feature": "SDK background batching with non-throwing retries (best-effort durability contract)", + "sourceUrl": "https://langfuse.com/docs/observability/features/queuing-batching.md", + "gap": "context" + } + ] + }, + { + "name": "LangSmith LLM Gateway", + "category": "observability platform gateway (beta)", + "language": "n/a (SaaS; self-hosted Helm)", + "status": "active (beta; announced May 2026)", + "url": "https://docs.langchain.com/langsmith/llm-gateway", + "capabilities": [ + { + "area": "protocol", + "feature": "One API key serving /v1/chat/completions, /v1/messages, /v1/responses with cross-provider translation (provider/model ids)", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway", + "gap": "6" + }, + { + "area": "cost", + "feature": "Hierarchical spend limits (org/workspace/key/user) with narrower-scopes-only-tighten rule, all-matching-checked, real-time HTTP 402 blocking", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway-spend-policies", + "gap": "2" + }, + { + "area": "guardrails", + "feature": "Data policies: in-path PII/secrets redaction to [SAFE_TO_USE:...] placeholders with event-by-event stream restoration; scanners fail open by default", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway-data-policy", + "gap": "7" + }, + { + "area": "auth", + "feature": "RBAC (gateway:invoke + workspaces:read), workspace-scoped service keys only, provider secrets server-side, audit logging (Enterprise), Anthropic OAuth pass-through", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway-access", + "gap": "context" + }, + { + "area": "observability", + "feature": "Every gateway call auto-traced with opt-in content capture; policy attributes on spans; Spend Monitoring dashboard", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway-monitoring", + "gap": "11" + }, + { + "area": "ops", + "feature": "Self-hosting via Helm agentGateway (containerPort 8083, HPA, optional Presidio sidecar)", + "sourceUrl": "https://docs.langchain.com/langsmith/llm-gateway-self-hosted", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTLP ingestion with GenAI/OpenInference attribute mapping (non-W3C id caveats documented)", + "sourceUrl": "https://docs.langchain.com/langsmith/trace-with-opentelemetry.md", + "gap": "11" + }, + { + "area": "cost", + "feature": "Run-level cost tracking from usage_metadata against workspace model-pricing tables (regex match, per-token-type breakdowns)", + "sourceUrl": "https://docs.langchain.com/langsmith/cost-tracking.md", + "gap": "2" + }, + { + "area": "mcp", + "feature": "Hosted MCP server (stateless streamable HTTP) for runs, prompts, datasets, experiments, billing; OAuth 2.1 dynamic registration or API key", + "sourceUrl": "https://docs.langchain.com/langsmith/langsmith-remote-mcp.md", + "gap": "8" + } + ] + }, + { + "name": "LiteLLM", + "category": "open-source gateway + SDK", + "language": "Python (Rust translation core beta)", + "status": "active (v1.105+ weekly minors, October 2026)", + "url": "https://docs.litellm.ai/", + "capabilities": [ + { + "area": "auth", + "feature": "Virtual keys via POST /key/generate: max_budget, budget_duration, tpm/rpm, max_parallel_requests, model allowlists, aliases, per-key config/router_settings overrides", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/virtual_keys", + "gap": "9" + }, + { + "area": "auth", + "feature": "Hierarchical org/team/user/key budgets with shared budget tables, end-user/customer budgets, object-level permissions (models, MCP servers, tools, toolsets)", + "sourceUrl": "ref/litellm/schema.prisma", + "gap": "9" + }, + { + "area": "auth", + "feature": "Key lifecycle: auto-rotation schedules, grace-period regeneration (Enterprise), block/unblock, temp budget increases, alias regex, generation bounds and policy hooks", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/virtual_keys", + "gap": "9" + }, + { + "area": "auth", + "feature": "Pluggable authN: JWT->virtual-key claim mapping, OAuth2 introspection, oauth2-proxy headers, SAML SSO, SCIM provisioning", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/token_auth", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "RBAC roles (proxy_admin, internal_user, team admin with team_admin_editable_team_fields, team_member_permissions)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/access_control", + "gap": "new:caller-sso-rbac" + }, + { + "area": "routing", + "feature": "Router strategies: simple-shuffle (weighted), least-busy, usage-based, latency-based, cost-based; routing_groups; order-based priority", + "sourceUrl": "https://docs.litellm.ai/docs/routing", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Retry engine with per-error-class RetryPolicy, x-litellm-num-retries header override, provider clients pinned to max_retries 0", + "sourceUrl": "https://docs.litellm.ai/docs/routing", + "gap": "1" + }, + { + "area": "routing", + "feature": "Cooldowns as per-deployment circuit breakers (allowed_fails per minute, immediate on 429 or >50% failure, per-error AllowedFailsPolicy)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/reliability", + "gap": "1" + }, + { + "area": "routing", + "feature": "Typed fallbacks: generic, context-window, content-policy, default wildcard targets with enforce_fallback_model_access/budget", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/reliability", + "gap": "1" + }, + { + "area": "routing", + "feature": "Pre-call checks: context-window filtering, region filtering, max_parallel_requests fast-fail, session affinity, enable_weighted_failover, mid-stream fallback continuation via assistant prefill", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/load_balancing", + "gap": "1" + }, + { + "area": "routing", + "feature": "Adaptive Router (beta): 7-type request classification, quality/cost weighting, persisted learning state, admin state endpoint, per-request tier override", + "sourceUrl": "https://docs.litellm.ai/docs/adaptive_router", + "gap": "13" + }, + { + "area": "routing", + "feature": "Shadow traffic: silent_model mirrors 100% of requests to evaluation models in the background", + "sourceUrl": "https://docs.litellm.ai/docs/traffic_mirroring", + "gap": "context" + }, + { + "area": "routing", + "feature": "Model access groups (keys/teams reference group names; wildcards Enterprise) and fleet-wide model_group_alias remapping", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/model_access_groups", + "gap": "9" + }, + { + "area": "rate-limiting", + "feature": "Batched pre-call limiter: tpm/rpm/max_parallel_requests per key/user/team with reserve-then-reconcile TPM (estimate prompt + max_tokens, settle actuals, refund on failure)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/users", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "Dynamic saturation-based limits (dynamic_rate_limiter_v3), per-model budgets, budget reservation before calls, fail-closed enforcement modes", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/hooks/dynamic_rate_limiter_v3.py", + "gap": "3" + }, + { + "area": "rate-limiting", + "feature": "Rate/budget tiers assignable to many keys via budget_id; io token rate limits; per-model rpm/tpm dicts with per-model remaining headers", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/rate_limit_tiers", + "gap": "3" + }, + { + "area": "cost", + "feature": "Per-request cost against a 2,752-model price map, x-litellm-response-cost header, spend logs with cache/reasoning splits, zero-cost anomaly counter", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/cost_tracking", + "gap": "2" + }, + { + "area": "cost", + "feature": "Budgets at every scope with periodic/calendar resets, soft budgets, multi-window budget_limits stacking, tag budgets via header or W3C baggage", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/users", + "gap": "2" + }, + { + "area": "cost", + "feature": "Provider budget routing: per-provider USD budgets with 429 enforcement and Prometheus remaining-budget metric", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/provider_budget_routing", + "gap": "2" + }, + { + "area": "cost", + "feature": "Budget reservation by default with fail_closed_budget_enforcement; documented failure mode when cost unestimatable", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/users", + "gap": "2" + }, + { + "area": "cost", + "feature": "Alerting: Slack/Teams/webhook with budget_alerts, spend reports, digest mode, projected-exceedance webhooks (beta)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/alerting", + "gap": "2" + }, + { + "area": "cost", + "feature": "Spend reporting APIs (/spend/keys, /spend/users, /spend/logs/v2 pagination, /user/daily/activity; Enterprise /global/spend/report and reset)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/cost_tracking", + "gap": "2" + }, + { + "area": "caching", + "feature": "Response caching across 8+ backends (Redis, Redis/Qdrant/Valkey semantic, S3, GCS, in-memory, disk, DualCache) with x-litellm-cache-key and /cache/delete", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/caching", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "Guardrail hooks in four modes (pre_call, during_call, post_call, logging_only) across ~50 integrations; reversible Presidio PII masking with output_parse_pii re-insertion; tool-result guardrails", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/guardrails/quick_start", + "gap": "7" + }, + { + "area": "mcp", + "feature": "MCP gateway: one endpoint for all tool traffic with six-level permissions, per-tool allowlists, pinned tool schemas (anti-poisoning), OAuth2/token-exchange/SigV4 upstream auth, /mcp-rest bridge, OpenAPI-to-MCP", + "sourceUrl": "https://docs.litellm.ai/docs/mcp", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Pass-through endpoints for 8 providers plus arbitrary custom routes with per-route cost tracking and (Enterprise) route allow/deny", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/pass_through", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Universal Anthropic /v1/messages across all providers incl. count_tokens; parameter mapping to Responses; native passthrough for OpenAI-compatible providers", + "sourceUrl": "https://docs.litellm.ai/docs/anthropic_unified/", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Endpoint-family breadth: chat, completions, responses, batches, files, assistants, images, audio, rerank, RAG, realtime, OCR, containers, fine-tuning, A2A, search", + "sourceUrl": "https://docs.litellm.ai/release_notes", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Realtime WebSocket relay at /v1/realtime across eight providers with protocol-version bridging and reduced default event logging", + "sourceUrl": "https://docs.litellm.ai/docs/realtime", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Rerank normalized to Cohere shape at /rerank across 12+ providers", + "sourceUrl": "https://docs.litellm.ai/docs/rerank", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Files/batches translation across seven providers with routing metadata embedded in resource IDs (no DB required)", + "sourceUrl": "https://docs.litellm.ai/docs/files_endpoints", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Audio transcription and image generation as mode-tagged model_list entries with fallbacks riding the request", + "sourceUrl": "https://docs.litellm.ai/docs/audio_transcription", + "gap": "context" + }, + { + "area": "auth", + "feature": "Upstream auth chains: Azure Entra with enable_azure_ad_token_refresh, Vertex service accounts/Workload Identity Federation, Bedrock full boto3 chain + AssumeRole with session tags + bearer API keys", + "sourceUrl": "https://docs.litellm.ai/docs/providers/bedrock", + "gap": "4" + }, + { + "area": "auth", + "feature": "Enterprise secret managers (AWS/Azure/GCP/HashiCorp/CyberArk/KMS) with store_virtual_keys and per-team settings", + "sourceUrl": "https://docs.litellm.ai/docs/secret_managers/overview.md", + "gap": "15" + }, + { + "area": "ops", + "feature": "NaCl SecretBox at-rest encryption of provider keys (LITELLM_SALT_KEY); spend/audit logs remain plaintext", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/security_encryption_faq.md", + "gap": "15" + }, + { + "area": "ops", + "feature": "Entity lifecycle audit logs with before/after values and async S3 export (Enterprise)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/audit_logs", + "gap": "context" + }, + { + "area": "caching", + "feature": "Prompt-cache marker translation (cache_control -> Bedrock cachePoint / OpenAI prompt_cache_breakpoint / Gemini cachedContents), injection points, one-flag Anthropic auto-caching, 300s prefix+tools deployment pinning", + "sourceUrl": "https://docs.litellm.ai/docs/tutorials/prompt_caching", + "gap": "14" + }, + { + "area": "protocol", + "feature": "/v1/messages/count_tokens auto-routed to provider-native counters (Anthropic, OpenAI Responses input_tokens, Bedrock, Vertex, Gemini)", + "sourceUrl": "https://docs.litellm.ai/docs/anthropic_count_tokens", + "gap": "6" + }, + { + "area": "cost", + "feature": "/spend/calculate pre-call estimate and post-call cost endpoint", + "sourceUrl": "https://github.com/BerriAI/litellm/blob/main/litellm/proxy/spend_tracking/spend_management_endpoints.py", + "gap": "2" + }, + { + "area": "observability", + "feature": "Prometheus: request/latency/TTFT/queue/overhead histograms, deployment-state gauges, budget-remaining gauges, admission counters, cardinality controls, optional dedicated metrics port", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/prometheus", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "~60 logging integrations (Langfuse, Datadog, OTel v1/v2, Sentry, S3/GCS/SQS/DynamoDB raw stores, custom CustomLogger from instance/S3/ConfigMap)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/logging", + "gap": "11" + }, + { + "area": "observability", + "feature": "StandardLoggingPayload schema with TTFT, cache_hit, full error_information; message redaction and per-request logging opt-outs (no-log, turn_off_message_logging)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/logging_spec", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTel export with traceparent propagation and forward_traceparent_to_llm_provider", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/logging", + "gap": "11" + }, + { + "area": "observability", + "feature": "Alerting across ~20 types (outages, region outages, hanging/slow calls, cooldown, budget, DB exceptions) with digest mode", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/alerting", + "gap": "context" + }, + { + "area": "ops", + "feature": "Streaming SLO mechanics: SSE : ping keepalives (keepalive_seconds 1-300), cancel_on_disconnect (499), always_include_stream_usage", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/config_settings", + "gap": "context" + }, + { + "area": "ops", + "feature": "Production topology: Postgres + Redis 7 multi-instance state, one-worker-per-pod, job-role split, DB-degradation serving mode, 30s config hot-reload via store_model_in_db", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/prod", + "gap": "context" + }, + { + "area": "ops", + "feature": "YAML-bootstrap-then-DB-overlay config precedence with Admin UI/API writes; credential_list YAML-only", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/configs", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Admin UI (/ui) with key/team/budget/model management, SSO, login rate limiting; model-compare playground (3-way, TTFT/tokens/cost) since v1.80; LiteLLM Lens trace UI (Oct 2026)", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/ui", + "gap": "19" + }, + { + "area": "extensibility", + "feature": "Full request-lifecycle proxy hooks (modify/reject/stream/header/model-filter), custom auth handler, key-policy hooks, per-request callback disabling", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/call_hooks", + "gap": "context" + }, + { + "area": "lifecycle", + "feature": "Model deprecation management: cost-map deprecation_date on 874 entries, /model/deprecations urgency buckets, daily-lock Slack alerts — never blocks", + "sourceUrl": "https://github.com/BerriAI/litellm/blob/main/litellm/proxy/common_utils/model_deprecation.py", + "gap": "18" + }, + { + "area": "protocol", + "feature": "Rust gateway core (beta): per-model rust:true runs /v1/messages translation in-process with Python fallback; standalone Axum binary mode", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/rust_gateway", + "gap": "context" + }, + { + "area": "cost", + "feature": "Auto client tagging: User-Agent added as request tag (Claude Code, Gemini CLI detection) plus extra_spend_tag_headers", + "sourceUrl": "https://docs.litellm.ai/docs/proxy/cost_tracking", + "gap": "2" + } + ] + }, + { + "name": "llm-api-key-proxy (Mirrowel)", + "category": "reference checkout (credential rotator proxy)", + "language": "Python", + "status": "active (pinned ref/llm-api-key-proxy at af40b91, 2026-05-27)", + "url": "https://github.com/Mirrowel/LLM-API-Key-Proxy", + "capabilities": [ + { + "area": "auth", + "feature": "Multi-credential pools per provider with balanced (weighted least-used by priority tier) and sequential (cache-locality-sticky) rotation", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/usage/selection/strategies/balanced.py", + "gap": "5" + }, + { + "area": "rate-limiting", + "feature": "Escalating per-model cooldowns (10/30/60/120/300 s), auth-error global lockouts, revoked-key detection, Retry-After parsing incl. compound durations, quota windows (rolling/daily/calendar/API-authoritative) with model quota groups", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/config/defaults.py", + "gap": "3;5" + }, + { + "area": "rate-limiting", + "feature": "Fair-cycle credential exhaustion policy (blocked keys released only when the whole pool is exhausted) and per-credential soft-optimal/hard-max concurrency limits", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/usage/limits/fair_cycle.py", + "gap": "3;5" + }, + { + "area": "routing", + "feature": "Sticky session routing by conversation fingerprint (TTL 1h, optional disk persistence) to preserve provider prompt-cache hits; small-cooldown same-key retry", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/session_tracking.py", + "gap": "14" + }, + { + "area": "auth", + "feature": "OAuth credential lifecycle: browser flows, background refresher (600 s) with provider-specific quota-baseline jobs, duplicate-account detection", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/background_refresher.py", + "gap": "new:oauth-credential-lifecycle" + }, + { + "area": "observability", + "feature": "Quota/usage admin API (/v1/quota-stats with force_refresh) plus interactive quota-viewer TUI", + "sourceUrl": "ref/llm-api-key-proxy/src/proxy_app/main.py", + "gap": "context" + }, + { + "area": "cost", + "feature": "Models.dev-sourced per-model pricing + capability metadata enriching /v1/models; /v1/token-count and /v1/cost-estimate endpoints", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/model_info_service.py", + "gap": "2" + }, + { + "area": "protocol", + "feature": "Stream retry-safety policy: retry only before first emitted chunk, or provably reasoning-only chunks when enabled (ambiguous fails closed)", + "sourceUrl": "ref/llm-api-key-proxy/src/rotator_library/client/stream_retry_policy.py", + "gap": "1" + }, + { + "area": "observability", + "feature": "Per-request forensic logging artifacts (request.json, streaming_chunks.jsonl) and raw I/O capture", + "sourceUrl": "ref/llm-api-key-proxy/README.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Embeddings endpoint with transparent micro-batching (100 ms / 64 inputs, fanned back per-item)", + "sourceUrl": "ref/llm-api-key-proxy/src/proxy_app/batch_manager.py", + "gap": "context" + } + ] + }, + { + "name": "LLMG (modpotatodotdev)", + "category": "Rust provider-unification library + gateway", + "language": "Rust", + "status": "active (pushed 2026-04-02)", + "url": "https://github.com/modpotatodotdev/LLMG", + "capabilities": [ + { + "area": "protocol", + "feature": "70+ provider registry auto-registered from environment variables, compile feature-gated, provider/model routing syntax with short aliases", + "sourceUrl": "https://modpotatodotdev.github.io/LLMG/providers/all/", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Embeddable as a Rust library (llmg-core/llmg-providers crates) with RoutingProvider usable in-process, incl. Rig agent-framework integration", + "sourceUrl": "https://raw.githubusercontent.com/modpotatodotdev/LLMG/develop/README.md", + "gap": "context" + } + ] + }, + { + "name": "Martian", + "category": "hosted model gateway + router SaaS", + "language": "n/a (SaaS)", + "status": "active (gateway live 2026-10-27)", + "url": "https://docs.withmartian.com/", + "capabilities": [ + { + "area": "protocol", + "feature": "Multi-protocol hosted gateway: /v1/chat/completions, /v1/responses, /v1/messages plus GET /v1/models usable with stock OpenAI/Anthropic SDKs; martian_metadata extension", + "sourceUrl": "https://docs.withmartian.com/api-reference/endpoints", + "gap": "6" + }, + { + "area": "routing", + "feature": "Live 312-model catalog across 49 providers with per-token USD pricing (incl. cache read/write, image, request, web_search, reasoning) and reliability tiers, refreshed every 5 minutes", + "sourceUrl": "https://api.withmartian.com/v1/models", + "gap": "context" + }, + { + "area": "routing", + "feature": "Trainable router platform: Router + Judge resources, RoutersClient.run_training_job, routing constraints (CostConstraint or QualityConstraint picking a Pareto point)", + "sourceUrl": "https://withmartian.github.io/martian-sdk-python/", + "gap": "13" + }, + { + "area": "caching", + "feature": "Anthropic prompt-caching pass-through (cache_control ephemeral breakpoints with 1024-token minimums and usage fields)", + "sourceUrl": "https://docs.withmartian.com/api-reference/advanced", + "gap": "14" + }, + { + "area": "cost", + "feature": "Dashboard real-time usage/cost/request history; prepaid credits with 403 on exhaustion", + "sourceUrl": "https://docs.withmartian.com/quickstart", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Documented 429/Retry-After semantics and upstream-503 error mapping", + "sourceUrl": "https://docs.withmartian.com/api-reference/errors", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Integrations: OpenAI/Anthropic/Vercel SDKs, Claude Code/Cursor/Cline/Codex/OpenCode; LiteLLM integration delegates routing/fallback/limits to LiteLLM", + "sourceUrl": "https://docs.withmartian.com/integrations/litellm", + "gap": "context" + } + ] + }, + { + "name": "mcp-gateway (MikkoParkkola)", + "category": "MCP tool gateway (ideas reference; PolyForm Noncommercial)", + "language": "Rust", + "status": "very active (last commit 2026-11-02; single ~12 MB binary; PolyForm NC 1.0.0 — no code reuse)", + "url": "https://github.com/MikkoParkkola/mcp-gateway", + "capabilities": [ + { + "area": "mcp", + "feature": "Meta-tool MCP gateway surface with on-demand tool discovery (gateway_search_tools/gateway_invoke or 2-tool Code Mode) over stdio/StreamableHTTP/SSE/WebSocket backends and REST APIs; A2A adapter", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md", + "gap": "8" + }, + { + "area": "guardrails", + "feature": "Fail-closed tool-poisoning and prompt-injection validator scanning every backend tool description; response-injection scanner; JSON-RPC -32002 firewall", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/security-controls.toml", + "gap": "7;8" + }, + { + "area": "guardrails", + "feature": "SHA-256 capability pinning with rug-pull quarantine (110+ capability YAMLs; pinned-file mismatch fails closed; changed tool definitions unload-and-quarantine)", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md", + "gap": "8" + }, + { + "area": "auth", + "feature": "End-user identity propagation: gateway-signed ES256 assertions, client-token passthrough, RFC 8693 token exchange — all fail-closed; per-user OAuth isolation; RFC 8707 resource indicators", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/CHANGELOG.md", + "gap": "new:caller-sso-rbac" + }, + { + "area": "routing", + "feature": "Per-slot circuit breakers (per backend and per-user session slot) plus tunable business-logic error-budget kill switch revivable only via meta-tool; 429 excluded from budget accounting", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/ARCHITECTURE.md", + "gap": "1" + }, + { + "area": "rate-limiting", + "feature": "Per-client rate limiting with burst bucket (token bucket per caller key/API-key)", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml", + "gap": "3" + }, + { + "area": "cost", + "feature": "Cost governance with per-key/per-tool/daily budgets and blocking tiers (50% log / 80% notify / 100% block), anti-tamper costs.json, gateway_cost_report meta-tool", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml", + "gap": "2" + }, + { + "area": "caching", + "feature": "SHA-256-keyed response cache plus three-state idempotency cache (in-flight -> completed, TTLs, failure eviction) and prompt_cache_key preservation", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/ARCHITECTURE.md", + "gap": "10" + }, + { + "area": "ops", + "feature": "~8 ms hot reload of backends and config without dropping connections (file watch + reload meta-tools + transition tracker)", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/README.md", + "gap": "16" + }, + { + "area": "auth", + "feature": "Key server: SHA-256 hashed expiring API keys with OIDC, policies, admin token; single-use dashboard bootstrap link; agent credentials", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/gateway.example.yaml", + "gap": "9" + }, + { + "area": "observability", + "feature": "Hash-chain transparency audit log of tool invocations, tested Prometheus alert rules, gateway_get_stats meta-tool", + "sourceUrl": "https://github.com/MikkoParkkola/mcp-gateway/releases", + "gap": "12" + }, + { + "area": "guardrails", + "feature": "SSRF guard with resolves-then-pins DNS rebinding defense, special-use IP blocking, cleartext-credential refusal at config load, Origin/Host cross-site validation, mTLS, HMAC+nonce signing", + "sourceUrl": "https://raw.githubusercontent.com/MikkoParkkola/mcp-gateway/main/security-controls.toml", + "gap": "context" + }, + { + "area": "ops", + "feature": "Full deployment stack: Docker incl. -full image, Helm, K8s manifests, Prometheus; setup wizard importing Claude/Cursor/Zed/Continue/Codex configs, doctor, web UI/dashboard", + "sourceUrl": "https://github.com/MikkoParkkola/mcp-gateway/tree/main/deploy", + "gap": "19" + } + ] + }, + { + "name": "mistral.rs (EricLBuehler)", + "category": "Rust LLM inference engine (dual-protocol serve)", + "language": "Rust", + "status": "active (v0.9.5 + 27 commits; last commit 2026-11-03)", + "url": "https://github.com/EricLBuehler/mistral.rs", + "capabilities": [ + { + "area": "protocol", + "feature": "Side-by-side OpenAI /v1 (chat, completions, embeddings, responses) and Anthropic /v1/messages + count_tokens in one server (AnthropicMessagesRequest converted through the OpenAI hub)", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/anthropic.rs", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Full canonical Anthropic SSE event machine plus custom agentic_tool_call_progress / agentic_tool_approval_required / file_produced events", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/anthropic.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Anthropic error-envelope fidelity: typed statuses incl. 409/529, exact prompt-is-too-long wording, generic messages hiding internal error detail", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/serve/anthropic-messages-api.md", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Agent permission system (auto/ask/deny) with streamed approval events, /v1/agent/approvals/{id}, remember_for_session, tighten-only ratchet", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/permissions-and-approvals.mdx", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Per-request context limits (min of model/paged-KV/--max-model-len) reported on /v1/models and enforced with per-API typed overflow errors", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/commit/master", + "gap": "context" + }, + { + "area": "caching", + "feature": "PagedAttention KV prefix caching (incl. multimodal) default-on for Metal plus in-situ re-quantization endpoint /re_isq", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs#what-makes-it-fast", + "gap": "14" + }, + { + "area": "routing", + "feature": "Model-name routing with default and aliases plus runtime load/unload/reload endpoints; requests to unloaded models wait out an automatic reload", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/serve/multiple-models.mdx", + "gap": "context" + }, + { + "area": "mcp", + "feature": "MCP client: Process/HTTP/WebSocket transports, per-server bearer tokens and headers, tool_prefix namespacing, per-call timeouts, max_concurrent_calls", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-mcp/README.md", + "gap": "8" + }, + { + "area": "mcp", + "feature": "MCP server mode: second listener serving JSON-RPC 2.0 at POST /mcp exposing the served model as a chat tool", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/expose-as-mcp.md", + "gap": "8" + }, + { + "area": "observability", + "feature": "Prometheus /metrics with TTFT/inter-token/queue histograms (vLLM-derived buckets), in-flight gauge, and client-disconnect outcome detection via observed body wrapper", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/metrics.rs", + "gap": "11;12;17" + }, + { + "area": "observability", + "feature": "Request-ID generation/echo incl. Anthropic-convention request-id header; JSON access logs with housekeeping-route exclusion and bounded error bodies", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-cli/src/args/server.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Endpoint breadth: Responses with background jobs + cancel, embeddings, audio speech/transcriptions/translations, image gen, Files API incl. containers, /v1/skills, /v1/sessions, LoRA adapter mgmt, /v1/system/info|doctor", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/route_registry.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Stateful server-side sessions incl. live Python interpreter state, export/import/delete, 30-min idle expiry, LRU 128, no-ID prefix heuristic", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/docs/src/content/docs/guides/agents/persist-sessions.mdx", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Embeddable MistralRsServerRouterBuilder (mount all routes in your own axum app) plus self-hosted OpenAPI/Swagger docs", + "sourceUrl": "https://github.com/EricLBuehler/mistral.rs/blob/master/mistralrs-server-core/src/lib.rs", + "gap": "context" + } + ] + }, + { + "name": "Model Context Protocol (specification)", + "category": "protocol specification", + "language": "n/a (specification)", + "status": "active (2025-11-25 stable; 2026-07-28 release candidate)", + "url": "https://modelcontextprotocol.io/specification/2025-11-25/basic/transports", + "capabilities": [ + { + "area": "protocol", + "feature": "Standard transports: stdio plus Streamable HTTP (single /mcp endpoint, optional SSE, Mcp-Session-Id, Last-Event-ID redelivery, mandatory Origin validation, MCP-Protocol-Version header)", + "sourceUrl": "https://modelcontextprotocol.io/specification/2025-11-25/basic/transports", + "gap": "context" + }, + { + "area": "auth", + "feature": "Spec-level OAuth 2.1 resource-server auth: RFC 9728 metadata, audience-bound RFC 8707 resource parameter, PKCE S256, 403 insufficient_scope step-up, no token passthrough", + "sourceUrl": "https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization", + "gap": "context" + }, + { + "area": "protocol", + "feature": "2026-07-28 revision: stateless operation (sessions removed), _meta capabilities + server/discover, Mcp-Method/Mcp-Name routing headers, list ttlMs/cacheScope hints, W3C trace context keys, MCP Apps sandboxed UIs", + "sourceUrl": "https://blog.modelcontextprotocol.io/posts/2026-07-28-release-candidate", + "gap": "context" + } + ] + }, + { + "name": "new-api (QuantumNous)", + "category": "open-source gateway (one-api fork)", + "language": "Go (+ React)", + "status": "active (v1.0.0-rc line, near-weekly; rc.47 Oct 2026)", + "url": "https://github.com/QuantumNous/new-api", + "capabilities": [ + { + "area": "protocol", + "feature": "RelayKit four-protocol cross-conversion (OpenAI Chat, Responses, Anthropic Messages, Gemini) plus realtime WebSocket, images, audio, embeddings, rerank, task plugins", + "sourceUrl": "https://github.com/QuantumNous/new-api", + "gap": "6" + }, + { + "area": "routing", + "feature": "Channel priority/weight routing, retries, channel affinity with per-model tracking, multi-key upstreams with polling, auto-disable/recovery", + "sourceUrl": "https://docs.newapi.ai/en/docs/guide/console/channel-management", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Request Policies engine (checks, sessions/retries, channel health) with decision records in logs", + "sourceUrl": "https://github.com/QuantumNous/new-api/releases", + "gap": "1" + }, + { + "area": "routing", + "feature": "Inline model modifiers (@thinking:on, @effort:high etc.) affecting routing and billing with canonical billing identities", + "sourceUrl": "https://github.com/QuantumNous/new-api/releases?page=3", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Per-IP minute/hour/day limits, per-group [rpm, rph] pairs, model-level total/successful-request limits; Redis-backed with per-node independence caveat", + "sourceUrl": "https://docs.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced", + "gap": "3" + }, + { + "area": "auth", + "feature": "OIDC/OAuth/passkeys/TOTP logins, Argon2id passwords, optional RSA-OAEP login-body encryption; scoped named access tokens for automation", + "sourceUrl": "https://github.com/QuantumNous/new-api", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "Per-key quota, expiry, model restrictions, IP whitelist, per-token quota query toggle", + "sourceUrl": "https://docs.newapi.ai/en/docs/guide/feature-guide", + "gap": "9;3" + }, + { + "area": "cost", + "feature": "Token-ratio billing plus fixed prices, expression-based tiered/conditional pricing, prompt-cache-hit billing at configurable ratio, subscription + prepaid wallet economy, redemption codes", + "sourceUrl": "https://docs.newapi.ai/en/docs/guide/feature-guide/admin/system-setting-advanced", + "gap": "2" + }, + { + "area": "guardrails", + "feature": "Sensitive-word screening of prompts and/or responses (off by default since rc.45)", + "sourceUrl": "https://github.com/QuantumNous/new-api/blob/main/model/request_policy.go", + "gap": "7" + }, + { + "area": "observability", + "feature": "Usage logs with stream status/timing and conditional-billing highlights, CSV export with admin caps, channel health monitoring and model performance dashboards, audit log with step-up verification", + "sourceUrl": "https://docs.newapi.ai/en/docs/installation/config-maintenance/environment-variables", + "gap": "context" + }, + { + "area": "observability", + "feature": "Continuous profiling hooks (Pyroscope URL, pprof on :8005)", + "sourceUrl": "https://docs.newapi.ai/en/docs/installation/config-maintenance/environment-variables", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Sandboxed JavaScript task-plugin system with marketplace (8 MiB source cap; experimental, not production-recommended)", + "sourceUrl": "https://github.com/QuantumNous/new-api/releases", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Built-in /v1/tools/search and /v1/tools/fetch endpoints billed per call (rc.45)", + "sourceUrl": "https://github.com/QuantumNous/new-api/releases", + "gap": "context" + }, + { + "area": "ops", + "feature": "Deployment limits and hardening: 128 MB post-decompression body cap, 64 MB stream scanner, streaming timeout, SQLite WAL, graceful shutdown, per-channel TLS-verify skip", + "sourceUrl": "https://docs.newapi.ai/en/docs/installation/config-maintenance/environment-variables", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Per-channel parameter-override engine (15 operations, conditional AND/OR rules, JSON-path targets)", + "sourceUrl": "https://docs.newapi.ai/en/docs/guide/console/channel-management", + "gap": "context" + }, + { + "area": "ops", + "feature": "Docker/Compose multi-node with shared SESSION_SECRET/CRYPTO_SECRET; separate log DB (MySQL/Postgres/ClickHouse via LOG_SQL_DSN)", + "sourceUrl": "https://github.com/QuantumNous/new-api", + "gap": "context" + } + ] + }, + { + "name": "Not Diamond", + "category": "routing-decision SaaS + prompt optimization", + "language": "n/a (SaaS; Go/TS helper libraries)", + "status": "active", + "url": "https://docs.notdiamond.ai/", + "capabilities": [ + { + "area": "routing", + "feature": "Per-query routing-decision API (/v2/modelRouter/modelSelect) returning the best-suited model from candidates — caller executes the call", + "sourceUrl": "https://docs.notdiamond.ai/reference/token_model_select_v2_modelrouter_modelselect_post.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Cost/latency/quality tradeoff controls: tradeoff=cost|latency or continuous 0-10 cost_quality_tradeoff blend (Pareto selection)", + "sourceUrl": "https://docs.notdiamond.ai/docs/key-concepts.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Custom router training from evaluation CSVs (min 15 samples) producing reusable preference_id; works for arbitrary custom endpoints", + "sourceUrl": "https://docs.notdiamond.ai/docs/router-training-quickstart.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Automated prompt optimization pipeline (start/status/results/costs endpoints) producing per-model optimized prompts", + "sourceUrl": "https://docs.notdiamond.ai/docs/what-is-prompt-optimization.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "OpenAI-compatible BYO-key proxy (notdiamond-proxy) with seed-activated response caching and self-deploy wrappers (Vercel/Workers/Lambda/Express)", + "sourceUrl": "https://github.com/Not-Diamond/notdiamond-proxy", + "gap": "10" + }, + { + "area": "routing", + "feature": "go-notdiamond client transport: ordered fallback, weighted LB, per-model retries/timeouts, rolling-latency fallback (requires Redis)", + "sourceUrl": "https://github.com/Not-Diamond/go-notdiamond", + "gap": "1" + }, + { + "area": "cost", + "feature": "/v2/models catalog with context lengths and per-million USD input/output prices; per-run cost accounting for optimization jobs", + "sourceUrl": "https://docs.notdiamond.ai/reference/list_models_v2_models_get.md", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Flat 50 requests/second platform limit with documented 429/Retry-After and backoff guidance", + "sourceUrl": "https://docs.notdiamond.ai/docs/rate-limits.md", + "gap": "context" + }, + { + "area": "auth", + "feature": "SOC-2/ISO-27001 posture, TLS 1.3, AES-256 at rest, optional content hashing for privacy on modelSelect", + "sourceUrl": "https://docs.notdiamond.ai/docs/privacy-security-and-local-deployments.md", + "gap": "context" + } + ] + }, + { + "name": "oc-go-cc (samueltuyizere)", + "category": "reference checkout (Claude Code -> OpenCode Go/Zen proxy)", + "language": "Go", + "status": "active (pinned ref/oc-go-cc at 99d756c, 2026-06-02)", + "url": "https://github.com/samueltuyizere/oc-go-cc", + "capabilities": [ + { + "area": "routing", + "feature": "Request-path model fallback chains (primary + configured fallbacks) with attempt counts", + "sourceUrl": "ref/oc-go-cc/internal/router/fallback.go", + "gap": "1" + }, + { + "area": "routing", + "feature": "Per-model three-state circuit breakers (3 failures / 30 s recovery / capped half-open probes) exposed via /health", + "sourceUrl": "ref/oc-go-cc/internal/router/fallback.go", + "gap": "1" + }, + { + "area": "routing", + "feature": "Retryable-error classification (timeout, connection refused/reset, rate limit, 429/500/502/503) so non-retryable errors fail fast", + "sourceUrl": "ref/oc-go-cc/internal/router/fallback.go", + "gap": "1" + }, + { + "area": "routing", + "feature": "Scenario-based content routing (long_context > complex > think > background > default) with token-threshold trigger and streaming TTFT downgrade", + "sourceUrl": "ref/oc-go-cc/internal/router/scenarios.go", + "gap": "13" + }, + { + "area": "protocol", + "feature": "Streaming keepalive heartbeat (3 s :keepalive), mid-stream chain fallback before completion, and in-band SSE error events after headers", + "sourceUrl": "ref/oc-go-cc/internal/handlers/messages.go", + "gap": "1" + }, + { + "area": "ops", + "feature": "Config hot reload via fsnotify + SIGHUP with debouncing and runtime log-level change", + "sourceUrl": "ref/oc-go-cc/internal/config/watcher.go", + "gap": "16" + }, + { + "area": "observability", + "feature": "p95/p99 latency percentiles from a 1000-sample ring plus full circuit-breaker states at /health", + "sourceUrl": "ref/oc-go-cc/internal/metrics/metrics.go", + "gap": "17" + } + ] + }, + { + "name": "ollamaMQ (Chleba)", + "category": "queue dispatcher / load balancer for local inference", + "language": "Rust", + "status": "active (v0.5.0 2026-09-27, 0.5.1 bump; last commit 2026-10-30)", + "url": "https://github.com/Chleba/ollamaMQ", + "capabilities": [ + { + "area": "routing", + "feature": "Per-user fair-share scheduling with exponentially decaying scores (300 s half-life), charged at dispatch regardless of outcome", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "new:admission-queueing" + }, + { + "area": "rate-limiting", + "feature": "Per-user FIFO queues with multi-layer bounded admission (100/user 429, 512 MiB global queued-body 503, 64 MiB body 413); metadata reads never queue; idle-user pruning", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "new:admission-queueing" + }, + { + "area": "routing", + "feature": "Least-connections balancing with global per-backend and per-model concurrency caps keyed on publisher/quant-stripped names", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "3;13" + }, + { + "area": "routing", + "feature": "Warm-model state-affinity routing: backends with the model loaded in RAM first, then cold backends, warned eviction last resort", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "new:engine-state-routing" + }, + { + "area": "protocol", + "feature": "API-family detection with learning (family excluded after two rejections), deterministic model matching (exact/tag/publisher/@quant), strict model pinning with 503 after stuck_timeout", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "Model load/unload control via TUI keys and admin HTTP API (GET /admin/models, POST /admin/models/load|unload) with per-backend serialization", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/control.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "Declarative model-residency config (name/identifier/max_ctx/keep_alive/backends) applied additively at startup and on live TUI reload", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/appconf.yaml.example", + "gap": "16" + }, + { + "area": "observability", + "feature": "Live ~10 Hz ratatui dashboard doubling as runtime queue-admin console (VIP, Boost, block user/IP, load/unload keys) with request-log overlay", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/tui.rs", + "gap": "context" + }, + { + "area": "auth", + "feature": "Constant-time single-key auth (X-API-Key/Bearer) plus per-backend egress token substitution overriding client Authorization", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs", + "gap": "context" + }, + { + "area": "routing", + "feature": "Background health probing (10 s, per-backend endpoint memory) marking offline backends excluded from routing", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/control.rs", + "gap": "1" + }, + { + "area": "routing", + "feature": "Fail-fast stuck_timeout 503 distinguishing unsatisfiable (pinned, no holder) from merely-busy; dispatcher scans past unroutable queue heads", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/dispatcher.rs", + "gap": "context" + }, + { + "area": "observability", + "feature": "Size-rotated JSONL request/response logging (64 KiB content cap, headers never logged) surfaced in the TUI viewer", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Aggregated /api/tags and /v1/models across backends; full Ollama + OpenAI + System One endpoint surface with opt-in catch-all passthrough (byte-for-byte, no translation)", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/blob/master/src/main.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "Docker/compose/systemd packaging and poison-tolerant mutex recovery (~150 lock sites)", + "sourceUrl": "https://github.com/Chleba/ollamaMQ/tree/master", + "gap": "context" + } + ] + }, + { + "name": "one-api (songquanpeng)", + "category": "open-source gateway", + "language": "Go (+ React)", + "status": "dormant (v0.6.10, 2025-02-02; last push 2026-01)", + "url": "https://github.com/songquanpeng/one-api", + "capabilities": [ + { + "area": "routing", + "feature": "Multi-channel load balancing with automatic retry on failure; admin-token channel pinning (KEY-CHANNEL_ID suffix); retry across channel priorities since v0.6.1", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.md", + "gap": "1" + }, + { + "area": "auth", + "feature": "Tokens with expiration, quota, allowed IP ranges, allowed models; group multipliers; OAuth/OIDC logins; INITIAL_ROOT_TOKEN bootstrap", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.md", + "gap": "9" + }, + { + "area": "cost", + "feature": "Ratio-based quota billing (group x model x completion ratio) with vouchers/invites, USD display, batch redemption", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.md", + "gap": "2" + }, + { + "area": "rate-limiting", + "feature": "GLOBAL_API_RATE_LIMIT / GLOBAL_WEB_RATE_LIMIT per-IP fixed windows that exclude the relay data path", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.en.md", + "gap": "3" + }, + { + "area": "ops", + "feature": "Scheduled channel testing, balance polling, success-rate auto-disable (threshold 0.8)", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.md", + "gap": "1" + }, + { + "area": "caching", + "feature": "Redis/memory config caching and batched DB quota writes for horizontal scale (config/quota caching, not response caching)", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/README.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Admin REST API via root access token; multi-machine master/slave deployment with SYNC_FREQUENCY", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/docs/API.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "OpenAI-format relay surface: chat, completions, edits, embeddings, images/generations, audio trio, moderations; files/assistants/fine-tuning are stubs", + "sourceUrl": "https://github.com/songquanpeng/one-api/blob/main/router/relay.go", + "gap": "context" + } + ] + }, + { + "name": "OpenAI platform", + "category": "provider API", + "language": "n/a (hosted API)", + "status": "active", + "url": "https://developers.openai.com/api/docs/deprecations", + "capabilities": [ + { + "area": "lifecycle", + "feature": "Deprecation policy with per-category notice windows (>=6 months GA, 3 months specialized, ~2 weeks preview) and optional machine-readable shutdown_date on /v1/models", + "sourceUrl": "https://developers.openai.com/api/docs/deprecations", + "gap": "18" + }, + { + "area": "caching", + "feature": "Implicit prompt caching from 1,024 tokens plus GPT-5.6+ explicit mode: prompt_cache_options (mode/ttl 30m|12h/prewarm), up to 4 prompt_cache_breakpoint markers, prompt_cache_key, cached_tokens/cache_write_tokens usage", + "sourceUrl": "https://developers.openai.com/api/docs/guides/prompt-caching", + "gap": "14" + } + ] + }, + { + "name": "OpenLIT", + "category": "OTel instrumentation SDK + self-hosted platform", + "language": "Python/TypeScript/Go SDKs; platform self-hosted", + "status": "active", + "url": "https://github.com/openlit/openlit", + "capabilities": [ + { + "area": "observability", + "feature": "openlit.init() auto-instrumentation for 70+ LLMs/frameworks exporting OTel traces/metrics under gen_ai.* semconv to any OTLP backend", + "sourceUrl": "https://github.com/openlit/openlit", + "gap": "11" + }, + { + "area": "observability", + "feature": "Self-hosted platform (Docker/K8s, ClickHouse) with cost monitoring across models/providers/users/sessions/agents and GPU collectors", + "sourceUrl": "https://github.com/openlit/openlit", + "gap": "11" + }, + { + "area": "observability", + "feature": "Coding-agent observability with zero code changes (claude-code/cursor/codex/opencode vendors) plus evals, guardrails, prompt hub, rule engine", + "sourceUrl": "https://github.com/openlit/openlit", + "gap": "11" + } + ] + }, + { + "name": "OpenPipe", + "category": "fine-tuning/logging platform (sunset)", + "language": "TypeScript (OSS repo)", + "status": "sunset — migrated to Weights & Biases/CoreWeave; docs.openpipe.ai offline; OSS development paused", + "url": "https://openpipe.ai/security", + "capabilities": [ + { + "area": "protocol", + "feature": "OpenAI-compatible proxy with server-side provider translation by model-name prefix (anthropic:, gemini:, external models) and metadata-tagged logging", + "sourceUrl": "https://web.archive.org/web/20250208230854/https://docs.openpipe.ai/features/chat-completions/anthropic", + "gap": "6" + }, + { + "area": "caching", + "feature": "Exact-match response caching (beta) via client cache option or op-cache header", + "sourceUrl": "https://web.archive.org/web/20250117152805/https://docs.openpipe.ai/features/caching", + "gap": "10" + }, + { + "area": "routing", + "feature": "Client-side fallback: fallback model in the openpipe param with its own timeout or a custom fallback client", + "sourceUrl": "https://web.archive.org/web/20250117132259/https://docs.openpipe.ai/features/fallback", + "gap": "1" + }, + { + "area": "tooling", + "feature": "OSS repo: OpenAI-SDK logging client, request-log querying, JSONL datasets, fine-tuning (GPT-3.5/Mistral/Llama) with hosted deployment or weight download", + "sourceUrl": "https://github.com/openpipe/openpipe", + "gap": "context" + } + ] + }, + { + "name": "openproxy (x5iu)", + "category": "open-source gateway", + "language": "Rust", + "status": "active (pushed 2026-04-28)", + "url": "https://github.com/x5iu/openproxy", + "capabilities": [ + { + "area": "routing", + "feature": "Weighted load balancing with priority tiers, fallback, background health checks with automatic recovery, and connection pooling", + "sourceUrl": "https://raw.githubusercontent.com/x5iu/openproxy/master/README.md", + "gap": "1;13" + }, + { + "area": "auth", + "feature": "Dynamic per-request credential fetch via shell templates (1Password op read, AWS Secrets Manager, files) and weighted upstream key pools; Anthropic OAuth bearer", + "sourceUrl": "https://raw.githubusercontent.com/x5iu/openproxy/master/README.md", + "gap": "5" + }, + { + "area": "protocol", + "feature": "Transparent WebSocket proxying (HTTP/1.1 upgrade and HTTP/2 Extended CONNECT) covering OpenAI Realtime, plus opt-in HTTP CONNECT forward-proxy tunnels with Proxy-Authorization", + "sourceUrl": "https://raw.githubusercontent.com/x5iu/openproxy/master/README.md", + "gap": "6" + }, + { + "area": "ops", + "feature": "rustls TLS 1.3 HTTPS listener with HTTP/2 negotiation, YAML config hot-reload on SIGHUP, zero-downtime binary upgrade on SIGUSR2, PID file and systemd recipe", + "sourceUrl": "https://raw.githubusercontent.com/x5iu/openproxy/master/README.md", + "gap": "16" + } + ] + }, + { + "name": "OpenRouter", + "category": "hosted model marketplace/router", + "language": "n/a (SaaS)", + "status": "active", + "url": "https://openrouter.ai/docs", + "capabilities": [ + { + "area": "routing", + "feature": "Request-level provider routing preferences: order, allow_fallbacks, only/ignore, sort (price/throughput/latency), max_price hard block, preferred latency/throughput percentiles", + "sourceUrl": "https://openrouter.ai/docs/guides/routing/provider-selection.md", + "gap": "13" + }, + { + "area": "routing", + "feature": "Default price-weighted provider load balancing (inverse-square of price) with 30-second outage exclusion", + "sourceUrl": "https://openrouter.ai/docs/guides/routing/provider-selection.md", + "gap": "13" + }, + { + "area": "routing", + "feature": "Model fallbacks via models array (any error advances; billed at the serving model) and Anthropic-shaped fallbacks (max 3) on /api/v1/messages", + "sourceUrl": "https://openrouter.ai/docs/guides/routing/model-fallbacks.md", + "gap": "1" + }, + { + "area": "rate-limiting", + "feature": "Credits-tiered free-model limits (20 RPM; 50 or 1000 RPD) with X-RateLimit-* headers, Retry-After, and mid-stream SSE error finish_reason", + "sourceUrl": "https://openrouter.ai/docs/api_reference/limits.md", + "gap": "3" + }, + { + "area": "auth", + "feature": "Management-key API for key CRUD (USD limits, daily/weekly/monthly resets, expiry, workspace scoping); inference and admin keys are distinct credential classes", + "sourceUrl": "https://openrouter.ai/docs/guides/overview/auth/management-api-keys.md", + "gap": "9" + }, + { + "area": "auth", + "feature": "Key safety: no-limit/no-expiry flags, inactivity detection, bulk limit/disable/archive, account-wide max key lifetime, Enterprise IP allowlist, secret-scanning partner", + "sourceUrl": "https://openrouter.ai/docs/guides/overview/auth/security-settings.md", + "gap": "9" + }, + { + "area": "auth", + "feature": "OAuth PKCE flow issuing user-controlled workspace-scoped keys; workload-identity-token exchange endpoint", + "sourceUrl": "https://openrouter.ai/docs/guides/overview/auth/oauth.md", + "gap": "9" + }, + { + "area": "guardrails", + "feature": "Workspace guardrails: USD budgets with resets, model/provider allowlists, ZDR per model group, data regions, prompt-injection regex, PII redaction presets, custom regex filters (403 on block)", + "sourceUrl": "https://openrouter.ai/docs/guides/features/guardrails.md", + "gap": "7" + }, + { + "area": "cost", + "feature": "Layered spend caps (per-key limits, guardrail budgets, workspace budgets) checked independently; in-flight budget holds with 402 taxonomy (in_flight_budget_exhausted retryable)", + "sourceUrl": "https://openrouter.ai/docs/guides/best-practices/spend-controls.md", + "gap": "2" + }, + { + "area": "cost", + "feature": "Generation-level usage/cost APIs, credits balance endpoint, activity analytics incl. cached_tokens", + "sourceUrl": "https://openrouter.ai/docs/guides/features/logs.md", + "gap": "2" + }, + { + "area": "caching", + "feature": "Gateway-layer exact-match response cache (opt-in, free hits, zeroed usage, X-OpenRouter-Cache-Status; no coalescing)", + "sourceUrl": "https://openrouter.ai/docs/guides/features/response-caching.md", + "gap": "10" + }, + { + "area": "caching", + "feature": "Sticky routing for provider prompt caches (10-minute pin when cache-read price < prompt price; session_id/prompt_cache_key overrides) plus bidirectional marker translation", + "sourceUrl": "https://openrouter.ai/docs/guides/best-practices/prompt-caching.md", + "gap": "14" + }, + { + "area": "observability", + "feature": "Broadcast: OTLP traces (GenAI semconv) to ~20 destinations with sampling, privacy mode, filters; metadata-excluded-from-cache-key semantics", + "sourceUrl": "https://openrouter.ai/docs/guides/features/broadcast/webhook.md", + "gap": "11" + }, + { + "area": "observability", + "feature": "Request logs with upstream-attempt visibility; content stored only behind opt-in Input & Output Logging; zero-completion insurance", + "sourceUrl": "https://openrouter.ai/docs/guides/features/logs.md", + "gap": "context" + }, + { + "area": "auth", + "feature": "BYOK with prioritized/fallback placement, per-key model/API-key-hash/user filters, 5% metering, is_byok_only", + "sourceUrl": "https://openrouter.ai/docs/guides/overview/auth/byok.md", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "In-region routing (eu/us domains) with fail-closed residency guardrails (allowed_data_regions)", + "sourceUrl": "https://openrouter.ai/docs/guides/features/in-region-routing.md", + "gap": "context" + }, + { + "area": "mcp", + "feature": "Hosted MCP server at mcp.openrouter.ai with OAuth-issued 7-day spend-capped keys; billable send-message/generate-image plus read-only tools", + "sourceUrl": "https://openrouter.ai/docs/guides/overview/mcp-server.md", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Multi-protocol surface: chat/completions, responses (stateless), messages (Anthropic-native incl. cache_control), embeddings, batch, image/video, TTS/STT, rerank, containers", + "sourceUrl": "https://openrouter.ai/docs/_llms/api-reference.txt", + "gap": "6" + }, + { + "area": "routing", + "feature": "Service tiers (default/flex/priority/ultrafast) selectable per request with price trade-offs and honest service_tier reporting", + "sourceUrl": "https://openrouter.ai/docs/guides/features/service-tiers.md", + "gap": "13" + }, + { + "area": "ops", + "feature": "Continuous-delivery versioning: human-reviewed Breaking-tagged OpenAPI-diff changelog; v1 path-pinned API", + "sourceUrl": "https://openrouter.ai/docs/api_reference/versioning.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Request-mutating plugins (PDF extraction, response healing, middle-out context compression, Alignment/Armor beta) with admin-managed defaults", + "sourceUrl": "https://openrouter.ai/docs/guides/features/plugins.md", + "gap": "new:payload-cost-transforms" + }, + { + "area": "auth", + "feature": "Workspaces isolating keys/routing/guardrails/BYOK/presets/plugins/observability; SCIM + SSO; org/workspace/member roles", + "sourceUrl": "https://openrouter.ai/docs/guides/features/workspaces.md", + "gap": "context" + } + ] + }, + { + "name": "ostk-cache (os-tack)", + "category": "local prompt-cache engineering proxy", + "language": "Rust", + "status": "dormant (last commit 2026-06-12; not buildable from public repo — three private crates)", + "url": "https://github.com/os-tack/ostk-cache", + "capabilities": [ + { + "area": "caching", + "feature": "Prompt-cache hit-rate engineering via request mutation: single 1h cache_control collapse, HUD line on newest turn, proxy-owned breakpoint placement, byte-stable kernel text", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/README.md", + "gap": "14" + }, + { + "area": "protocol", + "feature": "Volatile billing-token stripper: schema-free recursive removal of Claude Code's per-turn cch=; correlator that busts cached prefixes (claimed $2.50->$0.29/turn)", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/billing_strip.rs", + "gap": "new:payload-cost-transforms" + }, + { + "area": "guardrails", + "feature": "History hygiene: system-reminder stripping from past turns with current-turn protection, tool_use/tool_result pairing preserved, tokens-saved estimates", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/system_reminder_strip.rs", + "gap": "new:payload-cost-transforms" + }, + { + "area": "cost", + "feature": "WARM/DEAD cache-write policy state machine with TTL-tier forecasting (median of trailing 5 gaps; 5m at 1.25x, 1h at 2x) tuned against acceptance criteria", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/write_policy.rs", + "gap": "14" + }, + { + "area": "protocol", + "feature": "Byte-stable synthetic history projection (rebuild): synthetic messages[0] kernel (envelope quadruplet, capped tool signatures, intent thread) re-served verbatim on WARM turns", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/rebuild.rs", + "gap": "new:payload-cost-transforms" + }, + { + "area": "protocol", + "feature": "Cross-provider cache-policy seam (PolicyBackend): Anthropic explicit breakpoints vs GPT prompt_cache_retention/prompt_cache_key with normalized UsageSnapshot reconciliation", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/provider_policy.rs", + "gap": "14" + }, + { + "area": "guardrails", + "feature": "Authority-parsed host gating for provider-only parameters (exact api.openai.com host match; chatgpt.com fails dark rather than 400-looping)", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/provider_policy.rs", + "gap": "context" + }, + { + "area": "observability", + "feature": "Streaming SSE usage rewriting: monotonic-max input_tokens overlay plus fabricated Anthropic invalid_request_error bodies to trigger client-side compaction; session allowlist gating", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/usage_truth.rs", + "gap": "new:payload-cost-transforms" + }, + { + "area": "guardrails", + "feature": "Progressive soft-cap body reduction pipeline (Tier A tool_result ejection -> Tier B pair pruning -> Tier C unused tool defs -> Tier D structured 413 with reduction payload)", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/README.md", + "gap": "new:payload-cost-transforms" + }, + { + "area": "observability", + "feature": "Per-turn JSONL ledger (byte columns incl. amplification stats), ostk-cache-stats CLI, and a two-window A/B passthrough protocol with 5% noise threshold", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/docs/PASSTHROUGH.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Host lifecycle-hooks ingestion (SessionStart/PreToolUse/PostToolUse/Stop) via ostk-cache-hooks installer posting to /hook/event with rotated JSONL", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/README.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "Opt-in full HTTP capture with redaction, sharded rotation, bounded-channel writer, and self-loop guard header", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/http_capture.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Cross-session harness transcript tailing (Claude Code/Codex session logs) building activity summaries and a tool_use_id seek index; only summaries go outbound", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/transcript_tail.rs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Content-addressed file-handle inlining (~96-byte file_id refs) with fail-open contract — every failure mode passes the original body unchanged", + "sourceUrl": "https://github.com/os-tack/ostk-cache/blob/main/src/rewrite_middleware.rs", + "gap": "new:payload-cost-transforms" + } + ] + }, + { + "name": "Otari (mozilla-ai)", + "category": "open-source gateway", + "language": "Python (FastAPI-style) + React dashboard + CLI", + "status": "very active (v0.37.0, 2026-11-04); pre-1.0 schema churn", + "url": "https://github.com/mozilla-ai/otari", + "capabilities": [ + { + "area": "cost", + "feature": "Pre-dispatch budget holds: worst case (prompt estimate + entire max_tokens) held before dispatch in one conditional update, reconciled after; crashed holds returned after a TTL", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md", + "gap": "2" + }, + { + "area": "cost", + "feature": "Scoped budgets: org/workspace/membership/key ceilings on USD, token and request axes plus rpm/tpm per user; rolling or UTC calendar resets; inert until a spend ceiling applies", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md", + "gap": "2;3" + }, + { + "area": "cost", + "feature": "Structured budget refusal: 403 with Otari-Error-Code: budget_exceeded, Otari-Budget-Scope header, Retry-After to window end, and a both-amounts body", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/budgets.md", + "gap": "2" + }, + { + "area": "auth", + "feature": "Workspace-scoped virtual keys (tk-...): expiration, model allowlists with provider wildcards, tool allowlists, own stacked budget, reject_user_mismatch, service-key flag; role-split issuance; instant-kill regeneration", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/api-keys.md", + "gap": "9" + }, + { + "area": "auth", + "feature": "Service keys with per-end-user spend, budgets and per-minute limits; pseudonymous provider identifiers on usage rows", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/end-users.md", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "Workload-identity federation: backends authenticate with platform OIDC identity tokens instead of key secrets", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/api-keys.md", + "gap": "new:caller-sso-rbac" + }, + { + "area": "routing", + "feature": "Routing policies: on_failure failover (incl. 400/401/429 and empty choices), conditional selection on budget_used_pct/budget_remaining_usd, weighted router, priority router skipping rate-full models; explain CLI + endpoint", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/routing.md", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Automatic provider selection over up to 32 offerings ranked by cost/health/region/latency with sticky prompt-cache sessions; colon spelling pins one provider", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/models.md", + "gap": "14" + }, + { + "area": "rate-limiting", + "feature": "Multi-dimensional rate limits (key/user/end-user/deployment/model/budget) with rpm, tpm (admission holds of prompt+max_tokens), max_concurrent slots, report mode, memory/Redis store, 30s live-edit propagation", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/configuration.md", + "gap": "3" + }, + { + "area": "mcp", + "feature": "Managed MCP tool loop: inline or workspace-stored servers with encrypted tokens, SSRF-checked https-only URLs, gateway executes tool calls and terminates the loop; caller-orchestrated mode", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/mcp.md", + "gap": "8" + }, + { + "area": "guardrails", + "feature": "Layered inference guardrails callers cannot remove: caller opt-in + org-mandated + operator-mandated on routing policies (on_unavailable: block fails closed)", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/guardrails.md", + "gap": "7" + }, + { + "area": "tooling", + "feature": "Built-in tools: sandboxed code execution (sandbox_url or e2b), web search/fetch with provider-native interception (auto|otari|provider) and max_uses caps; availability API", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/tools.md", + "gap": "context" + }, + { + "area": "observability", + "feature": "Agent traces with context-window map (8 otari.context.* attributes/span), OTLP ingestion, opt-in sealed content capture, Prometheus /metrics, W3C joining opt-in", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/traces.md", + "gap": "11;12" + }, + { + "area": "routing", + "feature": "Idempotency-Key replay of encrypted stored responses and streaming first-chunk hold (default 15 s) so pre-output failures return real statuses and can fall over", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/configuration.md", + "gap": "1" + }, + { + "area": "tooling", + "feature": "Agent Guardrails: standalone otari hook binary wiring repo-level gates (path globs, command phrases, verifier scripts, model-judged rubrics) into Claude Code/Codex hooks", + "sourceUrl": "https://github.com/mozilla-ai/otari/blob/main/docs/agent-guardrails.md", + "gap": "context" + } + ] + }, + { + "name": "Plano (katanemo)", + "category": "AI-native proxy/data plane for agentic apps (Envoy + proxy-wasm)", + "language": "Rust (proxy-wasm plugins) + TypeScript CLI", + "status": "active (release 0.4.39, 2026-10-24)", + "url": "https://docs.planoai.dev/", + "capabilities": [ + { + "area": "routing", + "feature": "NL-intent agent orchestration listener: hosted orchestrator model infers intent per turn and routes to plain-HTTP agents; route_on_user_only option", + "sourceUrl": "https://docs.planoai.dev/guides/orchestration.html", + "gap": "context" + }, + { + "area": "routing", + "feature": "Three-mode LLM routing (exact provider/model, semantic aliases, routing preferences via 4B router) with automatic fallback on 429/5xx and per-request max_switch_spend_pct budget", + "sourceUrl": "https://docs.planoai.dev/guides/llm_router.html", + "gap": "1;13" + }, + { + "area": "guardrails", + "feature": "Ordered guardrail filter chains: MCP (streamable-http) or HTTP filters that inspect/mutate/short-circuit (4xx = policy violation before the agent/LLM; 5xx = filter failure surfaced)", + "sourceUrl": "https://docs.planoai.dev/concepts/filter_chain.html", + "gap": "7" + }, + { + "area": "rate-limiting", + "feature": "Token-budget rate limits per model keyed on arbitrary header selectors (or * wildcard) via the governor crate", + "sourceUrl": "https://docs.planoai.dev/resources/configuration_reference.html", + "gap": "3" + }, + { + "area": "routing", + "feature": "Cheapest/fastest selection policies from live cost catalogs (models.dev, custom URL) and Prometheus latency queries, refreshed on an interval", + "sourceUrl": "https://docs.planoai.dev/resources/configuration_reference.html", + "gap": "13;new:synced-price-catalogs" + }, + { + "area": "routing", + "feature": "Model-affinity session pinning (X-Model-Affinity UUID) cached in-memory or Redis for multi-replica deployments; optional tenant header", + "sourceUrl": "https://docs.planoai.dev/guides/llm_router.html", + "gap": "14" + }, + { + "area": "observability", + "feature": "Agentic Signals: model-free behavioral quality detectors (3 layers, 7 categories) emitted as OTel attributes; incremental O(1) loop detection", + "sourceUrl": "https://docs.planoai.dev/concepts/signals.html", + "gap": "context" + }, + { + "area": "observability", + "feature": "OTel tracing with random sampling, span attributes from header prefixes, gRPC opentracing endpoint, exporters incl. PostHog; planoai trace inspector", + "sourceUrl": "https://docs.planoai.dev/resources/configuration_reference.html", + "gap": "11" + }, + { + "area": "observability", + "feature": "Prometheus stats (:19901) plus brightstaff RED + LLM-upstream metrics (TTFT, per-token time) and a shipped Prometheus+Grafana compose with a 19-panel dashboard", + "sourceUrl": "https://docs.planoai.dev/guides/observability/monitoring.html", + "gap": "12;17" + }, + { + "area": "observability", + "feature": "Envoy-format access logs split into ingress/LLM/internal streams with upstream service time and request ids", + "sourceUrl": "https://docs.planoai.dev/guides/observability/access_logging.html", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Ingress breadth: /v1/chat/completions, /v1/responses, /v1/messages cross-provider with streaming in both SDKs", + "sourceUrl": "https://docs.planoai.dev/concepts/llm_providers/client_libraries.html", + "gap": "6" + }, + { + "area": "protocol", + "feature": "Ten first-class providers + provider/* wildcard auto-configuration (refreshed, non-text filtered) with passthrough_auth and custom headers/http_host overrides", + "sourceUrl": "https://docs.planoai.dev/concepts/llm_providers/llm_providers.html", + "gap": "context" + }, + { + "area": "caching", + "feature": "Persistent conversation state in Postgres backing /v1/responses continuation; overrides.optimize_context_window trims history to fit", + "sourceUrl": "https://docs.planoai.dev/resources/configuration_reference.html", + "gap": "new:payload-cost-transforms" + }, + { + "area": "mcp", + "feature": "MCP-native filter ecosystem plus declarative prompt_targets (typed params bound to HTTP endpoints, intent threshold, auto LLM dispatch; local nearest-neighbor matching via acap)", + "sourceUrl": "https://github.com/katanemo/plano/blob/main/crates/llm_gateway/Cargo.toml", + "gap": "8" + }, + { + "area": "ops", + "feature": "planoai CLI lifecycle (up/down/build/logs/init/trace), single Docker image under supervisord, K8s deployment guidance (rollout restart on config change)", + "sourceUrl": "https://docs.planoai.dev/resources/deployment.html", + "gap": "context" + } + ] + }, + { + "name": "Portkey (Prisma AIRS AI Gateway)", + "category": "AI gateway + control plane (OSS gateway + hosted/hybrid)", + "language": "TypeScript (Rust front-end process in self-hosted enterprise since v2.30.0)", + "status": "OSS gateway dormant since 2026-05-25 (v1.15.2); enterprise/hybrid line active under Palo Alto Networks (acquired 2026-05-29)", + "url": "https://portkey.ai/docs/product/ai-gateway", + "capabilities": [ + { + "area": "routing", + "feature": "Config objects: recursive fallback / loadbalance / conditional strategies with nested targets, on_status_codes triggers, and per-target default/override/drop params (dot/array/wildcard paths)", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/configs", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Automatic retries: up to 5 attempts, 1-16 s exponential backoff capped 60 s cumulative, use_retry_after_headers, x-portkey-retry-attempt-count header", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/automatic-retries", + "gap": "1" + }, + { + "area": "routing", + "feature": "Weighted load balancing with sticky sessions (hash_fields + TTL, in-memory + Redis two-tier cache)", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/load-balancing", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Per-strategy circuit breaker (failure_threshold, cooldown >=30 s, failure_status_codes)", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/circuit-breaker", + "gap": "1" + }, + { + "area": "routing", + "feature": "Conditional routing over metadata/params/pathname with $eq/$ne/$in/$regex/comparison operators and nestable $and/$or", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/conditional-routing", + "gap": "13" + }, + { + "area": "routing", + "feature": "Request timeouts with 408 semantics feeding retry/fallback; streams exempt once one chunk arrives", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/request-timeouts", + "gap": "context" + }, + { + "area": "caching", + "feature": "Simple (exact) + semantic (cosine, chat-style routes <=8,191 tokens <=4 messages) caching, TTL 60 s-90 d, namespaces, force-refresh; self-hosted with own embedder + Milvus/Pinecone", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/cache-simple-and-semantic", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "20+ deterministic + LLM checks with async (zero-latency log) vs sync (deny 446 / flag 246) modes feeding fallback orchestration; streaming output guardrails informational", + "sourceUrl": "https://portkey.ai/docs/product/guardrails", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Irreversible pre-LLM PII redaction ({{EMAIL_ADDRESS_1}}-style placeholders); partner guardrail marketplace incl. Presidio-integrating vendors", + "sourceUrl": "https://portkey.ai/docs/product/guardrails/pii-redaction", + "gap": "7" + }, + { + "area": "auth", + "feature": "Model Catalog: org-level Integrations (credential vault) provisioned as slugged AI Providers (@openai-prod/gpt-4o addressing); workspace budgets/limits cascade", + "sourceUrl": "https://portkey.ai/docs/product/model-catalog", + "gap": "9" + }, + { + "area": "rate-limiting", + "feature": "Budget limits: USD (min $1) or token caps auto-expiring the key; weekly/monthly resets; thresholds alert; immutable once set (duplicate to change)", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/virtual-keys/budget-limits", + "gap": "2" + }, + { + "area": "rate-limiting", + "feature": "Enterprise usage policies: AND-conditions + group_by counters (user/model/provider/key/MCP server) with periodic resets; breach 412/429", + "sourceUrl": "https://portkey.ai/docs/product/enterprise-offering/budget-policies", + "gap": "2;3" + }, + { + "area": "auth", + "feature": "Dual-secret key rotation (POST /v2/api-keys/:id/rotate with key_transition_period_ms, id preserved) plus scheduled rotation policies and org-enforced cadence", + "sourceUrl": "https://portkey.ai/docs/product/enterprise-offering/org-management/api-key-rotation", + "gap": "9" + }, + { + "area": "observability", + "feature": "Logs with full bodies, cache/retry/fallback/loadbalance status badges, replay; plan-tiered retention; x-portkey-debug content toggle", + "sourceUrl": "https://portkey.ai/docs/product/observability/logs", + "gap": "context" + }, + { + "area": "observability", + "feature": "W3C traceparent/baggage extraction into trace ids; OTLP ingest; experimental outbound GenAI OTLP export", + "sourceUrl": "https://portkey.ai/docs/product/observability/opentelemetry", + "gap": "11" + }, + { + "area": "observability", + "feature": "Self-hosted Prometheus /metrics (request_count, cost/token gauges, ~10 duration histograms incl. gateway overhead)", + "sourceUrl": "https://portkey.ai/docs/self-hosting/prometheus-metrics", + "gap": "12" + }, + { + "area": "observability", + "feature": "Enterprise JSONL log export jobs (50k/job) with field-level deny lists", + "sourceUrl": "https://portkey.ai/docs/product/observability/logs-export", + "gap": "context" + }, + { + "area": "observability", + "feature": "Admin audit logs (indefinite retention) of create/update/delete/publish/export/rotate actions", + "sourceUrl": "https://portkey.ai/docs/product/enterprise-offering/audit-logs", + "gap": "context" + }, + { + "area": "cost", + "feature": "Centralized pricing JSON (24 h cache) with custom pricing and org-wide Pricing Adjustments (discount/markup multipliers)", + "sourceUrl": "https://portkey.ai/docs/product/observability/cost-management", + "gap": "2" + }, + { + "area": "mcp", + "feature": "MCP Gateway: authenticated proxy at mcp.portkey.ai/{slug}/mcp with managed upstream OAuth, per-tool provisioning, tool-call logging and throttling", + "sourceUrl": "https://portkey.ai/docs/product/mcp-gateway", + "gap": "8" + }, + { + "area": "mcp", + "feature": "Gateway-side execution of remote MCP tools in LLM requests (@portkey-mcp labels, allowed_tools filtering, credentials never stored) — beta", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/remote-mcp", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Universal API: three formats (chat/completions, responses, messages) against any provider via @slug/model; custom_host for private endpoints (private IPs SSRF-blocked)", + "sourceUrl": "https://portkey.ai/docs/product/ai-gateway/universal-api", + "gap": "6" + }, + { + "area": "ops", + "feature": "Hybrid deployment: data plane (gateway + Redis cache + log store) in the customer VPC, hosted control plane, 1-minute delta sync, prompts never leave", + "sourceUrl": "https://portkey.ai/docs/self-hosting/hybrid-deployments/architecture", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Prompt Engineering Studio: immutable prompt versions, labels, mustache multimodal templates, A/B via loadbalance configs, Render API to production", + "sourceUrl": "https://portkey.ai/docs/product/prompt-engineering-studio/prompt-versioning", + "gap": "context" + }, + { + "area": "ops", + "feature": "Secret references (AWS SM/Azure KV/HashiCorp Vault) resolved at runtime and cached 5 min; AWS KMS envelope encryption; FIPS 140-3 images", + "sourceUrl": "https://portkey.ai/docs/product/enterprise-offering/secret-references.md", + "gap": "15" + } + ] + }, + { + "name": "preflight (ghuntley)", + "category": "local DLP/secret-scanning inspection proxy", + "language": "Rust", + "status": "development dormant (5 commits on 2026-09-20; no release; weekly rules CI green through 2026-11-02)", + "url": "https://github.com/ghuntley/preflight", + "capabilities": [ + { + "area": "guardrails", + "feature": "Request-path DLP secret scanning with the full Gitleaks rule DB (pinned v8.30.1) compiled in, gated by an explicit 18-rule profile; keyword prefilter -> regex -> entropy -> SHA-256 carnet allowlist", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/scanner.rs", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Stable-placeholder redaction ([REDACTED:rule-id]) with redact / no-go (409) / advisory modes; un-rewritable findings fail closed; opaque finding UUIDs in errors", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/main.rs", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Local OCR and attachment inspection (JPEG/PNG/APNG/PDF, barcodes, metadata) via Tesseract/Poppler/QPDF/ExifTool/ZBar behind typed argv adapters; rebuilt artifacts re-inspected before approval", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/attachments.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "Bubblewrap-sandboxed networkless worker processes (--unshare-all, per-job tmpfs, 120 s, 16 MiB stdout cap); unknown formats and PDF active content fail closed", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/docs/adr/0002-rust-workers-and-content-cache.md", + "gap": "context" + }, + { + "area": "caching", + "feature": "Content-addressed verdict/artifact cache (memory + WAL SQLite secure_delete) binding content digest + credential scope + profile fingerprint, single-flight, generation-based purge, admin socket", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/cache.rs", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "SSRF-hardened remote materialization: https-only, no redirects/proxy, DNS resolved once and pinned, global-unicast-only IPs, byte budgets, inline data: URL replacement", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/resolver.rs", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Concurrency admission control: 16-permit semaphore returning 503 inspection_capacity; separate attachment/scan slot limits", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/main.rs", + "gap": "3;new:admission-queueing" + }, + { + "area": "observability", + "feature": "Prometheus /metrics with lock-free counters and an 8-bucket request-to-headers histogram; aggregate labels only", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/metrics.rs", + "gap": "12" + }, + { + "area": "observability", + "feature": "Secret-free structured JSON logging contract and cross-proxy x-request-id correlation (ingress-minted UUIDv4 propagated upstream)", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/correlation.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "SIGHUP atomic config reload with immutable runtime snapshots (failed reload keeps previous runtime; admitted requests keep their snapshot)", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/main.rs", + "gap": "16" + }, + { + "area": "auth", + "feature": "Split credentials: optional client key, upstream key substitution after hop-by-hop stripping, separate file-API key", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/src/main.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "NixOS hardened service module (DynamicUser, ProtectSystem=strict, MemoryMax...) plus QEMU VM test; weekly fully-gated Gitleaks rules auto-update with provenance", + "sourceUrl": "https://github.com/ghuntley/preflight/blob/main/nixos/module.nix", + "gap": "context" + } + ] + }, + { + "name": "Presidio (Data Privacy Stack)", + "category": "PII detection/anonymization toolkit", + "language": "Python", + "status": "active (community-governed since the Microsoft hand-off)", + "url": "https://presidio.dataprivacystack.org/", + "capabilities": [ + { + "area": "guardrails", + "feature": "Four components: Analyzer (NER + regex + checksum recognizers with context), Anonymizer (redact/mask/replace/encrypt/hash/surrogate), Image Redactor, Structured", + "sourceUrl": "https://presidio.dataprivacystack.org/", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Reversible anonymization: encrypt operator with decrypt for deanonymization (the reversible-masking pattern gateways embed)", + "sourceUrl": "https://presidio.dataprivacystack.org/anonymizer/", + "gap": "7" + }, + { + "area": "ops", + "feature": "Deployment breadth incl. HTTP/REST services and a LiteLLM proxy integration for masking LLM calls", + "sourceUrl": "https://presidio.dataprivacystack.org/", + "gap": "7" + } + ] + }, + { + "name": "PromptCache (messkan)", + "category": "self-hosted semantic cache proxy", + "language": "Go", + "status": "quiet (last commit 2026-08-18; v0.4.0 2026-04-24; unreleased main additions)", + "url": "https://github.com/messkan/prompt-cache", + "capabilities": [ + { + "area": "caching", + "feature": "Two-threshold semantic matching: >=0.70 direct hit, <=0.30 miss, gray zone between (env-tunable, validated bounds)", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/semantic/semantic.go", + "gap": "10" + }, + { + "area": "caching", + "feature": "Small-model intent verification for gray-zone hits (gpt-4o-mini / mistral-small / claude-3-haiku yes-no judge)", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/semantic/providers.go", + "gap": "10" + }, + { + "area": "protocol", + "feature": "SSE streaming end-to-end: misses forward the provider stream while buffering a completion for the cache; hits synthesize OpenAI-style SSE chunks", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/RELEASE_v0.4.0.md", + "gap": "10" + }, + { + "area": "tooling", + "feature": "Runtime threshold and provider tuning via authenticated GET/PATCH /v1/config without restart", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/RELEASE_v0.4.0.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Retryable upstream client: 429/500/502/503/504 with 500 ms x2 jittered backoff capped 30 s (not idempotency-aware)", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/http/client.go", + "gap": "1" + }, + { + "area": "observability", + "feature": "Prometheus /metrics (hits/misses/gray-zone/failures, latency, stored vectors) plus JSON /v1/stats with hit rate", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/metrics/metrics.go", + "gap": "12" + }, + { + "area": "auth", + "feature": "Constant-time bearer auth on management endpoints (metrics/stats/config/cache); inference and health public by design", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/middleware/middleware.go", + "gap": "context" + }, + { + "area": "caching", + "feature": "Cache lifecycle: 24 h TTL, 100k-entry LRU, hourly background cleanup, POST /v1/cache/warm bulk insert, GET/DELETE management", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/cache/cache.go", + "gap": "10" + }, + { + "area": "caching", + "feature": "Namespace-partitioned isolation with ANN partition safety (namespaced lookups use filtered linear scans so candidates never cross partitions) — unreleased", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/semantic/semantic.go", + "gap": "10" + }, + { + "area": "protocol", + "feature": "Multi-provider embedding/verification backends (OpenAI, Mistral, Claude via Voyage embeddings) with an OpenAI-to-Claude translation lane", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/semantic/providers.go", + "gap": "context" + }, + { + "area": "observability", + "feature": "X-Cache / X-Similarity-Score / X-Cache-Key / X-Provider diagnostic response headers computed allocation-free", + "sourceUrl": "https://github.com/messkan/prompt-cache/blob/main/internal/middleware/middleware.go", + "gap": "context" + } + ] + }, + { + "name": "proxyllm (llm-proxy/llm-proxy)", + "category": "reference checkout (in-process routing library)", + "language": "Python", + "status": "dormant (pinned ref/llm-proxy at fb588f6, 2024-05-24; last commit May 2024)", + "url": "https://github.com/llm-proxy/llm-proxy", + "capabilities": [ + { + "area": "routing", + "feature": "Cheapest-first cost routing: per-provider tokenization, worst-case cost estimate from price tables, min-heap cascade on failure with per-model error collection", + "sourceUrl": "ref/llm-proxy/proxyllm/proxyllm.py", + "gap": "13" + }, + { + "area": "routing", + "feature": "Zero-shot category routing (bart-large-mnli over 10 categories with per-model rank tables) and Elo-rating routing; both cascade down a min-heap", + "sourceUrl": "ref/llm-proxy/proxyllm/utils/categorization.py", + "gap": "13" + } + ] + }, + { + "name": "Respan (formerly Keywords AI)", + "category": "gateway + observability platform (with OSS Rust gateway)", + "language": "Rust (OSS gateway) + platform", + "status": "active (changelog entries through 2026-10-19)", + "url": "https://www.respan.ai/docs/", + "capabilities": [ + { + "area": "auth", + "feature": "Scoped API keys (Read/Write/Gateway/Admin) with Production/Test environments, expiry, spend caps, tag management, instant revocation, /api/temporary-keys/ minting", + "sourceUrl": "https://www.respan.ai/docs/documentation/admin/respan-api-keys", + "gap": "9" + }, + { + "area": "routing", + "feature": "Retries on by default (<=3, 0.2 s initial) with org-level fallback chains and per-request fallback_models; validation/context/credential failures never retried", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/retries", + "gap": "1" + }, + { + "area": "routing", + "feature": "Weighted load balancing across BYOK deployments (weight 0 drains) and across models per request (load_balance_models entries or saved groups)", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/load-balancing", + "gap": "13" + }, + { + "area": "rate-limiting", + "feature": "Layered limits: per-key cost/request/token over minute-to-month windows, lifetime caps, alert-at and block-at thresholds (429 key_usage_limit_exceeded pre-provider), customer 402, org-plan RPM", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/limits", + "gap": "2;3" + }, + { + "area": "caching", + "feature": "Exact-match response cache (messages-only key) with TTL <=30 min, per-customer/model scoping, dashboard with hits and savings; provider prompt-cache pass-through with cached-token reporting", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/caching", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "Eight-category request blocking (injection, system-prompt leak, 6 content categories) via span-01-security, free, org-wide; fail-open on check errors", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/guardrails", + "gap": "7" + }, + { + "area": "mcp", + "feature": "Remote MCP server (mcp.respan.ai/mcp, OAuth or bearer) exposing logs, traces, prompts, monitors; OSS gateway lets any model use MCP servers/tools", + "sourceUrl": "https://www.respan.ai/docs/documentation/mcp", + "gap": "8" + }, + { + "area": "protocol", + "feature": "OpenAI-compatible gateway plus Anthropic-native and Responses endpoints over 1000+ models / 30+ providers (one base-URL swap)", + "sourceUrl": "https://www.respan.ai/docs/documentation/features/gateway/gateway-quickstart", + "gap": "6" + }, + { + "area": "observability", + "feature": "Spans with full I/O, metrics/log dashboards, threshold monitors with scheduled reports, SQL editor over span data, OTel plus integrations, org-level PII redaction", + "sourceUrl": "https://www.respan.ai/docs/", + "gap": "11" + }, + { + "area": "tooling", + "feature": "OSS Rust gateway (dual MIT/Apache-2.0): single binary with plugin API, cross-protocol translation incl. Realtime voice translation, routing with fallback/cheapest/fastest/racing, request-log web UI at /ui/", + "sourceUrl": "https://www.respan.ai/docs/oss-gateway/introduction", + "gap": "context" + }, + { + "area": "cost", + "feature": "BYOK deployments with managed-credit fallback on rate limits; programmatic limit policies via admin keys", + "sourceUrl": "https://www.respan.ai/docs/changelog/changelog", + "gap": "context" + } + ] + }, + { + "name": "RouteLLM (lm-sys)", + "category": "research router framework", + "language": "Python", + "status": "dormant (last push 2024-08-10; not archived)", + "url": "https://github.com/lm-sys/RouteLLM", + "capabilities": [ + { + "area": "routing", + "feature": "Threshold-based strong/weak routing with four trained routers (matrix factorization, similarity-weighted Elo, BERT, causal LLM) claiming up to 85% cost reduction at 95% GPT-4 quality", + "sourceUrl": "https://raw.githubusercontent.com/lm-sys/RouteLLM/main/README.md", + "gap": "13" + }, + { + "area": "tooling", + "feature": "Deployment modes: OpenAI drop-in client and OpenAI-compatible server with threshold-calibration CLI plus an eval framework (MMLU, GSM8K, MT-Bench)", + "sourceUrl": "https://raw.githubusercontent.com/lm-sys/RouteLLM/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "rust-llm-proxy (habibi-dev)", + "category": "open-source gateway (Gemini-focused)", + "language": "Rust", + "status": "active (pushed 2026-04-26)", + "url": "https://github.com/habibi-dev/rust-llm-proxy", + "capabilities": [ + { + "area": "auth", + "feature": "User accounts with HMAC-signed tokens, admin CRUD for users/api-keys/services (issue, rotate, revoke), per-interaction job auditing to SQLite via SeaORM with auto-migrations", + "sourceUrl": "https://raw.githubusercontent.com/habibi-dev/rust-llm-proxy/master/README.md", + "gap": "9" + } + ] + }, + { + "name": "SMG (Shepherd Model Gateway)", + "category": "engine-agnostic LLM gateway", + "language": "Rust (Python launcher/bindings)", + "status": "very active (v1.14.0, 2026-11-03; commits 2026-11-04)", + "url": "https://lightseek.org/smg/v1.14.0/concepts/routing/cache-aware", + "capabilities": [ + { + "area": "routing", + "feature": "KV-cache-aware routing as default policy: four index modes (engine KV events, token-ID radix, text radix, TTL'd hash), match threshold 0.3, spill gate, per-model overrides, LRU-bounded trees", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/routing/cache-aware", + "gap": "new:engine-state-routing" + }, + { + "area": "routing", + "feature": "Engine-state-aware overload protection: queue-depth and KV-usage vetoes from /v1/loads, /metrics, gRPC GetLoads, KV-event records, or ZMQ; optional 503 shedding with Retry-After", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/overload-protection", + "gap": "new:engine-state-routing" + }, + { + "area": "protocol", + "feature": "Two-level tokenization cache: L0 exact-match (256 MB budget, approximate-LRU) plus L1 prefix-at-special-token boundaries (50 MB) so multi-turn chats tokenize only the suffix", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/performance/tokenizer-caching", + "gap": "context" + }, + { + "area": "extensibility", + "feature": "WASM Component Model plugin middleware (OnRequest/OnResponse) hot-deployed via POST/GET/DELETE /wasm without restart, sandboxed (64 MB / 1 s); streaming bypasses OnResponse", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/extensibility/wasm-plugins", + "gap": "context" + }, + { + "area": "rate-limiting", + "feature": "Priority admission scheduling: four classes (system/interactive/default/bulk) via x-smg-priority with reserved floors, strict-priority drain, starvation promotion, preemption of lower classes before first byte", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/priority-scheduling", + "gap": "new:admission-queueing" + }, + { + "area": "rate-limiting", + "feature": "Per-tenant token/request budgets with reserve-then-settle (own estimate reserved, signed delta settled, RAII abandon on failure, 429 tenant_rate_limit_exceeded with Retry-After); fail-closed startup, fail-open unknown tenants", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/tenant-rate-limiting", + "gap": "2;3" + }, + { + "area": "rate-limiting", + "feature": "Gateway-wide admission control: concurrency permits (whole stream), bounded FIFO queue, optional sustained-rate token bucket, 429/503 with Retry-After and X-SMG-Error-Code", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/rate-limiting", + "gap": "3;new:admission-queueing" + }, + { + "area": "routing", + "feature": "Request-path retries: fixed retryable set (408/429/500/502/503/504; 5 attempts), 50 ms x1.5 to 30 s with +-20% jitter, bodies buffered <=1 MiB, re-selection skipping failed workers, never retry a started stream", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/retries", + "gap": "1" + }, + { + "area": "routing", + "feature": "Per-worker three-state circuit breakers (10 failures open, 60 s to half-open, 3 successes close; 429 uncounted; health-check promotion)", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/reliability/circuit-breakers", + "gap": "1" + }, + { + "area": "auth", + "feature": "Split control/data-plane auth: OIDC SSO (Keycloak/Auth0/Azure/Okta) with JWKS discovery and unknown-kid refetch rate-limit, SHA-256 hashed control-plane keys with constant-time compare, admin/user RBAC roles, tenant keys, control-plane audit log on by default", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/security/authentication", + "gap": "new:caller-sso-rbac;9" + }, + { + "area": "mcp", + "feature": "Gateway-side MCP client: stdio (static) and streamable HTTP (static + dynamic, pooled by URL+credential hash), tool aliasing/dedup/renames, ordered allow/deny/deny_with_reason approvals, internal:true hidden servers, tool-loop cap 10, per-call Prometheus metrics", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/concepts/extensibility/mcp", + "gap": "8" + }, + { + "area": "protocol", + "feature": "Wide API surface: /v1/responses (MCP tool loops, stored responses/conversations), /v1/messages + count_tokens, Gemini /v1/interactions, /v1/embeddings, /v1/rerank, /v1/classify, /v1/audio/transcriptions, tokenize/detokenize, SGLang /generate, /v1/decisions, Realtime WS/WebRTC, per-vendor dialect profiles (Kimi, MiniMax, z.ai GLM, DeepSeek)", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/reference/api/openai", + "gap": "6" + }, + { + "area": "observability", + "feature": "150+ Prometheus metrics across six layers (runtime incl. jemalloc, HTTP error_code, router TTFT/TPOT histograms, worker health/breakers/KV-events, discovery, MCP), cardinality caps, zeroed-at-startup alert families, engine metrics passthrough at /engine_metrics", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/reference/metrics", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "OTel trace export via --enable-trace / --otlp-traces-endpoint; optional JSON logs with daily rotation", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/reference/metrics", + "gap": "11" + }, + { + "area": "tooling", + "feature": "Async admin API with job model: worker CRUD (202 + pollable jobs), runtime tokenizer CRUD, POST /flush_cache (206 partial), GET /inflight, POST /heap_profile — on an optional --admin-port", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/reference/api/admin", + "gap": "context" + }, + { + "area": "routing", + "feature": "Sticky sessions/routing keys (header-derived, <=128 bytes, 4 h idle pin, four assignment modes) plus HA mesh with SWIM gossip and CRDT-replicated state incl. radix-tree delta gossip", + "sourceUrl": "https://lightseek.org/smg/v1.14.0/reference/configuration", + "gap": "14" + } + ] + }, + { + "name": "sub2api (Wei-Shaw)", + "category": "multi-tenant subscription-to-API gateway (design reference; ToS-flagged)", + "language": "Go (Gin + Ent + Vue 3)", + "status": "extremely active (last push 2026-11-04; releases every 2-5 days); provider-adversarial features make it a non-integration design reference", + "url": "https://github.com/Wei-Shaw/sub2api", + "capabilities": [ + { + "area": "routing", + "feature": "Account-pool schedulability state machine: five independent time-based isolation states (overload, rate-limit reset, temp-unschedulable with account-or-model scope, expiry, session windows) persisted as columns", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/account.go", + "gap": "5" + }, + { + "area": "routing", + "feature": "Sticky-session scheduling with prompt-cache affinity: session key from client session id, else SHA-256 of cache_control:{ephemeral} content, else client digest; Redis binding 1 h TTL; sticky wait queues; account switches reclassify cache billing", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/gateway_service.go", + "gap": "14" + }, + { + "area": "routing", + "feature": "In-request failover loop keyed to 401/403/429: per-account retry counters and deadlines, switch budgets, mid-stream switching after SSE comments only; client disconnects classified 499 and stop failover", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/handler/failover_loop.go", + "gap": "1;new:upstream-cancellation" + }, + { + "area": "routing", + "feature": "Status-code to account-health taxonomy: 401 OAuth cache invalidation with recoverable cooldown, 402 permanent stop, 429 5h/7d quota windows parsed from headers, 529 overload cooldown", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/ratelimit_service.go", + "gap": "5" + }, + { + "area": "auth", + "feature": "Background OAuth token refresh with distributed lock, AccountRuntimeBlocker, and a stale-snapshot rollback guard; per-platform refreshers", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/token_refresh_service.go", + "gap": "new:oauth-credential-lifecycle" + }, + { + "area": "cost", + "feature": "Append-only token-level cost ledger: 5m/1h cache variants, per-kind USD costs, total vs actual (post-multiplier) with rate-multiplier snapshots, first-token latency, model-mismatch flag, billing circuit breaker", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/ent/schema/usage_log.go", + "gap": "2" + }, + { + "area": "cost", + "feature": "Layered budgets: per-key 5h/1d/7d USD windows, user subscription quotas, and Redis in-flight balance reservation (input + output price x max_tokens, TTL-renewed, fail-open on Redis loss)", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/migrations/064_add_api_key_rate_limits.sql", + "gap": "2" + }, + { + "area": "cost", + "feature": "Per-user-per-group rate multipliers (billing markup), RPM overrides, and a scheduler profit gate with veto cap and anti-livelock handling", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/user_group_rate.go", + "gap": "context" + }, + { + "area": "cost", + "feature": "Remote pricing catalog (model-price-repo) verified against a separately-fetched .sha256, cached locally with override patch merge, 10-minute hash rechecks, hot reload", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/deploy/config.example.yaml", + "gap": "new:synced-price-catalogs" + }, + { + "area": "rate-limiting", + "feature": "Redis concurrency slots per account and per user (sorted-sets, expiry sweeps, stale cleanup) feeding multi-layer load-aware selection (model pins > sticky > priority tiers > load); SSE pings while queued", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/service/concurrency_service.go", + "gap": "3" + }, + { + "area": "protocol", + "feature": "Broad surface: messages(+count_tokens), responses(+subpaths), chat/completions, embeddings, models, usage/billing, /v1/live realtime, images (sync/async/tasks), videos, tts/stt/custom-voices, x_search, Gemini cachedContents CRUD, codex compatibility", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/server/routes/gateway.go", + "gap": "6" + }, + { + "area": "mcp", + "feature": "Management MCP server: stateless Streamable-HTTP endpoint with OAuth consent, four immutable tool sets (user/admin x read/read-write), bounded envelopes, per-tool annotations", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/internal/managementmcp/server.go", + "gap": "8" + }, + { + "area": "observability", + "feature": "Synthetic channel monitors (probe/quota/quota_probe with AES-256-GCM keys, jittered intervals, rollups), async prompt-audit guard, dashboard aggregation caches, optional Server-Timing", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/backend/ent/schema/channel_monitor.go", + "gap": "context" + }, + { + "area": "auth", + "feature": "Hardened admin tooling: scoped admin keys (operation/resource limits, expiry, IP allowlists), passkeys/WebAuthn/TOTP/step-up, session fingerprint binding with family revocation, OIDC connect, Idempotency-Key on admin writes", + "sourceUrl": "https://github.com/Wei-Shaw/sub2api/blob/main/deploy/README.md", + "gap": "context" + } + ] + }, + { + "name": "Switchyard (NVIDIA-NeMo)", + "category": "model-routing toolkit (embeddable library + demo server + bindings)", + "language": "Rust (Python/C-ABI/Node bindings)", + "status": "very active (v0.3.0 stable 2026-09-22; v0.4.0a1; last commit 2026-11-03; Apache-2.0)", + "url": "https://github.com/NVIDIA-NeMo/Switchyard", + "capabilities": [ + { + "area": "routing", + "feature": "Ordered per-candidate fallback: each candidate exhausts its retry budget (max_retries default 2, Retry-After honored, backoff 250 ms to 2 s) before a rewritten fallback request advances", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/run.rs", + "gap": "1" + }, + { + "area": "routing", + "feature": "Per-backend failure cooldown: failure_cooldown_ms (default 5000, 0 disables) parks an atomic unavailable-until map after transient failures; TemporarilyUnavailable during cooldown; terminal cooldown = 503", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/reference/toml_schema.md", + "gap": "1" + }, + { + "area": "routing", + "feature": "Rate-limit-aware fallback policy: fallback_on_rate_limit decides whether an exhausted 429 falls through or returns verbatim; forwarded credentials retry request-locally without poisoning shared cooldown", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/run.rs", + "gap": "1" + }, + { + "area": "routing", + "feature": "Whole-attempt-chain deadline: one timeout_ms covering every attempt, retry delay, and the complete response incl. stream reads; template-compatibility retry inside the same budget", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy-llm-client/src/backend.rs", + "gap": "new:timeout-hierarchy" + }, + { + "area": "protocol", + "feature": "Native triple-protocol server: /v1/chat/completions, /v1/messages, /v1/responses plus /v1/decision, messages/responses count_tokens variants, /v1/responses/compact, GET /v1/models — route ids become callable models in all three wire formats", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/lib.rs", + "gap": "6" + }, + { + "area": "routing", + "feature": "Agent-aware routing suite: auto presets, composite (judge + stage router), llm_classifier with message-hash fallback, plan_execute, advisor gates, prefill-router, random/passthrough/noop", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/routing_algorithms/overview.md", + "gap": "13" + }, + { + "area": "observability", + "feature": "Prometheus /metrics (upstream attempts by outcome incl. 429/504, client responses incl. client_disconnected, router_retry_recovered, TTFT/routing-overhead histograms) plus OTel OpenInference/GenAI spans (libsy.run with fallback plan, libsy.client_call with cache/reasoning tokens)", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/metrics.rs", + "gap": "11;12" + }, + { + "area": "cost", + "feature": "Durable JSONL routing log: per-request records of chosen model, fallbacks, usage, and cost plus session snapshots; queryable via GET /v1/stats with reset", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/routing_log.rs", + "gap": "2" + }, + { + "area": "guardrails", + "feature": "Credential redaction middleware scrubbing configured provider keys from client-facing response headers and decompressed JSON bodies", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/redaction.rs", + "gap": "7" + }, + { + "area": "auth", + "feature": "Caller-credential forwarding: routes declare Anthropic/OpenAI CallerAuthKind and forward the caller's own credential upstream with format-mismatch rejection; Claude Code plugin ships", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/examples/claude/README.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Responses continuation state (previous_response_id/conversation replay) with responses_reasoning preserve_encrypted|drop policy; /v1/responses/compaction", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/docs/reference/toml_schema.md", + "gap": "6;new:payload-cost-transforms" + }, + { + "area": "observability", + "feature": "Theoretical KV-cache hit-rate tracking: prefix-fingerprint match rate per model reported as theoretical_cache_hit_rate (fallback-breaks-cache-affinity signal)", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/switchyard-server/src/stats/cache_eligibility.rs", + "gap": "new:engine-state-routing" + }, + { + "area": "routing", + "feature": "Unreleased main: speculative parallel model/decision calls with loser cancellation, C-ABI host aborts, serializable cross-turn routing state", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard/blob/main/CHANGELOG.md", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Multi-language embedding surface (Rust libsy core, Python/PyPI, C ABI, Node/TS, NeMo Relay plugin) plus ops harness: soak-test crate, benchmark configs, server CLI with TLS flags, graceful-shutdown timeout, listen backlog", + "sourceUrl": "https://github.com/NVIDIA-NeMo/Switchyard#components", + "gap": "context" + } + ] + }, + { + "name": "TensorZero", + "category": "open-source LLMOps gateway (Rust)", + "language": "Rust", + "status": "archived 2026-06-12 (final release 2026.6.0; CVE-2026-54457 fix)", + "url": "https://github.com/tensorzero/tensorzero", + "capabilities": [ + { + "area": "auth", + "feature": "Gateway API-key auth backed by Postgres: sk-t0- keys with expiration, UI or CLI creation, disable by public id, cached lookups", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/set-up-auth-for-tensorzero.mdx", + "gap": "9" + }, + { + "area": "rate-limiting", + "feature": "Scoped rate-limit rules over model_inferences/tokens/cost per second-to-month with priority resolution, token-bucket burst form, state in Valkey or Postgres shared across instances", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/enforce-custom-rate-limits.mdx", + "gap": "2;3" + }, + { + "area": "routing", + "feature": "Three-level retry/fallback hierarchy: model provider routing, variant retries (truncated exponential backoff with jitter), function-level fallback variants", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/guides/retries-fallbacks.mdx", + "gap": "1" + }, + { + "area": "routing", + "feature": "Granular timeouts per provider/model/variant: non_streaming total, streaming ttft and total, bounded by a global outbound cap", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/guides/retries-fallbacks.mdx", + "gap": "new:timeout-hierarchy" + }, + { + "area": "caching", + "feature": "Exact-match inference cache with four modes (off/on/write_only/read_only), max_age_s staleness, ClickHouse or Valkey backends, per-model-request keys", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/guides/inference-caching.mdx", + "gap": "10" + }, + { + "area": "cost", + "feature": "JSON-Pointer-configured cost computation per provider (negative values for cache discounts, cost_per_unit for tools, batch and embedding schedules) surfaced as tensorzero_cost", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/track-usage-and-cost.mdx", + "gap": "2" + }, + { + "area": "auth", + "feature": "Upstream credential management: env, dynamic per-request, fallback pairs, per-type defaults", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/manage-credentials.mdx", + "gap": "5" + }, + { + "area": "observability", + "feature": "Full-fidelity observability store in the operator's ClickHouse/Postgres with self-hosted UI, playground, and query API", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/observability/query-historical-inferences.mdx", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "OTLP trace export (gRPC) with W3C traceparent, custom per-request headers/attributes, GenAI or OpenInference format; /metrics with overhead histogram", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/export-opentelemetry-traces.mdx", + "gap": "11" + }, + { + "area": "protocol", + "feature": "Structured outputs normalized across providers (strict constrained decoding or forced tool calls) with documented per-provider caveats", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/generate-structured-outputs.mdx", + "gap": "context" + }, + { + "area": "routing", + "feature": "Experimentation on the request path: static weighted A/B with episode-sticky variants, adaptive bandits with automatic stopping, namespaces for tenant-scoped experiments", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/guides/retries-fallbacks.mdx", + "gap": "context" + }, + { + "area": "ops", + "feature": "Two-tier gateway relay centralizing auth/rate limits/credentials while teams keep their own deployments", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/centralize-auth-rate-limits-and-more.mdx", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Native /inference API plus OpenAI-compatible /openai/v1/chat/completions and embeddings; 21 provider integrations; no Anthropic-compatible client endpoint", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/api-reference/inference-openai-compatible.mdx", + "gap": "context" + }, + { + "area": "mcp", + "feature": "MCP tool integration via standard tool calling (client-side pattern; example shipped); /mcp server added in final release notes without a docs page", + "sourceUrl": "https://github.com/tensorzero/tensorzero/releases/tag/2026.4.0", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Escape hatches: extra_body/extra_headers per variant/provider/request, include_raw_response, direct SQL access to stored data", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/operations/extend-tensorzero.mdx", + "gap": "context" + }, + { + "area": "ops", + "feature": "Self-published benchmark: 0.94 ms p99 overhead at 10k QPS on c7i.xlarge (not independently verified)", + "sourceUrl": "https://raw.githubusercontent.com/tensorzero/tensorzero/main/docs/gateway/benchmarks.mdx", + "gap": "context" + } + ] + }, + { + "name": "ThinkWatch (ThinkWatchProject)", + "category": "self-hosted AI + MCP gateway with console", + "language": "Rust (6-crate workspace) + React 19 console", + "status": "very active (v3.11.0, 2026-11-03); BSL 1.1 (10M tokens + 10k MCP calls/month free tier)", + "url": "https://github.com/ThinkWatchProject/ThinkWatch", + "capabilities": [ + { + "area": "auth", + "feature": "OIDC/SSO sign-in (Zitadel, Okta, Entra, Keycloak, Auth0) via Authorization Code with Redis-held state/nonce; 900 s JWTs + 7-day rotating refresh with replay detection; IP-bound sessions; Argon2id", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "RBAC with five built-in roles (admin/operator/user/viewer/service) plus custom roles over models, tools, admin pages; MCP tool-level access control", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "new:caller-sso-rbac" + }, + { + "area": "auth", + "feature": "tw- virtual keys stored as SHA-256 hashes, scoped to AI and/or MCP gateway, with rotation dual-key grace periods, inactivity timeouts, per-key limits layered on the owner's", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "9" + }, + { + "area": "mcp", + "feature": "MCP gateway where tool calls run as the real user: per-user GitHub/Notion/Linear/Slack/Atlassian OAuth or personal tokens so upstream audits show who acted; -32050 with authorization URL when unconnected; AES-256-GCM tokens at rest; per-user response caches", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/tree/main/crates/mcp-gateway/src", + "gap": "8" + }, + { + "area": "guardrails", + "feature": "PII/secrets redaction with placeholder restore in streams: 131 rules in 20 groups (v3.11.0), provider keys recognized by value, per-value exceptions", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.11.0", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Dangerous-command guards (credentials to unknown hosts, local files uploaded externally) and content filter over caller text and streamed answers with held-back tail matching; refuse/delete/replace/record actions", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#behaviour-worth-knowing", + "gap": "7" + }, + { + "area": "rate-limiting", + "feature": "Sliding-window rate limits (requests or tokens, 1 min to 1 week incl. MCP tool calls) and spend budgets (daily/weekly/monthly) with 50/80/95/100% one-time alerts, 429 + Retry-After, fail-open default with fail-closed switch, client-facing GET /v1/usage", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#behaviour-worth-knowing", + "gap": "2;3" + }, + { + "area": "cost", + "feature": "Cost accounting with chargeback CSV reports, month-end forecast, cost centers, and per-model quota weights (crates/gateway cost_tracker.rs)", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "2" + }, + { + "area": "observability", + "feature": "ClickHouse audit trail (six tables, 90/30-day TTLs) of every model request and tool call with optional pre-redacted bodies; SIEM forwarding via Syslog/Kafka REST/signed webhooks; S3 offload over 256 KiB", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/.env.example", + "gap": "context" + }, + { + "area": "routing", + "feature": "Weight/latency/health routing with tw-breaker circuit breaker, per-route sliding-minute RPM/TPM caps pushing traffic to the next route", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Transient upstream retry window (gateway.retry_wait_secs default 120 s) before any content reaches the client, with reasoning-effort levels auto-adapted on rejection", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/releases", + "gap": "1" + }, + { + "area": "routing", + "feature": "Provider-encrypted conversation pinning: conversations with provider-encrypted history stay pinned 30 days; unreadable history fails with a clear SealedHistoryRefused error", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/releases", + "gap": "14" + }, + { + "area": "protocol", + "feature": "Four client dialects on one port (OpenAI Chat Completions, OpenAI Responses, Anthropic Messages, Gemini) translated to OpenAI/Anthropic/Google/Azure/Bedrock (SigV4 + event-stream decoding via tw-bedrock)", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#highlights", + "gap": "6;4" + }, + { + "area": "observability", + "feature": "Bearer-gated /metrics endpoint (unmounted when METRICS_BEARER_TOKEN unset); min-generation-speed, buffering-pressure, audit-backlog metrics", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/.env.example", + "gap": "12" + }, + { + "area": "ops", + "feature": "Docker Compose + Helm (distroless, secret generation on install), adaptive execution honoring container limits with mimalloc, runbooks for backup/restore and secret rotation; static + DB config with Redis pub/sub sync", + "sourceUrl": "https://github.com/ThinkWatchProject/ThinkWatch/blob/main/README.md#deployment", + "gap": "context" + } + ] + }, + { + "name": "tokdiet (agiwhitelist)", + "category": "local token-metering + context-compaction proxy", + "language": "TypeScript (Node 20)", + "status": "dormant (12 commits 2026-06-16/18; npm 0.1.0 only; 32 stars) — proof-of-concept snapshot", + "url": "https://github.com/agiwhitelist/tokdiet", + "capabilities": [ + { + "area": "protocol", + "feature": "Fail-open mutating streaming proxy core: 64 MiB buffered bodies, per-stage guards degrading any internal error to byte-transparent passthrough; loopback-only bind", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/proxy.ts", + "gap": "new:payload-cost-transforms" + }, + { + "area": "protocol", + "feature": "Context virtual-memory compaction: hot/warm/cold tiers, loss-free dedup, recoverable elision (240-char previews + 12 salient lines, originals in SQLite elided_blobs), optional mid-summarize; published -71% input tokens", + "sourceUrl": "https://github.com/agiwhitelist/tokdiet/tree/main/src/compactor", + "gap": "new:payload-cost-transforms" + }, + { + "area": "guardrails", + "feature": "Shadow-eval quality guard: ~5% of compacted requests re-sent uncompacted and scored, 50-sample rolling average vs a 2% max-degradation budget, tripping overall or per-strategy safe-mode; priors persisted across restarts", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/quality/guard.ts", + "gap": "new:shadow-eval-guard" + }, + { + "area": "tooling", + "feature": "Published 66-task A/B quality-parity benchmark: paired governed/ungoverned runs on a real model, objective grader + LLM judge, majority voting, per-category persistent-regression list (95-97% parity)", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/bench/run.mjs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Page-fault detection (model echoing paged-out cg- ids or complaining) with one-shot blob re-injection (maxReinjections default 1); streaming skipped", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/pagefault.ts", + "gap": "new:payload-cost-transforms" + }, + { + "area": "cost", + "feature": "Hierarchical USD budget gate: session -> UTC day -> repo-month precedence with warn/compact/block actions (block = 429)", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/src/budget.ts", + "gap": "2" + }, + { + "area": "cost", + "feature": "Cost metering from an out-of-band SSE usage tee against a user-editable hot-reloaded pricing.json incl. cache read/write rates; keys forwarded, never persisted", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/ARCHITECTURE.md", + "gap": "new:synced-price-catalogs" + }, + { + "area": "caching", + "feature": "Anthropic prompt-cache-aware and thinking-safe mutation guards: never rewrite at/before a cache_control breakpoint; never mutate signed/thinking blocks", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/README.md", + "gap": "14" + }, + { + "area": "observability", + "feature": "Live SSE dashboard on a second loopback port (session, savings, quality + safe-mode, tool/repo, strategy leaderboard) over better-sqlite3", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/ARCHITECTURE.md", + "gap": "19" + }, + { + "area": "tooling", + "feature": "Claude Code marketplace plugin: PreToolUse/PostToolUse metering hooks appending JSONL records (sizes only), never blocking, plus /tokdiet command", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/hooks/tool-meter.mjs", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Four provider adapters (Anthropic, OpenAI, Gemini, MiniMax) with per-provider upstream override env vars; unknown JSON fields pass through untouched", + "sourceUrl": "https://raw.githubusercontent.com/agiwhitelist/tokdiet/main/README.md", + "gap": "context" + } + ] + }, + { + "name": "Traefik Hub AI Gateway", + "category": "AI gateway middleware (Kubernetes)", + "language": "Go", + "status": "active", + "url": "https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit", + "capabilities": [ + { + "area": "rate-limiting", + "feature": "ai-rate-limit (token bucket, burst-tolerant) vs ai-quota (sliding window, no bursts) over input/output/total-token or USD costLimit; sourceCriterion scoping; body-length/4 pre-request rejection; Redis-backed fail-open", + "sourceUrl": "https://doc.traefik.io/traefik-hub/ai-gateway/middlewares/token-rate-limit", + "gap": "3" + } + ] + }, + { + "name": "Tyk AI Gateway / AI Studio", + "category": "open-source AI gateway (AGPL core + Enterprise)", + "language": "Go", + "status": "active (stable 2.2.x; v2.3.0 in unstable prereleases, 2026-10-27)", + "url": "https://github.com/TykTechnologies/ai-studio", + "capabilities": [ + { + "area": "protocol", + "feature": "Unified OpenAI-format ingress with namespaced {llmSlug}/{model} models plus four surfaces: vendor-native passthrough, OpenAI translation, Anthropic-to-Bedrock bridge, legacy routes", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/proxy.md", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Governed native SDK operations (unreleased 2.3.0): Anthropic batches/count_tokens, OpenAI Responses/Conversations/Batches/Files with cryptographically signed ids scoped to the owning App", + "sourceUrl": "https://github.com/TykTechnologies/ai-studio/blob/main/CHANGELOG-v2.3.0.md", + "gap": "6" + }, + { + "area": "auth", + "feature": "App API keys with RBAC grants per LLM/datasource/router, admin activation workflow (approve/reject issuance), 403 for ungranted slugs incl. fallback visibility", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/apps.md", + "gap": "context" + }, + { + "area": "auth", + "feature": "Secrets management: $SECRET/ references encrypted at rest under TYK_AI_SECRET_KEY, $ENV/ against an operator allowlist, edge gateways never resolve references", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/secrets.md", + "gap": "15" + }, + { + "area": "routing", + "feature": "Per-provider failover waterfalls (max 10 rungs; 408/429/5xx/timeout/connect triggers, never 401/403; pre-first-token only; X-Tyk-Served-LLM/Failover headers; per-attempt logging)", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/llm-management.md", + "gap": "1" + }, + { + "area": "routing", + "feature": "Model Router (Enterprise): glob-pattern pools with priority, round-robin/weighted vendors, per-vendor model mapping", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/model-router.md", + "gap": "13" + }, + { + "area": "routing", + "feature": "Semantic Router (Enterprise): staged deciders (explicit route, session affinity, keywords, embedding similarity, LLM judge, default) with shadow mode and degrade-to-default on classifier failure", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/semantic-router.md", + "gap": "13" + }, + { + "area": "cost", + "feature": "Per-token pricing with cache and batch tiers (x10000 fixed-point), service tiers in analytics, models-in-use spend/deprecation views", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/llm-management.md", + "gap": "2" + }, + { + "area": "cost", + "feature": "Monthly budgets at LLM/App/team level with pre-forward checks (403 budget_exceeded / 503 budget_check_unavailable) and ~30 s block-list pulse to edges; enforcement Enterprise-only", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/budgeting.md", + "gap": "2" + }, + { + "area": "observability", + "feature": "Prometheus metrics per OTel GenAI semconv (TTFT, time-per-output-token, token usage) plus aistudio_* governance counters; secure-by-default /metrics; Server-Timing splitting gateway vs upstream; always-on W3C traceparent", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/observability.md", + "gap": "11;12" + }, + { + "area": "guardrails", + "feature": "Guardrail filters calling detection providers (built-in patterns, Lakera, Azure, Presidio, Bedrock, HTTP classifiers) with block/redact/quarantine/log, timeouts, fail modes, streaming re-evaluation; tool-result screening from 2.3", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/guardrails.md", + "gap": "7" + }, + { + "area": "guardrails", + "feature": "Tengo script filters with vendor-agnostic redaction helpers (tyk.redact_pattern, tyk.quarantine_segment) over normalized messages incl. tool results", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/filters.md", + "gap": "7" + }, + { + "area": "mcp", + "feature": "Tyk Gateway v5.13+ MCP Gateway: OAuth 2.1 with Protected Resource Metadata, per-key allowlists of proxies/methods/tools/resources/prompts, per-tool rate limits and circuit breakers, filtered discovery, per-tool analytics", + "sourceUrl": "https://tyk.io/docs/ai-management/mcp-gateway/overview.md", + "gap": "8" + }, + { + "area": "mcp", + "feature": "AI Studio tools exposed over MCP at /tools/{slug}/mcp with OpenAPI-derived schemas, privacy levels gating which LLMs may use them", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/tools.md", + "gap": "8" + }, + { + "area": "tooling", + "feature": "Hub-and-spoke edge microgateways with checksum config sync, pull-on-miss credential caching, namespaces; unified gRPC-isolated plugin SDK extending gateway/UI/analytics/endpoints", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/edge-gateways.md", + "gap": "context" + }, + { + "area": "caching", + "feature": "Community exact-match LLM response cache plugin; Enterprise semantic cache (cosine >=0.95, partition keys, memory or Redis, X-Cache-Status headers)", + "sourceUrl": "https://tyk.io/docs/ai-management/ai-studio/semantic-caching", + "gap": "10" + }, + { + "area": "ops", + "feature": "Outbound SSRF protection on by default from 2.3 (post-DNS and redirect checks, per-asset admin overrides) and UUIDv7 X-Tyk-AI-Request-IDs traced through logs/events/analytics", + "sourceUrl": "https://raw.githubusercontent.com/TykTechnologies/ai-studio/main/docs/site/docs/proxy.md", + "gap": "context" + } + ] + }, + { + "name": "uni-api (yym68686)", + "category": "open-source gateway (config-file-first)", + "language": "Rust (rewritten from Python)", + "status": "active (v1.7.x line, Sep 2026)", + "url": "https://github.com/yym68686/uni-api", + "capabilities": [ + { + "area": "routing", + "feature": "Six per-key channel scheduling algorithms: fixed_priority, round_robin, weighted_round_robin, lottery, random, and success-rate-aware smart_round_robin (72 h window)", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "AUTO_RETRY to next channel (true/false or count, ceiling 100 attempts) with tiered cooldowns (global 300 s, per-provider, Codex 6 h, per-key 429)", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "1" + }, + { + "area": "rate-limiting", + "feature": "Stacked per-key windows (15/min,10/day style), per-model substring/prefix limits with default fallback, parallel global and per-upstream-key knobs", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "3" + }, + { + "area": "auth", + "feature": "YAML-declared API keys with model wildcards, role:admin keys unlocking /v1/stats and /v1/generate-api-key, keys-as-channels routed in-process, Codex account_id+refresh_token credentials with auto-minting", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "9" + }, + { + "area": "cost", + "feature": "SQLite/Postgres usage stats, per-model $/M pricing config, per-key dollar-credit quotas (30-day default, 0 blocks) and tokens-per-request caps", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "2" + }, + { + "area": "observability", + "feature": "OTel/Fugue telemetry (sample rate), /v1/observability/runtime introspection, async billing-fact export (4096-event queue, 64 MiB retry spool, SHA-256 credential fingerprints, no prompts)", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "11" + }, + { + "area": "guardrails", + "feature": "Optional omni-moderation-latest screening of user messages to reduce ban risk; configurable error_triggers forcing channel failure", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "7" + }, + { + "area": "routing", + "feature": "Fine-grained timeout policy engine (match on endpoint/stream/request_type/method/engine/provider/model with connect/write/pool/first_byte/idle/total budgets, 0 disables each) plus SSE keepalive heartbeat", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "new:timeout-hierarchy" + }, + { + "area": "tooling", + "feature": "Config-file-only operations: hot-reloading api.yaml, POST /v1/channel-controls in-memory ordering/disable, X-Uni-API-Provider single-attempt diagnostics header; separate uni-api-web UI project", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "16" + }, + { + "area": "ops", + "feature": "Resource admission control (cgroup-memory-aware weighted admit/queue), 128 MiB body cap, request spooling supporting retries of large bodies, per-channel exclusion rules", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "new:admission-queueing" + }, + { + "area": "protocol", + "feature": "Wide endpoint breadth: chat, responses(+compact), messages, images, embeddings, transcriptions, speech, moderations, models, search across 16+ engines", + "sourceUrl": "https://raw.githubusercontent.com/yym68686/uni-api/main/README.md", + "gap": "6" + } + ] + }, + { + "name": "Upstash AI Gateway", + "category": "negative finding (product does not exist)", + "language": "n/a", + "status": "verified not to exist (11 live checks incl. docs sitemap, llms.txt, GitHub, npm, DNS, Wayback; only a Neon AI Gateway guide on Upstash docs)", + "url": "https://upstash.com/docs/llms.txt", + "capabilities": [ + { + "area": "other", + "feature": "No AI Gateway product exists at this vendor; the brief's description matches Vercel AI Gateway / Neon AI Gateway", + "sourceUrl": "https://upstash.com/docs/llms.txt", + "gap": "context" + } + ] + }, + { + "name": "Vercel AI Gateway", + "category": "cloud gateway (managed)", + "language": "n/a (SaaS; AI SDK clients)", + "status": "active (GA)", + "url": "https://vercel.com/docs/ai-gateway", + "capabilities": [ + { + "area": "cost", + "feature": "Spend budgets at team/project/API-key/user scopes (every in-scope budget must pass; 402 quota_for_entity_exceeded), daily/weekly/monthly/none UTC resets, per-scope defaults, 50/75/100% email alerts, soft-cap semantics", + "sourceUrl": "https://vercel.com/docs/ai-gateway/observability-and-spend/budgets", + "gap": "2" + }, + { + "area": "cost", + "feature": "Prepaid credits with auto top-up, zero markup on tokens, itemized add-on surcharges, per-request cost metadata (cost, marketCost, generationId)", + "sourceUrl": "https://vercel.com/docs/ai-gateway/pricing", + "gap": "2" + }, + { + "area": "cost", + "feature": "Custom Reporting API: group usage by day/user/model/tag/provider/credential with filters; per-request user/tags attribution headers", + "sourceUrl": "https://vercel.com/docs/ai-gateway/observability-and-spend/custom-reporting", + "gap": "2" + }, + { + "area": "auth", + "feature": "Team API keys with once-shown secrets, spend attribution (user vs team), CLI/REST lifecycle, unauthenticated compromised-secret revocation endpoint", + "sourceUrl": "https://vercel.com/docs/ai-gateway/authentication-and-byok/api-keys", + "gap": "9" + }, + { + "area": "auth", + "feature": "12-hour project-bound OIDC tokens as an alternative to static keys; AI SDK resolves automatically; OIDC spend counts to project budgets only", + "sourceUrl": "https://vercel.com/docs/ai-gateway/authentication-and-byok/oidc", + "gap": "new:caller-sso-rbac" + }, + { + "area": "routing", + "feature": "Health-aware provider routing with per-request order/only and sort by cost/ttft/tps; full attempt tree in gateway.routing metadata; degraded providers penalized", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/provider-filtering-and-ordering", + "gap": "1;13" + }, + { + "area": "routing", + "feature": "Ordered model fallback chains (models array after all providers of the primary fail) composing with provider ordering; decision-based conditional reruns", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/model-fallbacks", + "gap": "1" + }, + { + "area": "routing", + "feature": "Team-wide routing rules: model rewrites and denies (403 with ruleId) applying to fallback candidates; 10-rewrite chain cap", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/routing-rules", + "gap": "13" + }, + { + "area": "routing", + "feature": "Virtual models (vmc/ slugs) bundling provider order, fallbacks, sort, service tier, caching, timeouts, region, capabilities, compliance under one stable name; settings only tighten", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/virtual-models", + "gap": "13" + }, + { + "area": "caching", + "feature": "Automatic prompt caching: gateway-injected cache_control breakpoints for explicit providers, cache_ttl 5m|1h, advisory cache_anchor_items for mutating prompts, x-session-affinity passthrough", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/automatic-caching", + "gap": "14" + }, + { + "area": "observability", + "feature": "Request logs with per-attempt routing cards (TTFT, timing spans, region, credential source), 36-day filters, live tail, CSV/JSON export", + "sourceUrl": "https://vercel.com/docs/ai-gateway/observability-and-spend/logs", + "gap": "17" + }, + { + "area": "observability", + "feature": "OTel trace drains per request (OTLP/HTTP, no prompts) with gen_ai.* + vercel.ai_gateway.* attributes incl. cost and cache tokens", + "sourceUrl": "https://vercel.com/docs/ai-gateway/observability-and-spend/trace-drains", + "gap": "11" + }, + { + "area": "rate-limiting", + "feature": "Tier-based limiting: paid tier has no gateway rate limits (429 error taxonomy documented), free tier per-model limits, budget rejections distinguished as 402", + "sourceUrl": "https://vercel.com/docs/ai-gateway/rate-limits", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Compliance routing: ZDR default with per-request/team ZDR pinning to contracted providers, no-training control, provider and model allowlists (403)", + "sourceUrl": "https://vercel.com/docs/ai-gateway/security-and-compliance", + "gap": "context" + }, + { + "area": "guardrails", + "feature": "Regional inference pinning (scope + geoRegion) incl. overriding BYOK-credential regions", + "sourceUrl": "https://vercel.com/docs/ai-gateway/authentication-and-byok/byok", + "gap": "context" + }, + { + "area": "auth", + "feature": "BYOK: dashboard-stored team credentials and per-request providerOptions.gateway.byok (ordered credentials, Azure modelMappings) with system-credential fallback; BYOK spend outside budgets", + "sourceUrl": "https://vercel.com/docs/ai-gateway/authentication-and-byok/byok", + "gap": "context" + }, + { + "area": "routing", + "feature": "Per-provider BYOK timeouts (1 s to ~13 min) aborting the attempt for fallback; cleared on first stream chunk; recorded in attempt metadata", + "sourceUrl": "https://vercel.com/docs/ai-gateway/models-and-providers/provider-timeouts", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Multi-protocol surface sharing auth/routing/billing: AI SDKs, OpenAI chat/completions, Responses + OpenResponses, Anthropic messages, embeddings, Cohere rerank, TypeSafe, /v1/decisions", + "sourceUrl": "https://vercel.com/docs/ai-gateway/sdks-and-apis", + "gap": "6" + }, + { + "area": "tooling", + "feature": "vercel ai-gateway CLI (keys, budgets, rules, virtual models, models) and one-command agent onboarding (29 agents, macOS Keychain storage, dedicated compat endpoints)", + "sourceUrl": "https://vercel.com/docs/ai-gateway/coding-agents", + "gap": "context" + }, + { + "area": "tooling", + "feature": "Gateway-hosted Browserbase web-search and fetch tools callable by any tool-calling model; MCP control-plane tools", + "sourceUrl": "https://vercel.com/changelog/ai-gateway-adds-browserbase-search-and-fetch-tools", + "gap": "context" + } + ] + }, + { + "name": "vLLM Production Stack", + "category": "Kubernetes serving stack with request router", + "language": "Python (FastAPI router) + Helm", + "status": "active (vllm-stack-0.1.13, 2026-09-29)", + "url": "https://github.com/vllm-project/production-stack", + "capabilities": [ + { + "area": "routing", + "feature": "Pluggable routing algorithms: roundrobin, session (consistent hash ring), prefixaware, kvaware (LMCache lookup, 2000-token threshold), loadaware (cache benefit minus beta-weighted load), priority-header, disaggregated prefill (plain and orchestrated)", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/routers/routing_logic.py", + "gap": "1;13;new:engine-state-routing" + }, + { + "area": "routing", + "feature": "Request-level failover: up to max_instance_failover_reroute_attempts+1 attempts excluding failed endpoints; only connection/header-phase failures retry", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/services/request_service/request.py", + "gap": "1" + }, + { + "area": "routing", + "feature": "Service discovery: Kubernetes label-selector or static backends/models/aliases with optional polled health checks excluding failed nodes", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/README.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "External (non-vLLM) provider routing with async live model validation, alias resolution, health checks, and merged /v1/models with owned_by attribution", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/external_providers/registry.py", + "gap": "context" + }, + { + "area": "protocol", + "feature": "Broad surface: chat/completions, completions, embeddings, pooling, tokenize/detokenize, rerank/score, responses, messages, audio trio, images, files, alpha batch API with files storage", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/routers/main_router.py", + "gap": "6" + }, + { + "area": "caching", + "feature": "Alpha feature-gated semantic cache (sentence-transformer, threshold 0.95, per-request overrides, optional persistence)", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/experimental/README.md", + "gap": "10" + }, + { + "area": "guardrails", + "feature": "Alpha feature-gated PII detection/blocking with selectable analyzer (presidio or regex)", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/experimental/README.md", + "gap": "7" + }, + { + "area": "observability", + "feature": "Prometheus /metrics: router resource gauges, per-engine QPS/latency/prefill/decoding/running counts, GPU prefix-cache hit rate, healthy_pods gauge; per-model token/error metrics", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/routers/metrics_router.py", + "gap": "11;12" + }, + { + "area": "observability", + "feature": "OTel tracing with W3C propagation and optional Sentry profiling; X-Request-Id injection; dependency-aware /health", + "sourceUrl": "https://docs.vllm.ai/projects/production-stack/en/latest/use_cases/distributed-tracing.html", + "gap": "11" + }, + { + "area": "caching", + "feature": "LMCache KV offloading and cross-instance sharing; KEDA autoscaling incl. scale-to-zero on queue depth; sleep/wake endpoints", + "sourceUrl": "https://docs.vllm.ai/projects/production-stack/en/latest/use_cases/sharing-kv-cache.html", + "gap": "context" + }, + { + "area": "ops", + "feature": "Hot-reloadable dynamic config (YAML/JSON watched every 10 s) surfaced on /health; Helm chart, operator CRD, Gateway API option via agentgateway", + "sourceUrl": "https://raw.githubusercontent.com/vllm-project/production-stack/main/src/vllm_router/README.md", + "gap": "16" + } + ] + }, + { + "name": "xLB (EeroEternal)", + "category": "streaming-first thin-L7/L4 load balancer for AI gateways", + "language": "Rust", + "status": "dormant (created 2026-10-14; 9 releases in six days; last push 2026-10-19; single maintainer)", + "url": "https://github.com/EeroEternal/xlb", + "capabilities": [ + { + "area": "routing", + "feature": "Phase-shaped timeout hierarchy modeled on LLM request phases: connect (5 s), optional first_byte TTFB budget (expiry before client byte = 504), idle per-direction in-stream silence (600 s), total ceiling, fin half-close grace; cross-field validation", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/architecture.md", + "gap": "new:timeout-hierarchy" + }, + { + "area": "routing", + "feature": "Wall-clock retry budget anchored at request arrival jointly bounding connect retries, stale-socket failovers, and pre-first-byte reroutes; exhaustion rejects with reason=retry_timeout", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/src/proxy/l7.rs", + "gap": "1" + }, + { + "area": "guardrails", + "feature": "Non-idempotent retry safety: non_idempotent_retry_statuses restricts POST/PUT/PATCH/DELETE rerouting to pre-execution refusals while safe methods keep the full list; respect_idempotency_key restores full reroute with a shared dedup ledger", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/changelog.md", + "gap": "1" + }, + { + "area": "routing", + "feature": "Pre-first-byte status rerouting with buffered request-body replay (<=64 MiB): drain error body, return socket to the idle pool, replay on the next healthy backend; never retried after the first response byte", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/README.md", + "gap": "1" + }, + { + "area": "protocol", + "feature": "Thin-L7 HTTP/1.1 framing only (httparse, bodies opaque, 16 KiB buffer) with cross-client POST connection pooling: per-backend LIFO idle pools, non-blocking liveness probes, Connection:close rewrite, h2/WebSocket/binary fallback to raw L4 splice", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/architecture.md", + "gap": "context" + }, + { + "area": "routing", + "feature": "Health checking: active probes with unhealthy/healthy hysteresis, configurable authenticated probe headers (CR/LF validated), pool purge on unhealthy, passive dial-failure ejection with eject_cooldown", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/gateways.md", + "gap": "1" + }, + { + "area": "observability", + "feature": "17 Prometheus metrics incl. xlb_ttft_delta_ms (proxy-added first-byte latency histogram), reroutes/rejected counters by reason, per-backend healthy gauge, bytes by direction", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/observability.md", + "gap": "12;17" + }, + { + "area": "rate-limiting", + "feature": "Per-backend concurrency caps via RAII semaphore permits (acquire_timeout) with leastconn/roundrobin preferring backends below the cap; 503 queue_full on timeout; client-side server.max_connections", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/config.toml", + "gap": "3;new:admission-queueing" + }, + { + "area": "ops", + "feature": "Client-abort propagation: client EOF/RST raced against backend reads so disconnects immediately close the upstream socket (counted as client error), letting gateways/GPU workers cancel generation; TCP keepalive both legs", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/docs/changelog.md", + "gap": "new:upstream-cancellation" + }, + { + "area": "observability", + "feature": "Sanitized multi-attempt trace correlation: secret-redacting trace-id derivation (sk-/rk-/ak- prefixes become ), 64-char cap, logged with attempt counters on every reroute; JSON logs via env", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/src/proxy/http1.rs", + "gap": "context" + }, + { + "area": "ops", + "feature": "Single static binary, TOML config with --check cross-field validation, graceful SIGINT/SIGTERM, SO_REUSEPORT, TCP_NODELAY, rustls TLS termination, musl/darwin checksummed releases, GHCR + Docker Hub multi-arch images", + "sourceUrl": "https://github.com/EeroEternal/xlb/releases/tag/v0.0.9", + "gap": "context" + }, + { + "area": "tooling", + "feature": "~87 KB integration suite starting real mock gateways and a real xLB: no-retry-after-first-byte, first_byte 504, idempotency split, retry ceiling, asymmetric reroute with pool reuse, pooled-backend-death failover, TLS/L4 fallback", + "sourceUrl": "https://github.com/EeroEternal/xlb/blob/main/tests/integration.rs", + "gap": "context" + } + ] + }, + { + "name": "yolorouter", + "category": "self-hosted gateway for AI coding CLIs", + "language": "Go", + "status": "active (v0.2.9, 2026-10-27; last commit 2026-10-28)", + "url": "https://github.com/yolorouter/yolorouter", + "capabilities": [ + { + "area": "auth", + "feature": "Upstream key pool with rate-limit benching: plain 429 benches a key for Retry-After clamped 1 s-10 min; benched keys demoted (soonest-to-recover first), never excluded — an all-cooling pool still dispatches", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/keypool.go", + "gap": "5" + }, + { + "area": "auth", + "feature": "Background key auto-recovery loop (30 s heartbeat, configurable interval) retesting system-kicked keys with no admin action", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/keyrecovery/loop.go", + "gap": "5" + }, + { + "area": "routing", + "feature": "Multi-provider failover with generation-guarded three-state circuit breaker (soft half-weight faults, time-based probe admission, generation token preventing mis-booked state; destination change resets records); failover completes before first byte, then provider locked", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/circuit/circuit.go", + "gap": "1" + }, + { + "area": "routing", + "feature": "Balanced scheduling mode: caller keys spread evenly and pinned to one provider to keep upstream prompt caches warm; re-binds during breaker outages; 4096 (key,model) LRU, process-local", + "sourceUrl": "https://github.com/yolorouter/yolorouter#scheduling-modes", + "gap": "14" + }, + { + "area": "cost", + "feature": "Request-path tool-output compression: anchored-regex detection of go test/pytest/vitest/npm/pnpm logs, git diffs, grep results through a most-specific-first compressor chain; live-zone protected; replace-only-if-shorter; fail-open with skip reasons and token-saved estimates", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/compress/compress.go", + "gap": "new:payload-cost-transforms" + }, + { + "area": "cost", + "feature": "Custom system-prompt injection implemented per egress protocol shape (deterministic for prompt-cache hits) with a published 150 on/off-pair benchmark and honest negative-result reporting", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/capability/systemprompt/inject.go", + "gap": "new:payload-cost-transforms" + }, + { + "area": "auth", + "feature": "Per-caller key governance: budgets (micros), RPM/TPM/concurrency limits, model allowlists, expiry, instant revocation; media jobs reserve priced bounds against budgets in flight", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/service/apikey/api_key_service.go", + "gap": "2;3;9" + }, + { + "area": "protocol", + "feature": "Four chat wire protocols any-to-any (OpenAI Chat/Responses, Anthropic Messages, Gemini) plus images, a pollable video job dialect, and speech; native media providers (DashScope, Kling, MiniMax, Volcengine)", + "sourceUrl": "https://github.com/yolorouter/yolorouter#protocols", + "gap": "6" + }, + { + "area": "routing", + "feature": "Seven-phase configurable relay timeouts (connect, TLS handshake, header, first-byte, body-idle, attempt, request) replacing a single wall clock; ordering validated at startup", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/config/config.go", + "gap": "new:timeout-hierarchy" + }, + { + "area": "cost", + "feature": "Community price catalog (yolorouter/price-catalog) with one-click model import and background verification of imported mappings (probe queue, vision detection, write-nothing probes); unpriced candidates never auto-enabled", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/CHANGELOG.md", + "gap": "new:synced-price-catalogs" + }, + { + "area": "observability", + "feature": "Request logs with full per-attempt routing chains (AttemptsDetail), final upstream URL, compress skip reasons, per-modality billing snapshots; console shows full client and upstream bodies", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/model/request_log.go", + "gap": "17" + }, + { + "area": "observability", + "feature": "W3C traceparent propagation into the audit log (strict version-00 parsing; malformed stores NULL, never invents an id)", + "sourceUrl": "https://github.com/yolorouter/yolorouter/blob/main/internal/gateway/traceparent.go", + "gap": "context" + }, + { + "area": "auth", + "feature": "Multi-user console with OAuth2/OIDC SSO (Zitadel, GitHub, Keycloak, DingTalk, Feishu), per-account usage scoping, admin disable killing keys instantly, AES-256 upstream keys at rest", + "sourceUrl": "https://github.com/yolorouter/yolorouter#why-yolorouter", + "gap": "new:caller-sso-rbac;15;19" + }, + { + "area": "ops", + "feature": "Single binary: embedded console (SQLite/Postgres), self-update, systemd/launchd/Windows installers, global or per-provider http/https/socks5/socks5h egress proxy with ~30 s hot reload, vision-fallback capability", + "sourceUrl": "https://github.com/yolorouter/yolorouter#quick-start", + "gap": "context" + } + ] + } + ] +} From ccf0ec2695734053f8af2aab4f8cfd61944dacb6 Mon Sep 17 00:00:00 2001 From: hmziqagent Date: Wed, 4 Nov 2026 21:30:57 +0100 Subject: [PATCH 2/2] chore: ignore the regenerable root harvest manifest --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ebd85a5..51f6148 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ research-repos/ research-issues/ research-repos-prep.log +research-manifest.json