model-profiles.ts

/**
 * Model-performance priors consumed by the portfolio router.
 *
 * The entries below are a small, auditable seed extracted from the 2026-03-16
 * BlockRun performance run.  They are deliberately weak priors: live data
 * injected by the host should replace them through configuration before a
 * release. Historical numbers must never be presented as a current
 * provider SLA or as task-quality measurements.
 */

import liveProfiles from "./model-profiles.generated.json";

export type ModelPerformanceProfile = {
  measuredAt: string;
  /** Gateway end-to-end latency for the benchmark workload. */
  latencyMs: number;
  /** Tail latency is more relevant than mean latency for urgent requests. */
  p95LatencyMs?: number;
  outputTokensPerSecond: number;
  /** External intelligence index when one was available; not task success. */
  intelligenceIndex?: number;
  /** Failure fraction observed in the same benchmark run. */
  errorRate?: number;
  /** Number of sampled calls behind the observation. */
  samples?: number;
};

// Generated from benchmark files that satisfy the uncached-inference
// invariant. These are weak performance priors (speed/reliability), never
// task-quality labels.
export const LIVE_MODEL_PROFILES: Readonly<Record<string, ModelPerformanceProfile>> = Object.freeze(
  liveProfiles as Record<string, ModelPerformanceProfile>,
);

export const HISTORICAL_MODEL_PROFILES: Readonly<Record<string, ModelPerformanceProfile>> =
  Object.freeze({
    "anthropic/claude-haiku-4.5": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 2305,
      outputTokensPerSecond: 140.6,
    },
    "anthropic/claude-sonnet-4.6": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 2110,
      outputTokensPerSecond: 121.3,
    },
    "deepseek/deepseek-chat": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 1431,
      outputTokensPerSecond: 179.2,
      intelligenceIndex: 32,
    },
    "google/gemini-2.5-flash": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 1238,
      outputTokensPerSecond: 207.6,
      intelligenceIndex: 20,
    },
    "google/gemini-2.5-flash-lite": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 1353,
      outputTokensPerSecond: 192.5,
      intelligenceIndex: 20,
    },
    "google/gemini-2.5-pro": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 1294,
      outputTokensPerSecond: 197.8,
    },
    "google/gemini-3.1-pro": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 1609,
      outputTokensPerSecond: 167.2,
    },
    "openai/gpt-4o-mini": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 2764,
      outputTokensPerSecond: 92.8,
    },
    "openai/gpt-5.3-codex": {
      measuredAt: "2026-03-16T13:50:48Z",
      latencyMs: 7935,
      outputTokensPerSecond: 32.3,
    },
  });