diff --git a/.claude-plugin/mcp.json b/.claude-plugin/mcp.json new file mode 100644 index 0000000..1143eb9 --- /dev/null +++ b/.claude-plugin/mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "elevenlabs": { + "type": "http", + "url": "https://api.us.elevenlabs.io/v1/mcp" + } + } +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a1a7f78..d991630 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "elevenlabs", "displayName": "ElevenLabs", - "version": "0.4.0", + "version": "0.5.0", "description": "ElevenLabs voice AI — text-to-speech, voice agents, and speech engine skills, agent management via the ElevenLabs hosted MCP server, plus Architect skills for building and iterating on ElevenLabs agents.", "author": { "name": "ElevenLabs", @@ -15,5 +15,5 @@ "keywords": ["elevenlabs", "text-to-speech", "voice", "audio", "ai", "tts", "voice-agents", "mcp"], "skills": ["./skills/general", "./skills/mcp", "./architect/skills"], "commands": ["./architect/commands"], - "mcpServers": "./mcp.json" + "mcpServers": "./.claude-plugin/mcp.json" } diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index ff5944f..f231654 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "elevenlabs", - "version": "0.4.0", + "version": "0.5.0", "description": "ElevenLabs voice AI — text-to-speech, voice agents, and speech engine skills, agent management via the ElevenLabs hosted MCP server, plus Architect skills for building and iterating on ElevenLabs agents.", "author": { "name": "ElevenLabs", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 2caa930..7c5d421 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "elevenlabs", "displayName": "ElevenLabs", - "version": "0.4.0", + "version": "0.5.0", "description": "ElevenLabs voice AI — text-to-speech, voice agents, and speech engine skills, agent management via the ElevenLabs hosted MCP server, plus Architect skills for building and iterating on ElevenLabs agents.", "author": { "name": "ElevenLabs", diff --git a/README.md b/README.md index edc6a1e..6bdc430 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ Skills live in three trees: | [agents](skills/general/agents/SKILL.md) | Build real-time voice AI agents and assistants | | [dubbing](skills/general/dubbing/SKILL.md) | Dub audio and video into other languages, preserving the original speakers' voices | | [music](skills/general/music/SKILL.md) | Generate instrumental tracks, songs with lyrics, background music, and jingles | -| [setup-api-key](skills/general/setup-api-key/SKILL.md) | Guide users through configuring an ElevenLabs API key | +| [onboarding](skills/general/onboarding/SKILL.md) | Add ElevenLabs to a project from scratch: sign-in, API key, SDK, and a first request | +| [realtime](skills/general/realtime/SKILL.md) | Run agents on ElevenLabs Realtime models, or migrate an OpenAI Realtime voice app to them | +| [setup-api-key](skills/general/setup-api-key/SKILL.md) | Fix or hand-configure an existing ElevenLabs API key | | [sound-effects](skills/general/sound-effects/SKILL.md) | Generate sound effects and audio textures from text descriptions | | [speech-engine](skills/general/speech-engine/SKILL.md) | Add real-time voice conversations to your own agent runtime | | [speech-to-text](skills/general/speech-to-text/SKILL.md) | Transcribe audio to text with Scribe v2 | @@ -71,7 +73,7 @@ Architect also ships one command: [/thermo-nuclear-agent-triage-burndown](archit ### MCP Server -The [ElevenLabs hosted MCP server](https://api.elevenlabs.io/v1/mcp) — no local install, no API key. Your agent authenticates with your ElevenLabs account over OAuth and gets access to agent management (create, update, list, duplicate, delete agents; inspect widget config, links, and knowledge base size; estimate LLM usage) and text-to-speech generation. Configured in [mcp.json](mcp.json). +The [ElevenLabs hosted MCP server](https://api.elevenlabs.io/v1/mcp) — no local install, no API key. Your agent authenticates with your ElevenLabs account over OAuth and gets access to agent management (create, update, list, duplicate, delete agents; inspect widget config, links, and knowledge base size; estimate LLM usage) and text-to-speech generation. Configured in [mcp.json](mcp.json) for Cursor, [.claude-plugin/mcp.json](.claude-plugin/mcp.json) for Claude Code, and [.codex-plugin/mcp.json](.codex-plugin/mcp.json) for Codex. ## Installation diff --git a/skills/general/agents/SKILL.md b/skills/general/agents/SKILL.md index d178cde..f9b1afb 100644 --- a/skills/general/agents/SKILL.md +++ b/skills/general/agents/SKILL.md @@ -10,26 +10,29 @@ metadata: {"openclaw": {"requires": {"env": ["ELEVENLABS_API_KEY"]}, "primaryEnv Build voice AI agents with natural conversations, multiple LLM providers, custom tools, and easy web embedding. -> **Setup:** See [Installation Guide](references/installation.md) for CLI and SDK setup. +> **Setup, before writing any code:** if `ELEVENLABS_API_KEY` isn't set for this project, run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder first and follow what it prints: it signs the user in and stores a key for the project, so nobody copies one by hand. When the user approved in the browser, that also signed the ElevenLabs CLI in, so don't run `elevenlabs auth login` after it. When they added the key by hand (on a remote machine), or a CLI command answers 401, load the env file into the command instead, for example `(set -a && . ./.env.local && set +a && npx -y @elevenlabs/cli@latest agents list)`; the CLI reads a `.env` on its own. Otherwise see the [Installation Guide](references/installation.md) for CLI and SDK setup. ## Quick Start with CLI The ElevenLabs CLI is the recommended way to create and manage agents: ```bash -# Install CLI and authenticate +# Install CLI and authenticate. After onboarding, skip these two lines and run each +# command below through npx instead, e.g. `npx -y @elevenlabs/cli@latest agents init` npm install -g @elevenlabs/cli elevenlabs auth login # Initialize project and create an agent elevenlabs agents init -elevenlabs agents add "My Assistant" --template complete +elevenlabs agents add "My Assistant" --template default # Push to ElevenLabs platform elevenlabs agents push ``` -**Available templates:** `complete`, `minimal`, `voice-only`, `text-only`, `customer-service`, `assistant` +**Available templates:** `default`, `minimal`, `voice-only`, `text-only`, `customer-service`, `assistant` + +Once the agent exists, give the user its links: they can talk to it at `https://elevenlabs.io/app/talk-to?agent_id=` and edit it at `https://elevenlabs.io/app/agents/agents/`. On a data residency account, use its region's host instead of `elevenlabs.io`, such as `eu.residency.elevenlabs.io`. The talk-to page only opens public agents, which is the default; for an agent with authentication enabled, give only the dashboard link. ### Python @@ -168,15 +171,17 @@ function App({ signedUrl }: { signedUrl: string }) { | Provider | Models | |----------|--------| -| OpenAI | `gpt-6-sol`, `gpt-6-luna`, `gpt-6-astra`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5`, `gpt-5.5-2026-04-23`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.4-2026-03-05`, `gpt-5.4-mini-2026-03-17`, `gpt-5.4-nano-2026-03-17`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo` | -| Anthropic | `claude-opus-5-5`, `claude-opus-5`, `claude-opus-4-7`, `claude-sonnet-4-6`, `claude-sonnet-4-5`, `claude-sonnet-4`, `claude-haiku-4-5`, `claude-3-7-sonnet`, `claude-3-5-sonnet`, `claude-3-haiku` | -| Google | `gemini-3.8-flash`, `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.1-flash-lite-preview`, `gemini-3.1-pro-preview`, `gemini-3-pro-preview`, `gemini-3-flash-preview`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite` | -| ElevenLabs | `glm-52`, `deepseek-v41-flash`, `glm-45-air-fp8`, `qwen3-30b-a3b`, `qwen36-35b-a3b`, `qwen35-35b-a3b`, `qwen35-397b-a17b`, `gpt-oss-120b` | +| OpenAI | `gpt-6.1-sol`, `gpt-6.1-terra`, `gpt-6-sol`, `gpt-6-luna`, `gpt-6-astra`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5`, `gpt-5.5-2026-04-23`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.4-2026-03-05`, `gpt-5.4-mini-2026-03-17`, `gpt-5.4-nano-2026-03-17`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo` | +| Anthropic | `claude-opus-5-5`, `claude-opus-5`, `claude-sonnet-5-5`, `claude-opus-4-7`, `claude-opus-4-6`, `claude-sonnet-4-6`, `claude-sonnet-4`, `claude-haiku-4-5`, `claude-3-7-sonnet`, `claude-3-5-sonnet`, `claude-3-haiku` | +| Google | `gemini-4-argon`, `gemini-3.8-flash`, `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.5-flash`, `gemini-3.5-flash-lite`, `gemini-3.1-flash-lite-preview`, `gemini-3.1-pro-preview`, `gemini-3-pro-preview`, `gemini-3-flash-preview`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite` | +| ElevenLabs | `glm-52`, `deepseek-v41-flash`, `glm-45-air-fp8`, `qwen3-30b-a3b`, `qwen36-35b-a3b`, `qwen35-35b-a3b`, `qwen35-397b-a17b`, `nemotron-35-super-vl-nvfp4`, `gpt-oss-120b` | | Custom | `custom-llm` (bring your own endpoint) | Use `GET /v1/convai/llm/list` to inspect the current model catalog, including deprecation state, token/context limits, capability flags such as image-input support, and model-specific reasoning effort support. -**Popular voices:** `JBFqnCBsd6RMkjVDRZzb` (George), `EXAVITQu4vr4xnSDxMaL` (Sarah), `onwK4e9ZLuTAKqWW03F9` (Daniel), `XB0fDUnXU5powFXDhCwa` (Charlotte) +To run an agent on an ElevenLabs Realtime model instead of a separate LLM and TTS model (`conversation_config.realtime_model`), use the `realtime` skill. + +**Popular voices:** `JBFqnCBsd6RMkjVDRZzb` (George), `EXAVITQu4vr4xnSDxMaL` (Sarah), `onwK4e9ZLuTAKqWW03F9` (Daniel) **Turn eagerness:** `patient` (waits longer for user to finish), `normal`, or `eager` (responds quickly) @@ -298,7 +303,7 @@ No-auth APIs useful for prototypes (URLs must be HTTPS): Route conversations through discrete steps with branching logic. Define under the agent's top-level `workflow` field. Reference: [Agent Workflows](https://elevenlabs.io/docs/eleven-agents/customization/agent-workflows). -**Node types:** `start` (ID must be `"start_node"`), `end`, `override_agent` (subagent step with `label` + `additional_prompt`), `dispatch_tool` (executes a tool with success/failure routing), `agent_transfer`, `transfer_to_number`. +**Node types:** `start` (ID must be `"start_node"`), `end` (`outcome` records the conversation as `success`, the default, or `failure`), `override_agent` (subagent step with `label` + `additional_prompt`), `dispatch_tool` (executes a tool with success/failure routing), `agent_transfer`, `transfer_to_number`. **Edge types:** `unconditional`, `llm` (natural-language condition), `expression` (deterministic data check). Tool nodes have separate success/failure edges. @@ -337,14 +342,15 @@ Semantics worth knowing before writing any of these calls: - Nothing reaches the live agent until you publish. Publishing is not a procedure endpoint; one PATCH on the agent versions every changed procedure draft on the branch. - `GET .../procedures/{procedure_id}` reads branch HEAD and returns `404` until that procedure's first publish. Read the `/draft` variant to see a procedure you just created; do not retry the create. -- Turning a structured procedure's steps into the form the agent executes is called compiling. When you publish, the platform validates every structured procedure on the branch, compiles them, and stores the result with the new version. You do not compile anything yourself and do not send a `workflow` in the request; `agents.update` with `branch_id` publishes, and compilation happens as part of that. The compiled result is currently visible as read-only nodes in the dashboard's Workflow tab. A failed publish writes nothing, so publishing is also the validation step. To validate without publishing, save an agent draft with `POST /v1/convai/agents/{agent_id}/drafts?branch_id=...`, sending the agent's current `name`, `conversation_config`, `platform_settings`, and `workflow` unchanged, then discard that agent draft with `DELETE` on the same path; see [Using the Procedure API](references/using-procedure-api.md#validate-without-publishing). +- Turning a structured procedure's steps into the form the agent executes is called compiling. The platform compiles every structured procedure when the agent draft is saved or the agent is published; authors never compile anything themselves. Both draft save and publish validate every structured procedure on the branch. A failed operation writes nothing. To validate without publishing, save an agent draft with `POST /v1/convai/agents/{agent_id}/drafts?branch_id=...`, sending the agent's current `name`, `conversation_config`, `platform_settings`, and `workflow` unchanged, then discard that agent draft with `DELETE` on the same path; see [Using the Procedure API](references/using-procedure-api.md#validate-without-publishing). - If a structured procedure is invalid, the publish or draft save returns `400` with `status` `procedure_validation_failed` and `errors` keyed by procedure ID, each entry carrying the `path` of the offending field and a message. Nothing is written; repair the procedure draft and publish again. - A draft update replaces the whole body. Read the draft first, then resend `name`, `type`, and `trigger` alongside the new `content`. `type` cannot change after creation. - `content` is markdown for a `free_form` procedure, and a JSON-encoded object with a `steps` array for a `deterministic` one. The trigger is the top-level `trigger` field in both cases, not part of `content`. Serialize it; do not hand-escape quotes. +- Manually authored workflow edges cannot connect to or from a structured procedure. Use a `sub_procedure` step to chain structured procedures; entry from a matching trigger and completion are platform-managed. - A third `type`, `folder`, groups procedures in the dashboard. Folders carry no `content` or `trigger`; move a procedure into one with `POST .../procedures/{procedure_id}/move`. - The agent keeps the five most recently started procedures in context. When more have been started, the content, inline tools, and knowledge base documents of the oldest free-form ones drop out of the prompt, although the procedures remain active. Keep procedures focused and use sub-procedures so that few are active at once. - Routing is driven by the `trigger` text, not the procedure name. Write concrete, non-overlapping triggers that cover the phrasings a user would actually say. The model sees only a numbered menu of triggers; it never sees procedure names or IDs. -- To restrict the starting agent to selected procedures for one conversation, enable `platform_settings.overrides.enable_procedure_ids_from_client`, then pass their IDs as `procedure_ids` in conversation initiation data. An empty list disables all procedures for that starting agent. +- To restrict the starting agent to selected procedures for one conversation, enable `platform_settings.overrides.enable_procedure_ids_from_client`, then pass their IDs as `procedure_ids` in conversation initiation data. An empty list disables all procedures for that starting agent. The same setting allows `start_procedure_ids`: up to five free-form procedures that start with the conversation, for when the user's task is already known (for example from an IVR choice or the page hosting the widget). - Procedure APIs require `elevenlabs` (Python) or `@elevenlabs/elevenlabs-js` at `2.60.0` or newer. ## Guardrails @@ -365,7 +371,7 @@ Layered safety enforcement that runs independently of the LLM — configured und "name": "No medical diagnoses", "prompt": "Block the agent from providing medical diagnoses or treatment advice.", "execution_mode": "blocking", - "model": "gemini-2.5-flash-lite", + "model": "gemini-3.1-flash-lite", "history_message_count": 1, "trigger_action": {"type": "retry", "feedback": "Reason: {{trigger_reason}}"} }] @@ -495,9 +501,9 @@ elevenlabs agents pull --agent # Import specific agent elevenlabs agents push # Upload configurations elevenlabs agents push --dry-run # Preview changes first -# Add tools -elevenlabs tools add-webhook "Weather API" -elevenlabs tools add-client "UI Tool" +# Add tools (names use letters, numbers, - and _) +elevenlabs tools add weather-api --type webhook +elevenlabs tools add ui-tool --type client ``` ### Project Structure diff --git a/skills/general/agents/references/agent-configuration.md b/skills/general/agents/references/agent-configuration.md index 044cdbe..e8d4253 100644 --- a/skills/general/agents/references/agent-configuration.md +++ b/skills/general/agents/references/agent-configuration.md @@ -184,6 +184,7 @@ conversation_config={ |-------|------|---------|-------------| | `prompt` | string | `""` | System prompt defining agent behavior | | `llm` | string | - | Model ID (see LLM providers below) | +| `service_tier` | string | `"standard"` | `standard` or `fast`. `fast` trades a per-token price premium for higher token throughput; only some models offer it (see `available_service_tiers` in `GET /v1/convai/llm/list`) | | `temperature` | float | `0` | 0-1, higher = more creative | | `max_tokens` | int | `-1` | Max tokens for LLM response (-1 = unlimited) | | `reasoning_effort` | string | - | Reasoning depth: `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, or `max` (model-dependent) | @@ -211,10 +212,10 @@ to resolve per-environment auth connections at runtime. | Provider | Model IDs | |----------|-----------| -| OpenAI | `gpt-6-sol`, `gpt-6-luna`, `gpt-6-astra`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5`, `gpt-5.5-2026-04-23`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.4-2026-03-05`, `gpt-5.4-mini-2026-03-17`, `gpt-5.4-nano-2026-03-17`, `gpt-5`, `gpt-5-mini`, `gpt-5-nano`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo` | -| Anthropic | `claude-opus-5-5`, `claude-opus-5`, `claude-opus-4-7`, `claude-sonnet-4-6`, `claude-sonnet-4-5`, `claude-sonnet-4`, `claude-haiku-4-5`, `claude-3-7-sonnet`, `claude-3-5-sonnet`, `claude-3-haiku` | -| Google | `gemini-3.8-flash`, `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.1-flash-lite-preview`, `gemini-3.1-pro-preview`, `gemini-3-pro-preview`, `gemini-3-flash-preview`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite` | -| ElevenLabs | `glm-52`, `deepseek-v41-flash`, `glm-45-air-fp8`, `qwen3-30b-a3b`, `qwen36-35b-a3b`, `qwen35-35b-a3b`, `qwen35-397b-a17b`, `gpt-oss-120b` (hosted, ultra-low latency) | +| OpenAI | `gpt-6.1-sol`, `gpt-6.1-terra`, `gpt-6-sol`, `gpt-6-luna`, `gpt-6-astra`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5`, `gpt-5.5-2026-04-23`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano`, `gpt-5.4-2026-03-05`, `gpt-5.4-mini-2026-03-17`, `gpt-5.4-nano-2026-03-17`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4o`, `gpt-4o-mini`, `gpt-4-turbo` | +| Anthropic | `claude-opus-5-5`, `claude-opus-5`, `claude-sonnet-5-5`, `claude-opus-4-7`, `claude-opus-4-6`, `claude-sonnet-4-6`, `claude-sonnet-4`, `claude-haiku-4-5`, `claude-3-7-sonnet`, `claude-3-5-sonnet`, `claude-3-haiku` | +| Google | `gemini-4-argon`, `gemini-3.8-flash`, `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.5-flash`, `gemini-3.5-flash-lite`, `gemini-3.1-flash-lite-preview`, `gemini-3.1-pro-preview`, `gemini-3-pro-preview`, `gemini-3-flash-preview`, `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.0-flash`, `gemini-2.0-flash-lite` | +| ElevenLabs | `glm-52`, `deepseek-v41-flash`, `glm-45-air-fp8`, `qwen3-30b-a3b`, `qwen36-35b-a3b`, `qwen35-35b-a3b`, `qwen35-397b-a17b`, `nemotron-35-super-vl-nvfp4`, `gpt-oss-120b` (hosted, ultra-low latency) | | Custom | `custom-llm` (requires custom_llm config) | Use `GET /v1/convai/llm/list` to inspect the current model catalog, including deprecation state, token/context limits, and capability flags such as image-input support. @@ -366,7 +367,7 @@ Use `platform_settings.guardrails` to configure built-in safety controls for use | `name` | string | User-facing guardrail name. | | `prompt` | string | Instruction describing what to block. | | `execution_mode` | string | Guardrail execution mode: `streaming` or `blocking`. | -| `model` | string | LLM model used for custom guardrail evaluation, such as `gemini-2.5-flash-lite`, `claude-sonnet-4-6`, or `gpt-5.4-mini`. | +| `model` | string | LLM model used for custom guardrail evaluation, such as `gemini-3.1-flash-lite`, `claude-sonnet-4-6`, or `gpt-5.4-mini`. | | `history_message_count` | integer | Number of recent customer messages to include in guardrail history; `0` includes none. | | `trigger_action` | object | Action when triggered, such as retrying with feedback or ending the call. | | `evaluate_full_response_only` | bool | Evaluate the complete non-TTS response once. Requires `execution_mode` set to `blocking`; defaults to `false`. | @@ -550,7 +551,7 @@ agent dependencies and recursively delete the contents of non-empty folders. elevenlabs agents init # Create agent from template -elevenlabs agents add "My Agent" --template complete +elevenlabs agents add "My Agent" --template default elevenlabs agents add "Support Bot" --template customer-service # List agents @@ -572,12 +573,12 @@ elevenlabs agents pull --update # Override local configs elevenlabs agents templates list elevenlabs agents templates show -# Add tools -elevenlabs tools add-webhook "API Tool" -elevenlabs tools add-client "UI Tool" +# Add tools (names use letters, numbers, - and _) +elevenlabs tools add api-tool --type webhook +elevenlabs tools add ui-tool --type client # Generate widget code -elevenlabs agents widget +elevenlabs agents widget embed ``` ### SDK: List Agents @@ -771,7 +772,7 @@ agent = client.conversational_ai.agents.create( } } }, - "tts": {"voice_id": "XB0fDUnXU5powFXDhCwa", "model_id": "eleven_v4_turbo"}, + "tts": {"voice_id": "EXAVITQu4vr4xnSDxMaL", "model_id": "eleven_v4_turbo"}, "turn": {"turn_eagerness": "normal", "turn_timeout": 7}, "conversation": {"max_duration_seconds": 900} } diff --git a/skills/general/agents/references/client-tools.md b/skills/general/agents/references/client-tools.md index 5062b6e..ec7325f 100644 --- a/skills/general/agents/references/client-tools.md +++ b/skills/general/agents/references/client-tools.md @@ -369,6 +369,7 @@ When users want to go somewhere, use navigate_to.""", | Field | Type | Default | Description | |-------|------|---------|-------------| | `expects_response` | bool | `false` | Whether the tool returns data to the agent | +| `response_timeout_secs` | int | `20` | Seconds to wait for the client to respond (1-120), or `-1` to wait indefinitely. `-1` requires `expects_response` | ### Client Tool Return Values diff --git a/skills/general/agents/references/installation.md b/skills/general/agents/references/installation.md index a05302c..426bacb 100644 --- a/skills/general/agents/references/installation.md +++ b/skills/general/agents/references/installation.md @@ -33,11 +33,11 @@ Set `ELEVENLABS_API_KEY` in your environment — the CLI picks it up automatical export ELEVENLABS_API_KEY="your-api-key" ``` -Or authenticate with OAuth, which stores credentials in the OS keyring: +Or authenticate with OAuth, which stores credentials in the OS keyring (`onboard init` already did this when the user approved in the browser): ```bash elevenlabs auth login # Authenticate with OAuth -elevenlabs auth whoami # Verify current login status +elevenlabs auth status # Verify current login status elevenlabs auth logout # Remove stored credentials ``` @@ -140,13 +140,13 @@ elevenlabs agents create \ ## Getting an API Key +Run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder and follow what it prints: it signs the user in and stores a key for the project. For an existing key that stopped working or was issued by an administrator, use the `setup-api-key` skill. To do it by hand instead: + 1. Sign up at [elevenlabs.io](https://elevenlabs.io) 2. Go to [API Keys](https://elevenlabs.io/app/settings/api-keys) 3. Click **Create API Key** 4. Copy and store securely -Or use the `setup-api-key` skill for guided setup. - ## Environment Variables | Variable | Description | diff --git a/skills/general/agents/references/outbound-calls.md b/skills/general/agents/references/outbound-calls.md index e0ed2cb..3ac188d 100644 --- a/skills/general/agents/references/outbound-calls.md +++ b/skills/general/agents/references/outbound-calls.md @@ -184,6 +184,23 @@ Detection runs asynchronously. Its verdict arrives through the separate `answering_machine_detection` webhook event, which must be enabled in the workspace or agent webhook settings. +To act on the verdict during the call, set `twilio_machine_detection.actions` to a list of rules. +Each rule pairs an `action` (`hang_up`) with the `answered_by` verdicts that trigger it: `human`, +`fax`, `unknown`, `machine_start`, `machine_end_beep`, `machine_end_silence`, or +`machine_end_other`. The `machine_end_*` verdicts require `detect_message_end` mode. Verdicts +without a rule leave the call running. + +```json +{ + "telephony_call_config": { + "twilio_machine_detection": { + "mode": "enable", + "actions": [{ "action": "hang_up", "answered_by": ["machine_start", "fax"] }] + } + } +} +``` + ### Dynamic Variables Pass custom data to your agent's prompt using `dynamic_variables`. Reference them in your agent's prompt with `{{variable_name}}` syntax. diff --git a/skills/general/agents/references/using-procedure-api.md b/skills/general/agents/references/using-procedure-api.md index bd86787..3300b94 100644 --- a/skills/general/agents/references/using-procedure-api.md +++ b/skills/general/agents/references/using-procedure-api.md @@ -4,7 +4,7 @@ Procedures are reusable instruction blocks that an agent runs when a trigger mat For what belongs in `trigger` and `content`, see [Writing Procedures](writing-procedures.md). -One term to know: turning a structured procedure's steps into the form the agent executes is called compiling. The platform compiles every structured procedure on the branch when you publish; you never compile anything yourself or send a `workflow` in a request. The compiled result is currently visible as read-only nodes in the dashboard's Workflow tab. +Turning a structured procedure's steps into the form the agent executes is called compiling. The platform compiles every structured procedure when the agent draft is saved or the agent is published; authors never compile anything themselves. Both operations validate every structured procedure on the branch and return any errors. Workflow edges cannot connect to or from a structured procedure or its compiled elements. Use a Sub-procedure step to chain structured procedures; entry from a matching trigger and completion are platform-managed. The CLI exposes the complete procedure lifecycle, including draft operations. @@ -290,8 +290,6 @@ On a validation failure the publish returns `400` and nothing is written: Saving a procedure draft with `PATCH .../procedures/{procedure_id}/draft` does not validate structured content. The check happens at publish, or at agent draft save as described next. -The legacy `POST .../procedures/compile` endpoint still exists as a dry-run for existing callers, but do not use it. It will eventually be deprecated. - ### Validate without publishing A failed publish writes nothing, so when you are ready to publish, the publish itself is the validation step. Use the flow below only when you need to check structured content and are not ready to publish, for example while other edits on the branch are still pending, or on a protected branch you cannot publish to. diff --git a/skills/general/agents/references/widget-embedding.md b/skills/general/agents/references/widget-embedding.md index 96c2d82..c56db52 100644 --- a/skills/general/agents/references/widget-embedding.md +++ b/skills/general/agents/references/widget-embedding.md @@ -52,6 +52,9 @@ show a message entry point, and multimodal agents show both. | `dismissible` | Allow the user to minimize the widget | `false` | | `disable-banner` | Hide "Powered by ElevenLabs" | `false` | | `show-resize-button` | Show the expand and collapse control in the widget header | `true` | +| `mobile-fullscreen` | Open the widget fullscreen on mobile screens; set `false` to keep the compact layout (agent-level setting: `platform_settings.widget.mobile_fullscreen`) | `true` | +| `transcript-speech-sync` | Reveal the voice call transcript as the agent speaks; set `false` to show each message in full at once (agent-level setting: `platform_settings.widget.transcript_speech_sync_enabled`) | `true` | +| `end-confirmation` | Ask the user to confirm before ending a chat or call | `false` | ## Examples diff --git a/skills/general/agents/references/writing-procedures.md b/skills/general/agents/references/writing-procedures.md index c7d9f6c..1de13d3 100644 --- a/skills/general/agents/references/writing-procedures.md +++ b/skills/general/agents/references/writing-procedures.md @@ -75,6 +75,7 @@ Saving an agent draft or publishing rejects a structured procedure that breaks a - Only `ask` waits for the user. Every other step runs immediately and control moves to the next step within the same turn. There is no step that stops the turn other than `ask`; use `end_call` to end the conversation. - Reaching the end of a procedure does not end the turn. The agent returns to its normal behavior with the turn still open and may say more. +- Manually authored workflow edges cannot connect to or from a structured procedure. Use a `sub_procedure` step to chain structured procedures; entry from a matching trigger and completion are platform-managed. - `ask`, `tell`, and `say` steps have no tools. Do not write "do not call tools" into them. - Decisions made inside a `branch` arm are not remembered by later steps. Persist anything needed downstream with a tool call or a dynamic variable. diff --git a/skills/general/dubbing/SKILL.md b/skills/general/dubbing/SKILL.md index d3f17e0..a402f53 100644 --- a/skills/general/dubbing/SKILL.md +++ b/skills/general/dubbing/SKILL.md @@ -12,7 +12,7 @@ Dub audio or video into other languages while preserving the original speakers' > **Important:** Use the Dubbing Projects API — `elevenlabs.dubbing.project.*` in the SDKs, or the `/v1/dubbing/project` REST endpoints. Do **not** use the legacy v1 dubbing surface (`client.dubbing.create()`, `client.dubbing.get()`, `client.dubbing.audio.get()`, or bare `/v1/dubbing` routes) — that is the older dubbing API, now under Legacy in the API reference. -> **Setup:** See [Installation Guide](references/installation.md). The `elevenlabs` CLI and the SDKs read `ELEVENLABS_API_KEY` automatically; REST base URL is `https://api.elevenlabs.io` with your API key in the `xi-api-key` header. +> **Setup, before writing any code:** if `ELEVENLABS_API_KEY` isn't set for this project, run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder first and follow what it prints: it signs the user in and stores a key for the project, so nobody copies one by hand. Otherwise see the [Installation Guide](references/installation.md). The `elevenlabs` CLI and the SDKs read `ELEVENLABS_API_KEY` automatically; REST base URL is `https://api.elevenlabs.io` with your API key in the `xi-api-key` header. ## Concepts diff --git a/skills/general/dubbing/references/api-reference.md b/skills/general/dubbing/references/api-reference.md index fcd3ff5..c2aa515 100644 --- a/skills/general/dubbing/references/api-reference.md +++ b/skills/general/dubbing/references/api-reference.md @@ -239,7 +239,7 @@ A language can be added to a project that isn't `ready` yet — it stays `queued `GET /v1/dubbing/project/{project_id}/language/{language_id}` -Once `completed`, `outputs` carries a signed, time-limited download URL (valid ~1 hour; re-fetch the language for a fresh URL): +Once `completed`, `outputs` carries signed, time-limited download URLs (valid ~1 hour; re-fetch the language for a fresh URL): `lossless_audio` for the dubbed audio and `subtitles` for the translated lines as WebVTT subtitles timed to the source speech (`null` when none are available): ```json { @@ -249,7 +249,8 @@ Once `completed`, `outputs` carries a signed, time-limited download URL (valid ~ "status": "completed", "model_id": "dubbing_v2", "outputs": { - "lossless_audio": "https://storage.googleapis.com/eleven-dubbing/.../output.wav?X-Goog-Signature=..." + "lossless_audio": "https://storage.googleapis.com/eleven-dubbing/.../output.wav?X-Goog-Signature=...", + "subtitles": "" }, "revision": 0, "output_revision": 0, diff --git a/skills/general/dubbing/references/installation.md b/skills/general/dubbing/references/installation.md index d219122..66a069e 100644 --- a/skills/general/dubbing/references/installation.md +++ b/skills/general/dubbing/references/installation.md @@ -92,13 +92,13 @@ elevenlabs dubbing project create --file promo.mp4 --source-language en ## Getting an API Key +Run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder and follow what it prints: it signs the user in and stores a key for the project. For an existing key that stopped working or was issued by an administrator, use the `setup-api-key` skill. To do it by hand instead: + 1. Sign up at [elevenlabs.io](https://elevenlabs.io) 2. Go to [API Keys](https://elevenlabs.io/app/settings/api-keys) 3. Click **Create API Key** 4. Copy and store securely -Or use the `setup-api-key` skill for guided setup. - ## Environment Variables | Variable | Description | diff --git a/skills/general/music/SKILL.md b/skills/general/music/SKILL.md index a4b0fdc..4126f7f 100644 --- a/skills/general/music/SKILL.md +++ b/skills/general/music/SKILL.md @@ -10,7 +10,7 @@ metadata: {"openclaw": {"requires": {"env": ["ELEVENLABS_API_KEY"]}, "primaryEnv Generate music from text prompts - supports instrumental tracks, songs with lyrics, and fine-grained control via composition plans. -> **Setup:** See [Installation Guide](references/installation.md). For JavaScript, use `@elevenlabs/*` packages only. +> **Setup, before writing any code:** if `ELEVENLABS_API_KEY` isn't set for this project, run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder first and follow what it prints: it signs the user in and stores a key for the project, so nobody copies one by hand. Otherwise see the [Installation Guide](references/installation.md). For JavaScript, use `@elevenlabs/*` packages only. All examples below use `music_v2_5`, the most advanced generation model. Pass `music_v2` or `music_v1` only when an older model is explicitly requested. @@ -104,6 +104,10 @@ The API combines videos in order, accepts an optional natural-language descripti steer style with up to 10 tags such as `upbeat` or `cinematic`. This endpoint still defaults to `music_v1`; pass `model_id="music_v2_5"` to use the most advanced model. +Three optional flags shape the result: `use_audio_as_reference` takes genre, BPM, instrumentation, +and mood from the video's own audio, `add_sfx` adds sound effects that match the video, and +`separate_tracks` returns a ZIP with the music and sound effects as separate files for mixing. + ### Python ```python diff --git a/skills/general/music/references/api_reference.md b/skills/general/music/references/api_reference.md index 6e7fc4e..e9cf120 100644 --- a/skills/general/music/references/api_reference.md +++ b/skills/general/music/references/api_reference.md @@ -352,6 +352,9 @@ Videos are combined in order before music generation. | `description` | string | No | Optional text prompt describing the desired music (up to 1000 characters). | | `tags` | array of strings | No | Optional style tags such as `upbeat` or `cinematic` (up to 10 tags). | | `model_id` | string | No | Music model to use. Defaults to `music_v1` on this endpoint; pass `music_v2_5` for the most advanced model. | +| `use_audio_as_reference` | boolean | No | Use the genres, BPM, instrumentation, mood, and arc of the video's own audio track as reference. Defaults to `false`. | +| `add_sfx` | boolean | No | Add sound effects that match the video. Defaults to `false`. | +| `separate_tracks` | boolean | No | Return a ZIP with separate music and sound effects tracks instead of a single mixed file. Defaults to `false`. | | `sign_with_c2pa` | boolean | No | Sign generated MP3 output with C2PA metadata. Defaults to `false`. | | `output_format` | string | No | Output codec/sample-rate/bitrate, such as `mp3_44100_128`, `pcm_44100`, or `opus_48000_96`. | diff --git a/skills/general/music/references/installation.md b/skills/general/music/references/installation.md index 063afd1..e3ccd5e 100644 --- a/skills/general/music/references/installation.md +++ b/skills/general/music/references/installation.md @@ -86,11 +86,11 @@ elevenlabs music compose \ ## Getting an API Key +Run `npx -y @elevenlabs/cli@latest onboard init` from the app's folder and follow what it prints: it signs the user in and stores a key for the project. For an existing key that stopped working or was issued by an administrator, use the `setup-api-key` skill. To do it by hand instead: + 1. Sign up at [elevenlabs.io](https://elevenlabs.io) 2. Go to [API Keys](https://elevenlabs.io/app/settings/api-keys) 3. Click **Create API Key** 4. Copy and store securely -Or use the `setup-api-key` skill for guided setup. - **Note:** Music generation requires a paid ElevenLabs plan. diff --git a/skills/general/onboarding/SKILL.md b/skills/general/onboarding/SKILL.md new file mode 100644 index 0000000..8617727 --- /dev/null +++ b/skills/general/onboarding/SKILL.md @@ -0,0 +1,86 @@ +--- +name: onboarding +description: Add ElevenLabs to a project from scratch. Use when the user wants to start using ElevenLabs, add text to speech, speech to text, voice agents, real-time voice for their own agent runtime (Speech Engine), sound effects, music, voice changing, voice isolation, or dubbing to their app, or get set up with an ElevenLabs account or API key for the first time. Handles sign-in, API key creation, SDK install and a test request end to end; the user leaves the conversation once, to approve a browser page. Not for fixing a key that stopped working or placing one an administrator issued (use setup-api-key). +license: MIT +compatibility: Requires internet access to elevenlabs.io and api.elevenlabs.io, and Node.js for npx (or the ElevenLabs CLI installed another way). Where no browser can reach the machine (SSH, containers, cloud agents), the user adds the key by hand and the flow continues. +--- + +# ElevenLabs Onboarding + +The ElevenLabs CLI runs the whole setup and tells you each step. From the +folder of the app that will use the key (in a monorepo, that app's folder), +run: + +```bash +npx -y @elevenlabs/cli@latest onboard init +``` + +Every `onboard` command prints JSON. Act on it, and keep going until `next` +is `done`: + +- `next`: what to do now, spelled out in `instruction`. +- `run`: the exact next command. Run it as given. +- `say`: what to tell the user. Tell them as written. +- `details`: facts for the next step, such as how the app loads the key and + the SDK install command. +- `rules`: rules for the whole setup. Follow them. + +What each `next` means: + +- `run`: run `run`. +- `wait`: the approval page is open. Show `say` (it has the link), then run + `run` again right away, without ending your turn. Whenever you give the + user the link, copy it whole, exactly as printed: a shortened link doesn't + work. +- `ask_what_to_build`: tell the user `say`, which asks what they want to + build or whether to just add the SDK, and wait for their answer before + writing any code. Ask it even if they said earlier, and don't pick a product + for them. Once they answer, run `run`. +- `write_code`: write what they asked for, as `instruction` says: the SDK + alone, or the integration, with the skill that fits from + `details.fetch_skills` when there is one. If it needs something made in + ElevenLabs first, like an agent, make it yourself with the key (load the + env file in the command, never print it), not a script for the user to run. + Load the key the way `details.load_key` says, then run `run`. +- `load_key`: the code already uses the SDK, so there is nothing to build. + Make sure the app loads the key the way `details.load_key` says, change + nothing else, then run `run`. +- `fix`: do what `say` describes, then run `run`. If it needs the user's + shell, settings or secrets, ask them instead, as for `ask_user`. +- `ask_user`: ask the user what `say` says and wait for their answer, then + run `run`, unless they said no. +- `offer_test`: say what you set up and anything the user still has to do + themselves, like installing the packages you added. Then end your reply + with the question in `say`. Run `run` only if they say yes. +- `done`: tell the user `say`, then anything they still have to do + themselves, in one or two concrete steps. + +## Rules + +- Never open, print or `cat` the env file; `onboard check` verifies it. +- Never ask for an API key in chat, and never accept one pasted there. If one + appears, tell the user to rotate it on the API keys page. +- Never put the key in code; the SDK reads `ELEVENLABS_API_KEY`. +- Never run `git clean` or delete untracked files: the key is in one. +- Don't write or change any code until a reply's `next` is `write_code` or + `load_key`. +- Put anything you download only to read, like a package or an archive, in a + temp folder, never in the project. +- Don't run the app (or its start script) or make speech to try it until the + user says yes to the test request. API calls that what they asked for + needs, like creating an agent, are fine. Once the code is written, run + `onboard check` before you finish: it verifies the key and the code. + +## No npx + +If `npx` isn't available (Node.js isn't installed), install the CLI (or +update it if it's already installed: `brew upgrade elevenlabs`, or run the +installer again), then run `elevenlabs onboard init` instead. Everything after +that is the same. + +- macOS or Linux: `brew install elevenlabs/tap/elevenlabs`, or + `curl --proto '=https' --tlsv1.2 -LsSf https://github.com/elevenlabs/cli/releases/latest/download/elevenlabs-cli-installer.sh | sh` +- Windows: `powershell -ExecutionPolicy ByPass -c "irm https://github.com/elevenlabs/cli/releases/latest/download/elevenlabs-cli-installer.ps1 | iex"` + +If `elevenlabs` isn't found after the installer ran, use +`~/.cargo/bin/elevenlabs` in its place, here and in every `run` the CLI gives. diff --git a/skills/general/realtime/SKILL.md b/skills/general/realtime/SKILL.md new file mode 100644 index 0000000..e4bf935 --- /dev/null +++ b/skills/general/realtime/SKILL.md @@ -0,0 +1,359 @@ +--- +name: realtime +description: Use ElevenLabs Realtime models (`eleven_realtime_v1_mini`, `eleven_realtime_v1_max`), the low-latency models built for voice conversation. Use when the user wants to create an ElevenAgents agent on a Realtime model or switch an existing agent's Voice mode to Realtime, set `conversation_config.realtime_model`, choose between Realtime Mini and Max, or debug a Realtime agent that won't publish or transfer. Also use when migrating a voice app built on OpenAI's Realtime API and SDK (`@openai/agents-realtime`, `gpt-realtime`) to the OpenAI-compatible ElevenLabs Realtime API, including the `client_secrets` mint, the WebSocket transport, and audio after leaving WebRTC (closes with 4001/4003, no audio). Not for choosing an agent's LLM or TTS model in Custom mode, Scribe realtime speech-to-text, or migrating Chat Completions or Responses API text calls. +license: MIT +--- + +# ElevenLabs Realtime + +Realtime models are ElevenLabs models built for low-latency voice conversation. There are two ways to use them: + +- **An ElevenAgents agent** (an agent ID, the dashboard, or an `elevenlabs agents` project with `agent_configs/`): set the agent's Voice mode to Realtime with `conversation_config.realtime_model`, and ElevenLabs manages its LLM, TTS model, and ASR settings. See [Run an agent on a Realtime model](#run-an-agent-on-a-realtime-model). +- **A browser app built on OpenAI's Realtime SDK**: keep the SDK and point it at the OpenAI-compatible ElevenLabs Realtime API, `wss://api.elevenlabs.io/v1/convai/realtime`. See [Migrate an app from OpenAI Realtime](#migrate-an-app-from-openai-realtime). + +When the user is starting from scratch and it's unclear which they want, ask. An agent brings a managed system prompt, tools, knowledge base, widget, and telephony; the Realtime API is the quicker path for code that already speaks OpenAI's realtime events. For agent features beyond the voice mode (prompt, tools, workflows, widget), use the `agents` skill. + +## Models + +| Model | Model ID | Optimized for | +| ----------------------- | ------------------------- | ----------------------------------------------------------------------------------------- | +| Eleven Realtime v1 Mini | `eleven_realtime_v1_mini` | Lowest latency and cost. The default. | +| Eleven Realtime v1 Max | `eleven_realtime_v1_max` | Reasons before it responds, for complex instructions and multi-step tool use. Costs more. | + +Start with Mini. Choose Max when the work involves long or complex instructions or multi-step tool calls and Mini's responses aren't good enough, or when an app being migrated was on a larger OpenAI model for reasoning-heavy work. Agents add compliance and data residency constraints on the choice; see [Choose the model](#step-2-choose-the-model). + +## Run an agent on a Realtime model + +Every voice agent has a Voice mode. In **Custom** mode the agent's LLM, TTS model, and ASR settings are chosen individually. In **Realtime** mode the agent runs on a Realtime model and ElevenLabs manages those settings. The mode is one field: `conversation_config.realtime_model` set to a model ID means Realtime, `null` means Custom. + +The system prompt, first message, voice, languages, temperature, tools, knowledge base, and conversation flow settings (turn eagerness, timeouts, interruptions) work the same in both modes. + +### Before you start + +- These steps call the ElevenLabs API. The CLI and SDKs read `ELEVENLABS_API_KEY` from the environment; if it isn't set, use the `setup-api-key` skill. Never ask the user to paste the key into the conversation. +- Realtime models are rolling out gradually. If the API rejects `realtime_model` with `Realtime models are not available for this workspace.`, the workspace doesn't have access yet. Stop and tell the user; no configuration change fixes it. + +### Step 1: Check the agent can use Realtime + +For an existing agent, fetch its configuration first (`elevenlabs agents pull --agent `, or `agents.get` in the SDK). Keep the agent in Custom mode, and tell the user why, if any of these apply: + +| Blocker | Where to look | +| ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Custom LLM or Amazon Bedrock | `conversation_config.agent.prompt.llm` is `custom-llm` or a Bedrock model | +| Created through another ElevenLabs product, such as Speech Engine | Ask the user if unsure; only ElevenAgents agents can use a Realtime model | +| Needs a specific LLM or TTS model | Ask the user whether the current `agent.prompt.llm` or `tts.model_id` is a requirement | +| Workflow nodes override a [managed setting](#managed-settings) or `agent.prompt.max_tokens` | Overrides on the nodes under the agent's `workflow` | +| Transfers to or from agents in Custom mode | `transfer_to_agent` tools and `agent_transfer` workflow nodes on this agent, and on any agent that transfers into this one | +| Text-only agent | `conversation_config.conversation.text_only` is `true`; Realtime models are built for voice | + +A workflow override can be removed and a transfer partner can be switched to Realtime too, but only with the user's agreement: both change how other parts of their setup behave. + +### Step 2: Choose the model + +Start with `eleven_realtime_v1_mini`. Choose `eleven_realtime_v1_max` when the agent follows long or complex instructions or relies on multi-step tool calls and Mini isn't good enough, or when the agent must comply with Zero Retention Mode or HIPAA. Then check availability: + +| Environment | `eleven_realtime_v1_mini` | `eleven_realtime_v1_max` | +| ----------------------------- | ------------------------- | -------------------------- | +| US | Available | Available | +| EU data residency | Available | Available | +| India data residency | Available | Not available | +| Singapore data residency | Available | Not available | +| Zero Retention Mode and HIPAA | Not available | Available in the US and EU | + +Zero Retention Mode applies whether it's enforced on the workspace or enabled on the agent. Ask the user when the workspace's data residency or compliance settings aren't clear from the project. If neither model is available for the agent's environment, keep it in Custom mode. + +Max also needs `agent.prompt.max_tokens` of at least 1024, or `-1` for no limit. + +### Step 3: Set the model + +On an existing agent, change only `realtime_model`; leave the rest of the configuration alone. + +With the CLI, pull the agent, set the field in its file under `agent_configs/`, and push: + +```bash +elevenlabs agents pull --agent +``` + +```json +{ + "conversation_config": { + "realtime_model": "eleven_realtime_v1_mini" + } +} +``` + +```bash +elevenlabs agents push --agent +``` + +With the SDKs: + +```python +from elevenlabs import ElevenLabs + +elevenlabs = ElevenLabs() + +elevenlabs.conversational_ai.agents.update( + agent_id="", + conversation_config={ + "realtime_model": "eleven_realtime_v1_mini", + }, +) +``` + +```typescript +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; + +const elevenlabs = new ElevenLabsClient(); + +await elevenlabs.conversationalAi.agents.update("", { + conversationConfig: { + realtimeModel: "eleven_realtime_v1_mini", + }, +}); +``` + +For a new agent, put `realtime_model` (`realtimeModel` in TypeScript) in the `conversation_config` passed to `agents.create`, alongside the prompt, first message, and voice. Don't pick an LLM or TTS model for it; a Realtime agent ignores them. + +To turn Realtime off, set `realtime_model` to `null` (`None` in Python). The agent goes back to the LLM, TTS model, and ASR settings stored on it. + +### Managed settings + +While a Realtime model is set, the agent ignores the stored values of these fields in `conversation_config`. They come back into use when the agent switches to Custom, so don't clear them, and don't edit them to tune a Realtime agent: + +| Setting | Field | +| ------------------- | ----------------------------------------- | +| LLM | `agent.prompt.llm` | +| Reasoning effort | `agent.prompt.reasoning_effort` | +| Thinking budget | `agent.prompt.thinking_budget` | +| Parallel tool calls | `agent.prompt.enable_parallel_tool_calls` | +| Cascade timeout | `agent.prompt.cascade_timeout_seconds` | +| TTS model | `tts.model_id` | +| Expressive mode | `tts.expressive_mode` | +| ASR provider | `asr.provider` | + +`agent.prompt.max_tokens` keeps applying. Temperature does too, but the dashboard hides it in Realtime mode, so change it through the CLI or API. If the Realtime model errors or doesn't respond in time, the agent falls back to its backup LLMs (LLM cascading); the Realtime model sets how long it waits. + +### Step 4: Publish and verify + +The Step 1 rules are checked when the agent is published, not when a draft is saved: a draft that breaks one saves, then publishing fails with an error naming the setting to change. + +1. Fetch the agent and confirm `conversation_config.realtime_model` holds the chosen model. If the push, update, or the user's publish in the dashboard fails, match the error below. +2. Find every agent that transfers into this one and confirm it's in Realtime mode too. A Custom-mode agent's transfer into a Realtime agent is only caught when that agent is next published; until then the transfer is rejected mid-conversation and the caller stays with the current agent. +3. Have the user hold a test conversation. + +### Errors + +| Error | Cause and fix | +| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `Realtime models are not available for this workspace.` | The workspace doesn't have access yet. Nothing to fix in the configuration. | +| `Transfers between default and Realtime model modes are not supported.` | A transfer to or from a Custom-mode agent ("default" means Custom). Put both agents in the same mode. | +| An error naming the Realtime model | The model isn't available under the agent's compliance or data residency settings. See Step 2. | +| Publishing fails naming a setting | One of the Step 1 blockers or Max's token limit. Change that setting, or keep the agent in Custom. | + +### Pricing + +A Realtime agent is billed the standard ElevenAgents per-minute rate plus its input, cached input, and output tokens at the Realtime model's rates; Max costs more per token than Mini. Point the user to https://elevenlabs.io/pricing rather than quoting rates. + +## Migrate an app from OpenAI Realtime + +ElevenLabs Realtime speaks the same wire protocol as OpenAI's GA Realtime API: the same client and server events over WebSocket. An app built on OpenAI's Realtime SDK keeps that SDK; the migration changes where credentials are minted, which URL the transport connects to, and the model, voice, and session configuration. + +The one structural change: ElevenLabs Realtime is WebSocket-only. In a browser, `RealtimeSession` defaults to WebRTC, which wires the microphone and speaker for you. After switching to the WebSocket transport, the app has to capture and play audio itself. + +This covers apps where the browser opens the realtime connection. It doesn't cover apps where a server opens the connection and streams audio to its clients over its own protocol, such as a bespoke WebSocket relay or a telephony bridge. Connecting to ElevenLabs Realtime from a server should work, but this skill has no steps for it. Tell the user so, and suggest they open an issue or a pull request at https://github.com/elevenlabs/skills describing their setup, rather than guessing at the server-side migration. + +### Step 0: Inventory the app + +Find every place the app touches OpenAI Realtime before editing anything: + +```bash +grep -rnE "api\.openai\.com/v1/realtime|client_secrets|realtime/sessions|@openai/agents-realtime|@openai/agents/realtime|gpt-realtime|gpt-4o(-mini)?-realtime|OpenAIRealtimeWebRTC|OpenAIRealtimeWebSocket|RealtimeSession|OPENAI_API_KEY" --include='*.{ts,tsx,js,jsx,mjs,py}' . +``` + +Classify what you find, because each part migrates differently: + +- **Credential minting**: a server route that calls `/v1/realtime/client_secrets` (GA) or `/v1/realtime/sessions` (beta). Migrate in Step 1. +- **Browser session**: `RealtimeSession`, `OpenAIRealtimeWebRTC`, `OpenAIRealtimeWebSocket`, or a hand-rolled `WebSocket`/`RTCPeerConnection`. Migrate in Steps 2 to 4. +- **Server-side realtime connections**: a Node or Python process that opens the realtime socket itself, for example a relay that streams audio to clients over its own WebSocket protocol, a telephony bridge, `openai` Python's `client.realtime.connect()`, or a `RealtimeSession` running in Node with a full API key. Not covered; see the scope above. If the app also has a browser session, migrate only that part, and only with the user's agreement. +- **Features beyond a voice conversation**: function tools, handoffs, guardrails, MCP servers, image input, SIP. ElevenLabs documents the voice conversation flow only. Leave this code in place, list it for the user as unverified against ElevenLabs, and have them test it. + +`OPENAI_API_KEY` often serves other OpenAI calls too. Remove it only if nothing else uses it. + +### Step 1: Mint the client secret from ElevenLabs + +The browser never sees the ElevenLabs API key. A server route mints a short-lived client secret and returns only its `value`. + +| What | OpenAI | ElevenLabs | +| --------------- | -------------------------------------------------------- | ------------------------------------------------------------------ | +| URL | `POST https://api.openai.com/v1/realtime/client_secrets` | `POST https://api.elevenlabs.io/v1/convai/realtime/client_secrets` | +| Auth header | `Authorization: Bearer $OPENAI_API_KEY` | `xi-api-key: $ELEVENLABS_API_KEY` | +| `session.model` | `gpt-realtime`, … | `eleven_realtime_v1_mini` or `eleven_realtime_v1_max` | +| Response | `{ value, expires_at, session }` | Same shape | + +The request body keeps the GA shape, `{ session, expires_after }`. The session config sent here is the authoritative initial session for that credential. The `model` string selects the agent behavior directly; there is no `agent_id`. + +Mint with the ElevenLabs SDK (`npm install @elevenlabs/elevenlabs-js` or `pip install elevenlabs`), which sends the `xi-api-key` header for you: + +```typescript +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; + +const elevenlabs = new ElevenLabsClient(); + +const clientSecret = + await elevenlabs.conversationalAi.realtime.createClientSecret({ + session: { + type: "realtime", + model: "eleven_realtime_v1_mini", + instructions: + "You are a helpful, friendly voice assistant. Keep replies concise.", + outputModalities: ["audio"], + audio: { + input: { + format: { type: "audio/pcm", rate: 24000 }, + turnDetection: { type: "semantic_vad", eagerness: "medium" }, + }, + output: { + format: { type: "audio/pcm" }, + voice: "brian", + }, + }, + }, + expiresAfter: { anchor: "created_at", seconds: 600 }, + }); +``` + +```python +import os + +from elevenlabs import ( + CreateClientSecretRequest, + ExpiresAfter, + OpenAiAudioConfigInput, + OpenAiAudioInputConfigInput, + OpenAiAudioInputFormat, + OpenAiAudioOutputConfigInput, + OpenAiAudioOutputFormat, + OpenAiSessionConfigInput, + OpenAiTurnDetection, +) +from elevenlabs.client import ElevenLabs + +elevenlabs = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"]) + +client_secret = elevenlabs.conversational_ai.realtime.create_client_secret( + request=CreateClientSecretRequest( + session=OpenAiSessionConfigInput( + type="realtime", + model="eleven_realtime_v1_mini", + instructions="You are a helpful, friendly voice assistant. Keep replies concise.", + output_modalities=["audio"], + audio=OpenAiAudioConfigInput( + input=OpenAiAudioInputConfigInput( + format=OpenAiAudioInputFormat(type="audio/pcm", rate=24000), + turn_detection=OpenAiTurnDetection(type="semantic_vad", eagerness="medium"), + ), + output=OpenAiAudioOutputConfigInput( + format=OpenAiAudioOutputFormat(type="audio/pcm"), + voice="brian", + ), + ), + ), + expires_after=ExpiresAfter(anchor="created_at", seconds=600), + ), +) +``` + +Return only `clientSecret.value` (`client_secret.value` in Python) to the browser. + +Keep the route's existing framework, path, and response contract; only the upstream call changes. If the app minted with the beta `/v1/realtime/sessions` endpoint, its frontend reads `client_secret.value`. Either update the frontend to read the top-level `value`, or have the route return `{ client_secret: { value } }` so the frontend stays untouched. + +### Step 2: Convert the session config to the GA shape + +Send the GA session shape. Apps written against OpenAI's beta API use flat fields (`modalities`, `voice`, `input_audio_format`, `turn_detection`, …) that need converting, and ElevenLabs rejects the deprecated `modalities` field outright. This matters most for the mint request body and any hand-rolled `session.update`: the SDK's `config` option converts deprecated camelCase fields to the GA wire shape for you, though moving it to the GA form too removes the deprecation warnings. + +Read [Session config mapping](references/session-config.md) for the full beta-to-GA field table, voice handling, and the camelCase form the SDK's `config` option uses. + +### Step 3: Point the browser session at ElevenLabs + +Replace the transport with `OpenAIRealtimeWebSocket` aimed at the ElevenLabs URL, set the model on the session, and pass the minted `value` as the API key: + +```typescript +import { + OpenAIRealtimeWebSocket, + RealtimeAgent, + RealtimeSession, +} from "@openai/agents-realtime"; + +const { value: clientSecret } = await fetch("/api/token").then((res) => + res.json(), +); + +const agent = new RealtimeAgent({ + name: "Assistant", + instructions: + "You are a helpful, friendly voice assistant. Keep replies concise.", +}); + +const transport = new OpenAIRealtimeWebSocket({ + url: "wss://api.elevenlabs.io/v1/convai/realtime", +}); + +const session = new RealtimeSession(agent, { + transport, + model: "eleven_realtime_v1_mini", + config: { + outputModalities: ["audio"], + audio: { output: { voice: "brian" } }, + }, +}); + +await session.connect({ apiKey: clientSecret }); +``` + +Keep the app's existing `RealtimeAgent` name, instructions, and event handlers. Drop WebRTC-only options (`audioElement`, `mediaStream`, `changePeerConnection`) and any `transport: "webrtc"` setting; they have no WebSocket equivalent. + +### Step 4: Wire up audio (when the app was on WebRTC) + +If the app previously used WebRTC (an explicit `OpenAIRealtimeWebRTC`, `transport: "webrtc"`, or no transport at all in a browser), it now has to: + +1. Capture microphone audio as PCM16 at 24 kHz and stream it with `session.sendAudio(...)`. +2. Play PCM16 chunks from the session's `audio` event. +3. Stop playback on `audio_interrupted`. Without this, the assistant keeps talking over the user after a barge-in, because already-buffered audio keeps playing. +4. Release the microphone and close the session on teardown. + +Read [Browser audio over WebSocket](references/browser-audio.md) for a working implementation with `wavtools`, the helper OpenAI uses in its own WebSocket examples, and its pitfalls. + +Apps already on `OpenAIRealtimeWebSocket` have this wiring; check the sample rate is 24 kHz and keep it. + +### Step 5: Verify + +1. Find where the server loads `OPENAI_API_KEY` from (`.env`, `.env.local`, deployment secrets) and ask the user to store `ELEVENLABS_API_KEY` there. Don't ask for the key in the conversation or write it into a file yourself. Add an empty `ELEVENLABS_API_KEY=` to `.env.example` if the project has one. +2. Once the user confirms the key is in place, run the app, start a conversation, and speak. +3. Confirm the `session.created` transport event reports the `eleven_realtime_*` model and the chosen voice: + + ```typescript + session.on("transport_event", (event) => { + if (event.type === "session.created") console.log(event); + }); + ``` + +4. Interrupt the assistant mid-sentence and confirm playback stops. +5. Tell the user which features from Step 0 remain unverified (tools, handoffs, guardrails, …). + +### Troubleshooting + +| Symptom | Cause and fix | +| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| WebSocket closes with code `4003` | Realtime isn't enabled for this ElevenLabs workspace. Contact ElevenLabs; the code can't fix this. | +| WebSocket closes with code `4001` | Client secret rejected: expired, or not the minted `value` passed as `apiKey` to `session.connect()`. | +| Mint returns 401 | Still sending `Authorization: Bearer`; ElevenLabs uses the `xi-api-key` header. | +| Mint or `session.update` is rejected | Beta-shaped fields such as `modalities`. Convert with the [mapping](references/session-config.md). | +| Connects, but no audio either way | The app was on WebRTC and audio isn't wired yet. See Step 4. | +| Audio is noise, chipmunk-fast, or slowed down | Sample-rate or buffer mismatch. Capture and play at 24 kHz; see the pitfalls in the audio reference. | +| Assistant talks over the user after an interruption | Playback isn't stopped on `audio_interrupted`. | + +## References + +- [Session config mapping](references/session-config.md) +- [Browser audio over WebSocket](references/browser-audio.md) diff --git a/skills/general/realtime/references/browser-audio.md b/skills/general/realtime/references/browser-audio.md new file mode 100644 index 0000000..c1c9314 --- /dev/null +++ b/skills/general/realtime/references/browser-audio.md @@ -0,0 +1,101 @@ +# Browser audio over WebSocket + +The OpenAI Realtime SDK wires microphone and speaker automatically only for its WebRTC transport. ElevenLabs Realtime is WebSocket-only, so the app captures and plays audio itself. Both directions use PCM16, 24 kHz, mono. + +This reference uses [`wavtools`](https://www.npmjs.com/package/wavtools), the helper OpenAI uses in its own WebSocket examples. Any capture path that produces 24 kHz PCM16 `ArrayBuffer` chunks works, for example an `AudioWorklet` the app already has. + +```bash +npm install @openai/agents-realtime wavtools +``` + +## Implementation + +Framework-agnostic; in React, hold the returned handle in a `useRef` and call `stop()` on unmount. + +```typescript +import { + OpenAIRealtimeWebSocket, + RealtimeAgent, + RealtimeSession, +} from "@openai/agents-realtime"; +import { WavRecorder, WavStreamPlayer } from "wavtools"; + +const SAMPLE_RATE = 24000; + +export async function startVoiceSession() { + const tokenResponse = await fetch("/api/token"); + if (!tokenResponse.ok) + throw new Error(`Token request failed (${tokenResponse.status})`); + const { value: clientSecret } = await tokenResponse.json(); + + const recorder = new WavRecorder({ sampleRate: SAMPLE_RATE }); + const player = new WavStreamPlayer({ sampleRate: SAMPLE_RATE }); + await recorder.begin(); + await player.connect(); + + const session = new RealtimeSession( + new RealtimeAgent({ + name: "Assistant", + instructions: + "You are a helpful, friendly voice assistant. Keep replies concise.", + }), + { + transport: new OpenAIRealtimeWebSocket({ + url: "wss://api.elevenlabs.io/v1/convai/realtime", + }), + model: "eleven_realtime_v1_mini", + config: { + outputModalities: ["audio"], + audio: { output: { voice: "brian" } }, + }, + }, + ); + + session.on("audio", (event) => { + player.add16BitPCM(event.data, event.responseId); + }); + session.on("audio_interrupted", () => { + player.interrupt(); + }); + session.on("error", (event) => { + console.error(event.error); + }); + + try { + await session.connect({ apiKey: clientSecret }); + await recorder.record((data) => { + session.sendAudio(data.mono as unknown as ArrayBuffer); + }); + } catch (err) { + await stop(); + throw err; + } + + async function stop() { + await recorder.quit().catch(() => {}); + player.interrupt(); + session.close(); + } + + return { session, stop }; +} +``` + +Microphone audio streams continuously; server-side VAD decides when the user's turn ends, so there's no push-to-talk logic to port unless the app had one. + +## Pitfalls + +- **`data.mono` is mistyped.** `wavtools` types the `record()` callback's `mono` as `Int16Array`, but at runtime it's already the PCM16 `ArrayBuffer`. Pass it straight to `sendAudio`. Reading `.buffer` off it, or wrapping it in a new typed array, sends garbage. +- **Start from a user gesture.** `recorder.begin()` prompts for the microphone and `player.connect()` creates an `AudioContext`. Browsers block audio contexts created outside a click or keypress, so call `startVoiceSession()` from the handler that used to start the WebRTC session. +- **Stop playback on interruption.** `audio_interrupted` fires when the user barges in. `player.interrupt()` discards audio already queued; without it, the assistant keeps talking over the user. +- **Match the sample rate everywhere.** Recorder, player, and the mint body's `audio.input.format.rate` all use 24000. A mismatch plays back too fast, too slow, or as noise. +- **Tear down all three.** Quit the recorder (releases the microphone indicator), interrupt the player, and close the session. A leftover recorder keeps the microphone open after the conversation ends. + +## Replacing WebRTC-specific code + +| WebRTC-era code | WebSocket replacement | +| ------------------------------------------------------------------ | ------------------------------------------------------------------------- | +| `new OpenAIRealtimeWebRTC({ audioElement, mediaStream })` | `new OpenAIRealtimeWebSocket({ url })` plus the recorder and player above | +| `