diff --git a/.serena/.gitignore b/.serena/.gitignore new file mode 100644 index 00000000..2e510aff --- /dev/null +++ b/.serena/.gitignore @@ -0,0 +1,2 @@ +/cache +/project.local.yml diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 00000000..0f889c01 --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,169 @@ +# the name by which the project can be referenced within Serena/when chatting with the LLM. +project_name: "efficientAI" + +# list of language servers to start when using the LSP backend; choose from: +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart deno +# elixir elm erlang fortran fsharp +# gdscript gleam go groovy haskell +# haxe hlsl html java json +# julia kotlin latex lean4 lua +# luau markdown matlab msl nextflow +# nix ocaml pascal perl php +# php_phpactor php_phpantom powershell python python_basedpyright +# python_jedi python_pyrefly python_ty qml r +# rego ruby ruby_solargraph rust scala +# scss solidity svelte swift systemverilog +# terraform toml typescript typescript_vts vue +# wolfram yaml zig +# (This list may be outdated; generated with scripts/print_language_list.py; +# For the current list, see values of the LanguageServerId enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) +# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some language servers require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple language servers, the first language server that supports a given file will be used for that file. +# The first language server is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +language_servers: +- python + +# the encoding used by text files in the project +# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings +encoding: "utf-8" + +# optional shell command to run before the language backend (LSP or JetBrains) is initialised. +# the command runs in the project root directory and is only executed if the project is trusted +# (see trusted_project_path_patterns in the global configuration). +# serena waits for the command to exit: a non-zero exit code is logged as an error but does not +# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety +# backstop for non-terminating commands; on expiry the process is killed and activation continues. +# example: activation_command: "npx nx run-many -t build" +activation_command: + +# maximum time in seconds to wait for activation_command to complete before killing it (default 180s). +# must be a positive number. +activation_command_timeout: 180.0 + +# line ending convention to use when writing source files. +# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) +# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. +line_ending: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: + +# whether to use project's .gitignore files to ignore files +ignore_all_files_in_gitignore: true + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# The settings are considered only if the project is trusted (see global configuration to define trusted projects). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings +ls_specific_settings: {} + +# list of workspace folder paths (LSP backend only). +# These folders will be used to build up Serena's symbol index. +# Paths must be within the project root and should thus be relative to the project root. +# Furthermore, the paths should not be filtered by ignore settings. +# Default setting: The entire project root folder (".") is considered. +# In (large) monorepos, this can be used to index only subfolders of the project root, e.g. +# ls_workspace_folders: +# - "./subproject1" +# - "./subproject2" +ls_workspace_folders: +- "." + +# list of additional workspace folder paths for cross-package reference support. +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries, but these folders are not indexed by Serena, +# i.e. the respective symbols will not be found using Serena's symbol search tools. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +ls_additional_workspace_folders: [] + +# list of additional paths to ignore in this project. +# Same syntax as gitignore, so you can use * and **. +# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases. +# Example: +# ignored_paths: +# - "examples/**" +# - ".worktrees/**" +# - "**/bin/**" +# - "**/obj/**" +# Note: global ignored_paths from serena_config.yml are also applied additively. +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + +# list of tool names to exclude. +# This extends the existing exclusions (e.g. from the global configuration) +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +excluded_tools: [] + +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). +# This extends the existing inclusions (e.g. from the global configuration). +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html +fixed_tools: [] + +# list of mode names that are to be activated by default, overriding the setting in the global configuration. +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply +# for this project. +# This setting can, in turn, be overridden by CLI parameters (--mode). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +default_modes: + +# list of mode names to be activated additionally for this project, e.g. ["query-projects"] +# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. +# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes +added_modes: + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. +symbol_info_budget: + +# list of regex patterns which, when matched, mark a memory entry as read‑only. +# Extends the list from the global configuration, merging the two lists. +read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] diff --git a/README.md b/README.md index e14527d3..ef240f86 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,12 @@ There are two ways to run the application: | `redis` | Redis (Celery broker + usage counters) | | `api` | HTTP API + frontend | | `media` | Live voice WebSocket media server | - | `worker` | Celery: `celery` (evaluator cron dispatch), `audio-metrics` queues | + | `worker` | Celery: `celery` (evaluator cron runs), `audio-metrics` queues | | `beat` | Celery Beat scheduler + `platform` queue worker (alerts, FX, OSS prune) — **single replica** | | `worker-imports` | Celery: `imports`, `diarization`, `eval-control`, `evaluations` | - | `worker-usage` | Celery: `usage` queue (flush Redis counters + cost recompute) | + | `worker-usage` | Celery: `usage` queue (flush Redis counters, cost recompute, evaluator cron dispatch) | - **Usage costs:** token/cost rollups stay stale without `beat`, `worker-usage`, and default `worker` (evaluator crons; or `eai start-all`). + **Usage costs:** token/cost rollups stay stale without `beat`, `worker-usage`, and default `worker` (evaluator cron runs; or `eai start-all`). **Using a specific version:** ```bash @@ -416,7 +416,7 @@ eai usage recompute --config config.yml --sync | `USAGE_FLUSH_BEAT_SECONDS` | `120` | Celery Beat flush interval (~2 min lag vs Redis) | | `USAGE_FLUSH_LOCK_TTL_SECONDS` | `300` | Per-org flush lock TTL | | `USAGE_READ_CACHE_TTL_SECONDS` | `90` | Redis cache TTL for usage summary/breakdown/filters | -| `CRON_DISPATCH_INTERVAL_SECONDS` | `30` | Evaluator cron dispatcher tick (default worker) | +| `CRON_DISPATCH_INTERVAL_SECONDS` | `30` | Evaluator cron dispatch interval (Beat → worker-usage) | Usage UI reads Postgres only (summary/breakdown/filters); Redis counters flush on the Celery Beat schedule (~2 min eventual consistency). If Redis backlog grows, lower `USAGE_FLUSH_BEAT_SECONDS` or raise `USAGE_FLUSH_MAX_BATCHES_PER_RUN`. diff --git a/app/api/v1/api.py b/app/api/v1/api.py index 055bc147..e01f21dd 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -24,6 +24,7 @@ evaluator_suites, metrics, evaluator_results, + evaluator_result_metric_clusters, chat, playground, settings, @@ -76,6 +77,7 @@ api_router.include_router(evaluator_suites.router) api_router.include_router(metrics.router) api_router.include_router(evaluator_results.router) +api_router.include_router(evaluator_result_metric_clusters.router) api_router.include_router(chat.router) api_router.include_router(playground.router) api_router.include_router(settings.router) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 17094877..d7c84d06 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -33,11 +33,57 @@ GenerateTestSetupResponse, GeneratedScenarioDraftResponse, TestPromptSectionResponse, + TestAgentFirstMessageResponse, + TestAgentTemplateResponse, + TestAgentTemplateInput, +) +from app.services.testing.test_agent_template import ( + TestAgentFirstMessage, + TestAgentTemplate, + assemble_test_agent_prompt, + normalize_first_message, + normalize_sections, + template_from_generation, ) router = APIRouter(prefix="/agents", tags=["agents"]) +def _first_message_response(first_message: TestAgentFirstMessage) -> TestAgentFirstMessageResponse: + return TestAgentFirstMessageResponse( + production_mode=first_message.production_mode, + production_message=first_message.production_message, + caller_mode=first_message.caller_mode, + caller_message=first_message.caller_message, + ) + + +def _template_response(template: TestAgentTemplate) -> TestAgentTemplateResponse: + return TestAgentTemplateResponse( + sections=_test_prompt_section_responses(template.sections), + first_message=_first_message_response(template.first_message), + ) + + +def _template_input_to_storage(template_input: TestAgentTemplateInput) -> dict: + sections = normalize_sections([s.model_dump() for s in template_input.sections]) + first_message = normalize_first_message(template_input.first_message.model_dump()) + return template_from_generation(sections, first_message).to_dict() + + +def _apply_test_agent_template_fields( + *, + description: Optional[str], + template_input: Optional[TestAgentTemplateInput], +) -> tuple[Optional[str], Optional[dict]]: + """Return (description, test_agent_template_json) for persistence.""" + if template_input is None: + return description, None + template_dict = _template_input_to_storage(template_input) + assembled = assemble_test_agent_prompt(normalize_sections(template_dict.get("sections"))) + return assembled or description, template_dict + + def _validate_agent_phone_assignment( db: Session, *, @@ -130,7 +176,7 @@ class GenerateAgentDescriptionRequest(BaseModel): "well-formatted agent description in markdown.\n\n" "Guidelines:\n" "- Use clear markdown structure: headings, bullet points, numbered lists\n" - "- Include sections for: Purpose, Behavior, Expected Interactions, Personality Traits, and Constraints\n" + "- Include sections for: Role and Goal, Talking Style, Questions to Ask, Information to Relay, and Constraints\n" "- Be specific about the agent's role, tone of voice, and how it should handle conversations\n" "- Include example scenarios or edge cases where helpful\n" "- Return ONLY the description in markdown, no preamble or explanation about what you did" @@ -300,6 +346,8 @@ async def generate_test_prompt( return GenerateTestPromptResponse( sections=_test_prompt_section_responses(result.sections), test_agent_prompt=result.test_agent_prompt, + first_message=_first_message_response(result.first_message), + test_agent_template=_template_response(result.test_agent_template), provider=result.provider, model=result.model, ) @@ -433,6 +481,8 @@ async def generate_test_setup( return GenerateTestSetupResponse( sections=_test_prompt_section_responses(prompt_result.sections), test_agent_prompt=prompt_result.test_agent_prompt, + first_message=_first_message_response(prompt_result.first_message), + test_agent_template=_template_response(prompt_result.test_agent_template), scenarios=_scenario_draft_responses(scenario_result.scenarios), provider=prompt_result.provider, model=prompt_result.model, @@ -575,6 +625,11 @@ async def create_agent( # Generate unique 6-digit agent_id agent_id = generate_unique_agent_id(db) + + description, template_dict = _apply_test_agent_template_fields( + description=agent.description, + template_input=agent.test_agent_template, + ) db_agent = Agent( agent_id=agent_id, @@ -583,7 +638,8 @@ async def create_agent( name=agent.name, phone_number=agent.phone_number, language=agent.language, - description=agent.description, + description=description, + test_agent_template=template_dict, call_type=agent.call_type, call_medium=agent.call_medium, telephony_phone_number_id=agent.telephony_phone_number_id, @@ -813,6 +869,15 @@ async def update_agent( update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False) + if "test_agent_template" in update_data: + template_input = agent_update.test_agent_template + assembled_description, template_dict = _apply_test_agent_template_fields( + description=update_data.get("description", db_agent.description), + template_input=template_input, + ) + update_data["description"] = assembled_description + update_data["test_agent_template"] = template_dict + effective_call_medium = ( agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium ) diff --git a/app/api/v1/routes/evaluator_result_metric_clusters.py b/app/api/v1/routes/evaluator_result_metric_clusters.py new file mode 100644 index 00000000..f6a5d669 --- /dev/null +++ b/app/api/v1/routes/evaluator_result_metric_clusters.py @@ -0,0 +1,473 @@ +"""Metric-cluster routes for filtered evaluator-result scopes.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import APIRouter, Body, Depends, HTTPException, Query +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from app.database import get_db +from app.dependencies import get_organization_id, get_workspace_id, require_enterprise_feature +from app.models.schemas import ( + EvaluationMetricClustersRequest, + EvaluationMetricClustersState, + MetricClusterEligibleRow, + MetricClusterEligibleRowsResponse, + MetricFailurePoliciesResponse, + MetricFailurePoliciesSaveRequest, +) +from app.services.call_import_metric_clusters import ( + estimate_metric_clusters_llm_calls_for_source_rows, + metric_clusters_state_from_raw, +) +from app.services.call_import_user_insights import normalize_max_llm_calls +from app.services.evaluators.evaluator_result_metric_clusters import ( + apply_metric_clusters_cancel, + clustering_context_for_job, + failure_policies_response_for_job, + get_or_create_cluster_job, + has_clusterable_evaluator_results, + load_completed_evaluator_results, + metric_clusters_payload, + resolve_source_row_selection, +) +from app.services.metric_failure_policy import ( + merge_clustering_policies_from_raw, + merge_failure_policies_into_raw, + validate_failure_policies_for_metrics, +) + +router = APIRouter( + prefix="/evaluator-results/metric-clusters", + tags=["evaluator-results"], + dependencies=[Depends(require_enterprise_feature("evaluation_clustering"))], +) + + +def _parse_scope_uuids( + agent_id: Optional[str], + suite_id: Optional[str], + scenario_id: Optional[str], +) -> tuple[Optional[UUID], Optional[UUID], Optional[UUID]]: + agent_uuid: Optional[UUID] = None + suite_uuid: Optional[UUID] = None + scenario_uuid: Optional[UUID] = None + if agent_id: + try: + agent_uuid = UUID(agent_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid agent_id") from exc + if suite_id: + try: + suite_uuid = UUID(suite_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid suite_id") from exc + if scenario_id: + try: + scenario_uuid = UUID(scenario_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid scenario_id") from exc + return agent_uuid, suite_uuid, scenario_uuid + + +def _get_job( + db: Session, + organization_id: UUID, + workspace_id: UUID, + agent_id: Optional[str] = None, + suite_id: Optional[str] = None, + scenario_id: Optional[str] = None, +): + agent_uuid, suite_uuid, scenario_uuid = _parse_scope_uuids( + agent_id, suite_id, scenario_id + ) + return get_or_create_cluster_job( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_uuid, + suite_id=suite_uuid, + scenario_id=scenario_uuid, + ) + + +def _revoke_cluster_task(job) -> None: + from loguru import logger + + raw = job.metric_clusters + if not isinstance(raw, dict): + return + task_id = str(raw.get("celery_task_id") or "").strip() + if not task_id: + return + try: + from app.workers.celery_app import celery_app + + celery_app.control.revoke(task_id, terminate=True, signal="SIGTERM") + logger.info("Revoked evaluator metric-clusters task {} for job {}", task_id, job.id) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Failed to revoke evaluator metric-clusters task {} for job {}: {}", + task_id, + job.id, + exc, + ) + + +def _enqueue_cluster_job( + db: Session, + job, + *, + provider: Optional[str] = None, + model: Optional[str] = None, + credential_id: Optional[UUID] = None, + force: bool = False, + max_llm_calls: Optional[int] = None, + evaluation_row_ids: Optional[List[UUID]] = None, + selected_evaluation_row_ids: Optional[List[str]] = None, + failure_policies: Optional[Dict[str, Any]] = None, + row_limit: Optional[int] = None, +) -> None: + current = metric_clusters_payload(job) + if current is not None and current.status == "running" and not force: + return + + llm_budget = normalize_max_llm_calls(max_llm_calls) + total_calls = 1 + row_ids_for_task: Optional[List[str]] = None + + if selected_evaluation_row_ids is None: + try: + _filtered, selected_evaluation_row_ids = resolve_source_row_selection( + db, + job, + evaluation_row_ids=evaluation_row_ids, + row_limit=row_limit, + policies=failure_policies, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + metrics, _aggregates, policies_for_estimate, _source, _child_map, source_rows, _ = ( + clustering_context_for_job(db, job) + ) + if failure_policies: + policies_for_estimate = failure_policies + filtered_rows, _ = resolve_source_row_selection( + db, + job, + evaluation_row_ids=[UUID(rid) for rid in selected_evaluation_row_ids], + policies=policies_for_estimate, + ) + _, total_calls = estimate_metric_clusters_llm_calls_for_source_rows( + job.id, + metrics, + filtered_rows, + policies_for_estimate, + max_llm_calls=llm_budget, + ) + row_ids_for_task = list(selected_evaluation_row_ids) + + prior_raw = job.metric_clusters if isinstance(job.metric_clusters, dict) else {} + policy_blob: Dict[str, Any] = {} + if failure_policies: + from app.services.metric_failure_policy import failure_policies_to_db + + policy_blob = failure_policies_to_db(failure_policies, source="user") + + completed_count = len( + load_completed_evaluator_results( + db, + organization_id=job.organization_id, + workspace_id=job.workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + ) + + job.metric_clusters = { + "status": "running", + "groups": prior_raw.get("groups", []) if isinstance(prior_raw, dict) else [], + "discovered_problems": ( + prior_raw.get("discovered_problems", []) if isinstance(prior_raw, dict) else [] + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at_completed_rows": completed_count, + "progress": {"completed_llm_calls": 0, "total_llm_calls": total_calls}, + "provider": provider, + "model": model, + "max_llm_calls": llm_budget, + "llm_calls_used": 0, + "error_message": None, + "selected_evaluation_row_ids": selected_evaluation_row_ids or [], + **policy_blob, + } + flag_modified(job, "metric_clusters") + db.commit() + + from app.workers.tasks.generate_evaluator_result_metric_clusters import ( + generate_evaluator_result_metric_clusters_task, + ) + + async_result = generate_evaluator_result_metric_clusters_task.apply_async( + kwargs={ + "cluster_job_id": str(job.id), + "provider": provider, + "model": model, + "credential_id": str(credential_id) if credential_id else None, + "max_llm_calls": llm_budget, + "evaluation_row_ids": row_ids_for_task, + }, + queue="evaluations", + ) + if isinstance(job.metric_clusters, dict): + job.metric_clusters["celery_task_id"] = async_result.id + flag_modified(job, "metric_clusters") + db.commit() + + +@router.get( + "/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="getEvaluatorResultMetricClusterFailurePolicies", +) +def get_evaluator_result_metric_cluster_failure_policies( + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + db.commit() + return failure_policies_response_for_job(db, job) + + +@router.put( + "/failure-policies", + response_model=MetricFailurePoliciesResponse, + operation_id="saveEvaluatorResultMetricClusterFailurePolicies", +) +def save_evaluator_result_metric_cluster_failure_policies( + body: MetricFailurePoliciesSaveRequest, + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricFailurePoliciesResponse: + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + metrics, aggregates, _existing, _source, child_names_by_parent, _rows, _ = ( + clustering_context_for_job(db, job) + ) + try: + validate_failure_policies_for_metrics(body.policies, metrics) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + prior = job.metric_clusters if isinstance(job.metric_clusters, dict) else {} + job.metric_clusters = merge_failure_policies_into_raw( + prior, + body.policies, + source="user", + ) + flag_modified(job, "metric_clusters") + db.commit() + db.refresh(job) + return failure_policies_response_for_job(db, job) + + +@router.get( + "/eligible-rows", + response_model=MetricClusterEligibleRowsResponse, + operation_id="listEvaluatorResultMetricClusterEligibleRows", +) +def list_evaluator_result_metric_cluster_eligible_rows( + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + limit: Optional[int] = Query(default=None, ge=1), + count_only: bool = Query(default=False), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> MetricClusterEligibleRowsResponse: + from app.services.call_import_metric_clusters import list_eligible_cluster_source_rows + + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + metrics, _aggregates, policies, _source, _child_map, source_rows, _ = ( + clustering_context_for_job(db, job) + ) + all_eligible = list_eligible_cluster_source_rows(source_rows, metrics, policies) + total = len(all_eligible) + if count_only: + return MetricClusterEligibleRowsResponse(items=[], total=total) + raw_items = all_eligible if limit is None else all_eligible[:limit] + items = [MetricClusterEligibleRow.model_validate(item) for item in raw_items] + return MetricClusterEligibleRowsResponse(items=items, total=total) + + +@router.get( + "", + response_model=Optional[EvaluationMetricClustersState], + operation_id="getEvaluatorResultMetricClusters", +) +def get_evaluator_result_metric_clusters( + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> Optional[EvaluationMetricClustersState]: + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + db.commit() + state = metric_clusters_payload(job) + if state is None: + return None + _, _aggregates, _policies, _source, _child_map, _rows, completed_count = ( + clustering_context_for_job(db, job) + ) + if state.generated_at_completed_rows and completed_count > state.generated_at_completed_rows: + state.is_stale = True + return state + + +@router.post( + "", + response_model=EvaluationMetricClustersState, + operation_id="generateEvaluatorResultMetricClusters", +) +def generate_evaluator_result_metric_clusters( + body: EvaluationMetricClustersRequest = Body(default_factory=EvaluationMetricClustersRequest), + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + db.commit() + + if not body.regenerate and not body.force: + cached = metric_clusters_payload(job) + if cached is not None and cached.status in {"running", "completed"}: + return cached + + completed = load_completed_evaluator_results( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + if not completed: + raise HTTPException( + status_code=400, + detail=( + "No completed evaluator results in this scope yet. " + "Wait for at least one run to finish scoring before generating clusters." + ), + ) + + if body.evaluation_row_ids and body.row_limit is not None: + raise HTTPException( + status_code=400, + detail="Specify either evaluation_row_ids or row_limit, not both.", + ) + + metrics, aggregates, _inferred, _source, child_names_by_parent, _rows, _ = ( + clustering_context_for_job(db, job) + ) + merged_policies = merge_clustering_policies_from_raw( + body.failure_policies, + job.metric_clusters, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + try: + validate_failure_policies_for_metrics( + body.failure_policies or merged_policies, metrics + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not has_clusterable_evaluator_results(db, job, merged_policies): + raise HTTPException( + status_code=400, + detail=( + "No calls match any failure policy. Select failure values on " + "metrics that have matching rows, or leave metrics with no " + "failures unchecked — they are skipped automatically." + ), + ) + + try: + _filtered, selected_row_ids = resolve_source_row_selection( + db, + job, + evaluation_row_ids=body.evaluation_row_ids, + row_limit=body.row_limit, + policies=merged_policies, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not selected_row_ids: + raise HTTPException( + status_code=400, + detail=( + "No eligible rows to cluster. Select completed calls that match " + "at least one configured failure policy." + ), + ) + + _enqueue_cluster_job( + db, + job, + provider=body.provider, + model=body.model, + credential_id=body.credential_id, + force=body.force or body.regenerate, + max_llm_calls=body.max_llm_calls, + evaluation_row_ids=body.evaluation_row_ids, + selected_evaluation_row_ids=selected_row_ids, + failure_policies=merged_policies, + row_limit=body.row_limit, + ) + + db.refresh(job) + return metric_clusters_payload(job) or EvaluationMetricClustersState(status="running") + + +@router.post( + "/cancel", + response_model=EvaluationMetricClustersState, + operation_id="cancelEvaluatorResultMetricClusters", +) +def cancel_evaluator_result_metric_clusters( + agent_id: Optional[str] = Query(None), + suite_id: Optional[str] = Query(None), + scenario_id: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +) -> EvaluationMetricClustersState: + job = _get_job(db, organization_id, workspace_id, agent_id, suite_id, scenario_id) + db.commit() + if apply_metric_clusters_cancel(job): + _revoke_cluster_task(job) + flag_modified(job, "metric_clusters") + db.commit() + db.refresh(job) + return metric_clusters_payload(job) or EvaluationMetricClustersState(status="idle") diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index 962207c6..da191d65 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -262,6 +262,8 @@ def _resolve_speaker_segments(result: EvaluatorResult) -> Optional[List[Dict[str def get_evaluator_results_overview( agent_id: Optional[str] = Query(None, description="When set, return suites for this agent"), suite_id: Optional[str] = Query(None, description="When set, return scenarios for this suite"), + since: Optional[datetime] = Query(None), + until: Optional[datetime] = Query(None), organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), db: Session = Depends(get_db), @@ -286,6 +288,8 @@ def get_evaluator_results_overview( workspace_id=workspace_id, agent_id=agent_uuid, suite_id=suite_uuid, + since=since, + until=until, ) @@ -865,10 +869,37 @@ def re_evaluate_result( ) if not result.evaluator_id: - raise HTTPException( - status_code=400, - detail="Cannot re-evaluate: this result is not linked to an evaluator." - ) + if result.agent_id and result.persona_id and result.scenario_id: + from app.api.v1.routes.evaluators import generate_unique_evaluator_id + + evaluator = db.query(Evaluator).filter( + Evaluator.agent_id == result.agent_id, + Evaluator.persona_id == result.persona_id, + Evaluator.scenario_id == result.scenario_id, + Evaluator.organization_id == organization_id, + Evaluator.workspace_id == workspace_id, + ).first() + if not evaluator: + new_evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=new_evaluator_id, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=result.agent_id, + persona_id=result.persona_id, + scenario_id=result.scenario_id, + tags=["auto-created", "test-voice-agent"], + ) + db.add(evaluator) + db.commit() + db.refresh(evaluator) + result.evaluator_id = evaluator.id + db.commit() + else: + raise HTTPException( + status_code=400, + detail="Cannot re-evaluate: this result is not linked to an evaluator." + ) evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() if not evaluator: diff --git a/app/api/v1/routes/evaluator_suites.py b/app/api/v1/routes/evaluator_suites.py index d5c56d33..57a30243 100644 --- a/app/api/v1/routes/evaluator_suites.py +++ b/app/api/v1/routes/evaluator_suites.py @@ -9,10 +9,12 @@ from app.database import get_db from app.dependencies import get_organization_id, get_workspace_id -from app.models.database import Agent, EvaluatorSuite, Scenario +from app.models.database import Agent, EvaluatorSuite, Persona, Scenario from app.models.schemas import ( + EvaluatorSuiteAddPersonasRequest, EvaluatorSuiteAddScenariosRequest, EvaluatorSuiteCreate, + EvaluatorSuiteReplacePersonasRequest, EvaluatorSuiteResponse, EvaluatorSuiteUpdate, RunEvaluatorSuiteRequest, @@ -30,12 +32,15 @@ from app.services.evaluators.evaluator_run_service import queue_evaluator_runs from app.services.evaluators.evaluator_suite_service import ( activate_evaluator_suite, + add_personas_to_suite, add_scenarios_to_suite, create_evaluator_suite, delete_evaluator_suite, get_suite_or_404, pick_round_robin_combination, + remove_persona_from_suite, remove_scenario_from_suite, + replace_personas_in_suite, update_evaluator_suite, _build_suite_response, ) @@ -130,6 +135,42 @@ def remove_scenario( return remove_scenario_from_suite(db, suite, scenario_id) +@router.post("/{suite_id}/personas", response_model=EvaluatorSuiteResponse) +def add_personas( + suite_id: UUID, + data: EvaluatorSuiteAddPersonasRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + suite = get_suite_or_404(db, suite_id, organization_id, workspace_id) + return add_personas_to_suite(db, suite, data.persona_ids) + + +@router.put("/{suite_id}/personas", response_model=EvaluatorSuiteResponse) +def replace_personas( + suite_id: UUID, + data: EvaluatorSuiteReplacePersonasRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + suite = get_suite_or_404(db, suite_id, organization_id, workspace_id) + return replace_personas_in_suite(db, suite, data.persona_ids) + + +@router.delete("/{suite_id}/personas/{persona_id}", response_model=EvaluatorSuiteResponse) +def remove_persona( + suite_id: UUID, + persona_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + suite = get_suite_or_404(db, suite_id, organization_id, workspace_id) + return remove_persona_from_suite(db, suite, persona_id) + + @router.post("/{suite_id}/activate", response_model=EvaluatorSuiteResponse) def activate_suite( suite_id: UUID, @@ -252,9 +293,16 @@ def choose_next_combination( selected, idx, next_index = pick_round_robin_combination(db, suite) scenario = db.query(Scenario).filter(Scenario.id == selected.scenario_id).first() scenario_name = scenario.name if scenario else "Unknown Scenario" + persona = ( + db.query(Persona).filter(Persona.id == selected.persona_id).first() + if selected.persona_id + else None + ) return ChooseNextCombinationResponse( evaluator_id=selected.id, + persona_id=selected.persona_id, + persona_name=persona.name if persona else None, scenario_id=selected.scenario_id, scenario_name=scenario_name, combination_index=idx, @@ -287,6 +335,11 @@ def run_next_combination( selected, idx, next_index = pick_round_robin_combination(db, suite) scenario = db.query(Scenario).filter(Scenario.id == selected.scenario_id).first() scenario_name = scenario.name if scenario else "Unknown Scenario" + persona = ( + db.query(Persona).filter(Persona.id == selected.persona_id).first() + if selected.persona_id + else None + ) strategy = _resolve_run_strategy(agent) task_id = None @@ -316,6 +369,8 @@ def run_next_combination( return RunNextCombinationResponse( evaluator_id=selected.id, + persona_id=selected.persona_id, + persona_name=persona.name if persona else None, scenario_id=selected.scenario_id, scenario_name=scenario_name, combination_index=idx, diff --git a/app/api/v1/routes/observability.py b/app/api/v1/routes/observability.py index 4c26a549..5d93b643 100644 --- a/app/api/v1/routes/observability.py +++ b/app/api/v1/routes/observability.py @@ -209,6 +209,15 @@ def _upsert_call_recording( ) if call_recording: + if call_recording.source == CallRecordingSource.PLAYGROUND: + agent_obj = None + if call_recording.agent_id: + agent_obj = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + response = _serialize_call_recording( + call_recording, include_data=True, agent=agent_obj + ) + response["action"] = "skipped_playground" + return response call_recording.call_data = call_data_payload call_recording.status = CallRecordingStatus.UPDATED call_recording.source = source diff --git a/app/api/v1/routes/personas.py b/app/api/v1/routes/personas.py index de07c715..9f4382cb 100644 --- a/app/api/v1/routes/personas.py +++ b/app/api/v1/routes/personas.py @@ -3,8 +3,8 @@ CRUD for TTS provider-tied voice personas, voice-options catalog, and custom voice management (ungated). """ -from fastapi import APIRouter, Depends, HTTPException, status, Body, Query -from fastapi.responses import JSONResponse +from fastapi import APIRouter, Depends, HTTPException, status, Body, Query, UploadFile, File, Form +from fastapi.responses import Response from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any @@ -12,19 +12,22 @@ from pydantic import BaseModel from loguru import logger -from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key +from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key, require_enterprise_entitlement from app.models.database import ( Persona, Evaluator, EvaluatorResult, TestAgentConversation, CustomTTSVoice, - PromptOptimizationRun, CallRecording, Agent, + PromptOptimizationRun, CallRecording, Agent, AmbientNoiseAsset, ) -from app.models.enums import LanguageEnum, AccentEnum, GenderEnum, BackgroundNoiseEnum +from app.models.enums import LanguageEnum, AccentEnum, GenderEnum, BackgroundNoiseEnum, BackgroundNoiseSourceEnum from app.models.schemas import ( PersonaCreate, PersonaUpdate, PersonaResponse, PersonaCloneRequest, AgentPromptSourcesResponse, GeneratePersonaPromptRequest, GeneratePersonaPromptResponse, + AmbientNoiseAssetResponse, + AmbientNoiseAssetUpdateRequest, ) from app.models.enums import ModelProvider from app.services.ai.model_config_service import model_config_service from app.services.ai.llm_resolver import get_llm_provider_and_model as _get_llm_provider_and_model +from app.services.personas.configured_tts_providers import get_configured_tts_provider_keys from app.services.personas.persona_tts_config import ( normalize_persona_tts_config, validate_persona_tts_config, @@ -33,6 +36,22 @@ generate_persona_prompt_from_agent, resolve_agent_prompt_sources, ) +from app.services.personas.persona_ambient_noise import ( + ALLOWED_AMBIENT_EXTENSIONS, + MAX_AMBIENT_UPLOAD_BYTES, + persona_ambient_s3_key, + validate_persona_ambient_fields, +) +from app.services.personas.ambient_library import ( + ambient_library_s3_key, + new_ambient_asset_id, + sanitize_ambient_name, + validate_ambient_upload_bytes, +) +from app.services.audio.ambient_catalog import get_ambient_asset_provider, list_ambient_presets, normalize_ambient_preset +from app.services.audio.ambient_mixer import decode_audio_bytes_to_pcm_int16 +from app.services.storage.s3_service import s3_service, StorageError +from fastapi.responses import JSONResponse router = APIRouter(prefix="/personas", tags=["personas"]) @@ -41,6 +60,85 @@ def _normalized_persona_tts_config(provider: Optional[str], tts_config: Optional return normalize_persona_tts_config(provider, tts_config) +def _ambient_fields_from_create( + persona: PersonaCreate, + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, +) -> Dict[str, Any]: + return validate_persona_ambient_fields( + source=persona.background_noise_source.value, + preset=persona.background_noise_preset, + volume=persona.background_noise_volume, + s3_key=None, + asset_id=persona.background_noise_asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + require_custom_file=( + persona.background_noise_source == BackgroundNoiseSourceEnum.CUSTOM + and not persona.background_noise_asset_id + ), + ) + + +def _apply_ambient_update( + db_persona: Persona, + update_data: Dict[str, Any], + organization_id: UUID, + workspace_id: UUID, + db: Session, +) -> Dict[str, Any]: + if not any( + key in update_data + for key in ( + "background_noise_source", + "background_noise_preset", + "background_noise_volume", + "background_noise_asset_id", + ) + ): + return update_data + + source = update_data.get( + "background_noise_source", + db_persona.background_noise_source or BackgroundNoiseSourceEnum.NONE.value, + ) + if hasattr(source, "value"): + source = source.value + preset = update_data.get("background_noise_preset", db_persona.background_noise_preset) + volume = update_data.get("background_noise_volume", db_persona.background_noise_volume) + asset_id = update_data.get("background_noise_asset_id", db_persona.background_noise_asset_id) + s3_key = db_persona.background_noise_s3_key + + validated = validate_persona_ambient_fields( + source=source, + preset=preset, + volume=volume, + s3_key=s3_key, + asset_id=asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + require_custom_file=( + str(source).lower() == BackgroundNoiseSourceEnum.CUSTOM.value + and not asset_id + and not s3_key + ), + ) + update_data["background_noise_source"] = validated["background_noise_source"] + update_data["background_noise_preset"] = validated["background_noise_preset"] + update_data["background_noise_volume"] = validated["background_noise_volume"] + update_data["background_noise_asset_id"] = validated["background_noise_asset_id"] + if validated["background_noise_source"] != BackgroundNoiseSourceEnum.CUSTOM.value: + update_data["background_noise_s3_key"] = None + update_data["background_noise_asset_id"] = None + else: + update_data["background_noise_s3_key"] = validated["background_noise_s3_key"] + return update_data + + def _get_agent_for_workspace( db: Session, *, @@ -188,10 +286,12 @@ def _is_valid_persona_row(persona: Persona) -> bool: accent_value = str(getattr(persona, "accent", "neutral") or "neutral").lower() gender_value = str(getattr(persona, "gender", "neutral") or "neutral").lower() noise_value = str(getattr(persona, "background_noise", "none") or "none").lower() + source_value = str(getattr(persona, "background_noise_source", "none") or "none").lower() LanguageEnum(language_value) AccentEnum(accent_value) GenderEnum(gender_value) BackgroundNoiseEnum(noise_value) + BackgroundNoiseSourceEnum(source_value) return True except Exception: return False @@ -206,6 +306,12 @@ async def create_persona( ): """Create a new persona stamped with the active workspace.""" try: + ambient_fields = _ambient_fields_from_create( + persona, + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + ) db_persona = Persona( organization_id=organization_id, workspace_id=workspace_id, @@ -222,6 +328,10 @@ async def create_persona( response_delay_ms=persona.response_delay_ms, max_turns=persona.max_turns, allow_interruptions=persona.allow_interruptions, + background_noise_source=ambient_fields["background_noise_source"], + background_noise_preset=ambient_fields["background_noise_preset"], + background_noise_volume=ambient_fields["background_noise_volume"], + background_noise_asset_id=ambient_fields["background_noise_asset_id"], ) db.add(db_persona) db.commit() @@ -363,7 +473,10 @@ async def get_voice_options( "description": cv.description, }) - all_keys: set = set(TTS_VOICES.keys()) | set(model_voices_by_provider.keys()) | set(custom_by_provider.keys()) + configured_keys = get_configured_tts_provider_keys(organization_id, db) + all_keys: set = ( + set(TTS_VOICES.keys()) | set(model_voices_by_provider.keys()) | set(custom_by_provider.keys()) + ) & configured_keys if provider: all_keys = {k for k in all_keys if k == provider.lower()} @@ -584,6 +697,253 @@ async def delete_custom_voice( return {"message": "Custom voice deleted"} +# ============================================ +# AMBIENT NOISE (static paths before /{persona_id}) +# ============================================ + +def _guess_audio_media_type(filename: Optional[str], fallback: str = "audio/wav") -> str: + if not filename: + return fallback + ext = filename.rsplit(".", 1)[-1].lower() + return { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "ogg": "audio/ogg", + "m4a": "audio/mp4", + "flac": "audio/flac", + }.get(ext, fallback) + + +@router.get("/ambient-presets", operation_id="listAmbientPresets") +async def list_platform_ambient_presets( + api_key: str = Depends(get_api_key), +): + """List platform ambient presets available from installed asset packs.""" + return {"presets": list_ambient_presets()} + + +@router.get( + "/ambient-presets/{preset_id}/preview", + operation_id="previewAmbientPreset", +) +async def preview_ambient_preset( + preset_id: str, + api_key: str = Depends(get_api_key), +): + """Stream a platform preset for in-browser preview.""" + normalized = normalize_ambient_preset(preset_id) or preset_id + provider = get_ambient_asset_provider() + try: + file_bytes = provider.load_wav(normalized) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=f"Preset '{preset_id}' is not available") from exc + return Response(content=file_bytes, media_type=_guess_audio_media_type(f"{normalized}.wav")) + + +@router.get( + "/ambient-library", + response_model=List[AmbientNoiseAssetResponse], + operation_id="listAmbientLibrary", +) +async def list_ambient_library( + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + rows = ( + db.query(AmbientNoiseAsset) + .filter( + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ) + .order_by(AmbientNoiseAsset.created_at.desc()) + .all() + ) + return rows + + +@router.post( + "/ambient-library", + response_model=AmbientNoiseAssetResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="uploadAmbientLibraryAsset", +) +async def upload_ambient_library_asset( + file: UploadFile = File(...), + name: Optional[str] = Form(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Upload a reusable ambient bed to the workspace library.""" + if not s3_service.is_enabled(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=s3_service.get_status_message(), + ) + + filename = file.filename or "" + file_bytes = await file.read() + try: + extension = validate_ambient_upload_bytes(file_bytes, filename=filename) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + asset_id = new_ambient_asset_id() + display_name = sanitize_ambient_name(name, filename.rsplit(".", 1)[0] if "." in filename else filename) + s3_key = ambient_library_s3_key(organization_id, asset_id, extension) + content_type = file.content_type or _guess_audio_media_type(filename) + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + row = AmbientNoiseAsset( + id=asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + name=display_name, + s3_key=s3_key, + original_filename=filename or None, + ) + db.add(row) + db.commit() + db.refresh(row) + return row + + +@router.patch( + "/ambient-library/{asset_id}", + response_model=AmbientNoiseAssetResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="updateAmbientLibraryAsset", +) +async def update_ambient_library_asset( + asset_id: UUID, + data: AmbientNoiseAssetUpdateRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Rename a library ambient bed.""" + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + + row.name = sanitize_ambient_name(data.name, row.name) + db.commit() + db.refresh(row) + return row + + +@router.delete( + "/ambient-library/{asset_id}", + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="deleteAmbientLibraryAsset", +) +async def delete_ambient_library_asset( + asset_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + + in_use = db.query(Persona).filter(Persona.background_noise_asset_id == asset_id).count() + if in_use: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Ambient bed is used by {in_use} persona(s). Reassign them before deleting.", + ) + + if s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.s3_key) + except Exception as exc: + logger.warning("Failed to delete ambient library object {}: {}", row.s3_key, exc) + + db.delete(row) + db.commit() + return JSONResponse(status_code=204, content=None) + + +@router.get( + "/ambient-library/{asset_id}/preview", + operation_id="previewAmbientLibraryAsset", +) +async def preview_ambient_library_asset( + asset_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Stream a library ambient bed for in-browser preview.""" + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + if not s3_service.is_enabled(): + raise HTTPException(status_code=503, detail=s3_service.get_status_message()) + try: + file_bytes = s3_service.download_file_by_key(row.s3_key) + except StorageError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + return Response( + content=file_bytes, + media_type=_guess_audio_media_type(row.original_filename or row.s3_key), + ) + + +class AmbientLibraryPreviewUrlResponse(BaseModel): + url: str + expires_in: int + + +@router.get( + "/ambient-library/{asset_id}/preview-url", + response_model=AmbientLibraryPreviewUrlResponse, + operation_id="getAmbientLibraryPreviewUrl", +) +async def get_ambient_library_preview_url( + asset_id: UUID, + expiration: int = Query(default=3600, ge=60, le=86400), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Return a presigned URL for streaming ambient library preview in the browser.""" + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + if not s3_service.is_enabled(): + raise HTTPException(status_code=503, detail=s3_service.get_status_message()) + try: + url = s3_service.generate_presigned_url_by_key(row.s3_key, expiration=expiration) + except StorageError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + return AmbientLibraryPreviewUrlResponse(url=url, expires_in=expiration) + + # ============================================ # PERSONA BY ID (parameterized routes last) # ============================================ @@ -655,6 +1015,12 @@ async def update_persona( except ValueError as exc: raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) update_data["tts_config"] = _normalized_persona_tts_config(provider, update_data["tts_config"]) + try: + update_data = _apply_ambient_update( + db_persona, update_data, organization_id, workspace_id, db + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) for field, value in update_data.items(): setattr(db_persona, field, value) @@ -843,6 +1209,11 @@ async def clone_persona( response_delay_ms=source_persona.response_delay_ms, max_turns=source_persona.max_turns, allow_interruptions=source_persona.allow_interruptions, + background_noise_source=source_persona.background_noise_source, + background_noise_preset=source_persona.background_noise_preset, + background_noise_volume=source_persona.background_noise_volume, + background_noise_s3_key=source_persona.background_noise_s3_key, + background_noise_asset_id=source_persona.background_noise_asset_id, ) db.add(new_persona) db.commit() @@ -880,6 +1251,113 @@ async def clone_persona( ) +@router.post( + "/{persona_id}/ambient-audio", + response_model=PersonaResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="uploadPersonaAmbientAudio", +) +async def upload_persona_ambient_audio( + persona_id: UUID, + file: UploadFile = File(...), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Upload or replace custom ambient audio for a persona (enterprise).""" + if not s3_service.is_enabled(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=s3_service.get_status_message(), + ) + + db_persona = db.query(Persona).filter( + Persona.id == persona_id, + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ).first() + if not db_persona: + raise HTTPException(status_code=404, detail=f"Persona {persona_id} not found") + + filename = file.filename or "" + extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if extension not in ALLOWED_AMBIENT_EXTENSIONS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported ambient audio format. Allowed: {', '.join(sorted(ALLOWED_AMBIENT_EXTENSIONS))}", + ) + + file_bytes = await file.read() + if not file_bytes: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Uploaded file is empty") + if len(file_bytes) > MAX_AMBIENT_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Ambient audio must be at most {MAX_AMBIENT_UPLOAD_BYTES // (1024 * 1024)} MB", + ) + + try: + decode_audio_bytes_to_pcm_int16(file_bytes, 16000) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not decode ambient audio file: {exc}", + ) from exc + + s3_key = persona_ambient_s3_key(organization_id, persona_id, extension) + content_type = file.content_type or f"audio/{extension}" + try: + if db_persona.background_noise_s3_key and db_persona.background_noise_s3_key != s3_key: + try: + s3_service.delete_file_by_key(db_persona.background_noise_s3_key) + except Exception: + logger.warning("Could not delete previous ambient audio key {}", db_persona.background_noise_s3_key) + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + db_persona.background_noise_s3_key = s3_key + db_persona.background_noise_source = BackgroundNoiseSourceEnum.CUSTOM.value + db.commit() + db.refresh(db_persona) + return db_persona + + +@router.delete( + "/{persona_id}/ambient-audio", + response_model=PersonaResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="deletePersonaAmbientAudio", +) +async def delete_persona_ambient_audio( + persona_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Delete custom ambient audio for a persona (enterprise).""" + db_persona = db.query(Persona).filter( + Persona.id == persona_id, + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ).first() + if not db_persona: + raise HTTPException(status_code=404, detail=f"Persona {persona_id} not found") + + if db_persona.background_noise_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(db_persona.background_noise_s3_key) + except Exception as exc: + logger.warning("Failed to delete ambient audio {}: {}", db_persona.background_noise_s3_key, exc) + + db_persona.background_noise_s3_key = None + if db_persona.background_noise_source == BackgroundNoiseSourceEnum.CUSTOM.value: + db_persona.background_noise_source = BackgroundNoiseSourceEnum.NONE.value + db.commit() + db.refresh(db_persona) + return db_persona + + # ============================================ # SEED DATA (Helper for demo) # ============================================ diff --git a/app/api/v1/routes/vobiz_telephony.py b/app/api/v1/routes/vobiz_telephony.py index 052d03b9..a25d7167 100644 --- a/app/api/v1/routes/vobiz_telephony.py +++ b/app/api/v1/routes/vobiz_telephony.py @@ -679,6 +679,16 @@ async def vobiz_media_websocket(websocket: WebSocket): persona_id=persona_id, scenario_id=scenario_id, ) + from app.services.telephony.vobiz_agent_context import resolve_vobiz_telephony_run_params + + run_params = resolve_vobiz_telephony_run_params( + db, + context=context, + call_direction=session.direction, + persona_id=persona_id, + scenario_id=scenario_id, + evaluator_id=session.evaluator_id, + ) serializer = VobizFrameSerializer( stream_id=stream_id, call_id=call_id, @@ -696,7 +706,7 @@ async def vobiz_media_websocket(websocket: WebSocket): hangup_secs = resolve_agent_silence_hangup_secs(context.agent) await run_voice_bundle_fastapi( websocket, - context.system_instruction, + run_params.system_instruction, str(context.organization_id), str(context.workspace_id) if context.workspace_id else None, agent_id, @@ -711,6 +721,10 @@ async def vobiz_media_websocket(websocket: WebSocket): telephony_mode=True, call_short_id=call_short_id, silence_hangup_secs=hangup_secs, + call_direction=session.direction, + caller_speaks_first=run_params.caller_speaks_first, + caller_opening_text=run_params.caller_opening_text, + persona_speaks_via_tts=run_params.persona_speaks_via_tts, ) else: if not context.google_api_key: @@ -722,7 +736,7 @@ async def vobiz_media_websocket(websocket: WebSocket): await run_bot( websocket, context.google_api_key, - context.system_instruction, + run_params.system_instruction, str(context.organization_id), agent_id, persona_id, @@ -732,6 +746,9 @@ async def vobiz_media_websocket(websocket: WebSocket): telephony_mode=True, call_short_id=call_short_id, silence_hangup_secs=hangup_secs, + persona=context.persona, + call_direction=session.direction, + persona_speaks_via_tts=run_params.persona_speaks_via_tts, ) except ValueError as e: logger.error("Vobiz media websocket setup failed: {}", e) diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index e6d553b5..7fee1a9e 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -13,6 +13,12 @@ from app.dependencies import get_organization_id, get_api_key from app.models.database import AIProvider, ModelProvider, Integration, IntegrationPlatform, Workspace from app.core.encryption import decrypt_api_key + + +def _parse_bool_query(value: Optional[str], *, default: bool = False) -> bool: + if value is None: + return default + return value.strip().lower() in ("1", "true", "yes", "on") from app.services.voice_agent.bot_fast_api import run_bot from app.services.ai.llm_service import _resolve_azure_endpoint_from_provider from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi @@ -103,6 +109,7 @@ async def websocket_endpoint( agent_id = websocket.query_params.get("agent_id") persona_id = websocket.query_params.get("persona_id") scenario_id = websocket.query_params.get("scenario_id") + run_evaluation = _parse_bool_query(websocket.query_params.get("run_evaluation"), default=False) # Fetch agent and voice bundle once for routing and instructions agent = None @@ -292,21 +299,23 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: return system_instruction = None - instruction_parts = [] + caller_speaks_first = True + caller_opening_text = None + persona_speaks_via_tts = False - # Build system instruction as a bundle: Agent + Persona + Scenario + # Build system instruction from agent + persona + scenario from app.models.database import Agent, Persona, Scenario + persona = None + scenario = None + # 1. Add Agent description (base instruction) and get voice bundle for model model_name = None if agent: - if agent.description: - instruction_parts.append(agent.description) if voice_bundle and voice_bundle.bundle_type == "s2s" and voice_bundle.s2s_model: model_name = voice_bundle.s2s_model - # 2. Add Persona information (characteristics) - persona = None + # 2. Load Persona if persona_id: try: persona_uuid = UUID(persona_id) @@ -317,23 +326,10 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: if workspace_id is not None: persona_query = persona_query.filter(Persona.workspace_id == workspace_id) persona = persona_query.first() - if persona: - persona_parts = [] - persona_parts.append(f"\n\nPersona: {persona.name}") - if persona.gender: - gender_val = persona.gender.value if hasattr(persona.gender, "value") else persona.gender - persona_parts.append(f"Gender: {gender_val}") - if getattr(persona, "tts_provider", None): - persona_parts.append(f"Voice provider: {persona.tts_provider}") - if getattr(persona, "tts_voice_name", None): - persona_parts.append(f"Voice: {persona.tts_voice_name}") - - if persona_parts: - instruction_parts.append("\n".join(persona_parts)) except ValueError: pass - # 3. Add Scenario information (context and goals) + # 3. Load Scenario if scenario_id: try: scenario_uuid = UUID(scenario_id) @@ -344,24 +340,33 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: if workspace_id is not None: scenario_query = scenario_query.filter(Scenario.workspace_id == workspace_id) scenario = scenario_query.first() - if scenario: - scenario_parts = [] - scenario_parts.append(f"\n\nScenario: {scenario.name}") - if scenario.description: - scenario_parts.append(f"Description: {scenario.description}") - if scenario.required_info: - required_info_str = ", ".join([f"{k}: {v}" for k, v in scenario.required_info.items()]) if isinstance(scenario.required_info, dict) else str(scenario.required_info) - if required_info_str: - scenario_parts.append(f"Required information to collect: {required_info_str}") - - if scenario_parts: - instruction_parts.append("\n".join(scenario_parts)) except ValueError: pass - - # Combine all parts into final system instruction - if instruction_parts: - system_instruction = "\n".join(instruction_parts) + + if agent and persona and scenario: + from app.services.testing.test_agent_simulation_prompt import ( + build_live_test_agent_system_prompt, + ) + from app.services.testing.test_agent_template import ( + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, + ) + + system_instruction = build_live_test_agent_system_prompt(agent, persona, scenario) + persona_speaks_via_tts = True + first_message_config = resolve_first_message_from_agent(agent) + scenario_first_message = None + if scenario.required_info and isinstance(scenario.required_info, dict): + scenario_first_message = scenario.required_info.get("first_message") + caller_opening_text = resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona.name or "Test Caller", + scenario_first_message=scenario_first_message, + ) + caller_speaks_first = should_caller_speak_first(first_message_config) + elif agent and agent.description: + system_instruction = agent.description.strip() # Generate result_id BEFORE running bot (for meaningful S3 path) # Evaluator is only created if persona_id and scenario_id are provided @@ -498,6 +503,13 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: ).lower() == "azure" else None ) + from app.services.voice_agent.llm_voice_providers import resolve_voice_llm_base_url + + llm_base_url = ( + resolve_voice_llm_base_url(db, organization_id, voice_bundle, llm_provider) + if llm_provider + else None + ) # If in bridge mode, we need to bridge test agent to Retell call # For now, we'll run the voice bundle normally and note that bridging @@ -529,7 +541,11 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: tts_api_key=tts_api_key, llm_api_key=llm_api_key, llm_endpoint_url=llm_endpoint_url, + llm_base_url=llm_base_url, silence_hangup_secs=agent_silence_hangup_secs, + caller_speaks_first=caller_speaks_first, + caller_opening_text=caller_opening_text, + persona_speaks_via_tts=persona_speaks_via_tts, ) else: call_metadata = await run_bot( @@ -544,6 +560,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: result_id=result_id, model_name=model_name, # Pass model name from voice bundle silence_hangup_secs=agent_silence_hangup_secs, + persona=persona, ) except Exception as bot_error: logger.error(f"Error in run_bot: {bot_error}", exc_info=True) @@ -619,10 +636,13 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: else: result_name = "Test Call" - logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}") + logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}, run_evaluation={run_evaluation}") - # Create evaluator result with QUEUED status - # persona_id and scenario_id can be None for test calls without persona/scenario + initial_status = ( + EvaluatorResultStatus.QUEUED.value + if run_evaluation + else EvaluatorResultStatus.CALL_ENDED.value + ) evaluator_result = EvaluatorResult( result_id=result_id, organization_id=organization_id, @@ -633,7 +653,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: scenario_id=UUID(scenario_id) if scenario_id else None, # Optional name=result_name, duration_seconds=call_metadata.get("duration"), - status=EvaluatorResultStatus.QUEUED.value, # Use .value to get the string + status=initial_status, audio_s3_key=call_metadata.get("s3_key"), transcription=call_metadata.get("transcription"), speaker_segments=call_metadata.get("speaker_segments"), @@ -642,44 +662,42 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: db.commit() db.refresh(evaluator_result) - logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}") + logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}, status={initial_status}") - # Trigger Celery task - try: - logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") - - # Check if Celery app is properly configured - from app.workers.celery_app import celery_app - logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") - logger.info(f"Celery result backend: {celery_app.conf.result_backend}") - - # Verify task is registered - if 'process_evaluator_result' not in celery_app.tasks: - logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") - logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") - else: - logger.info("✅ Task 'process_evaluator_result' is registered") - - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") - - # Try to get task info to verify it was queued + if run_evaluation: try: - task_info = task.info - logger.info(f"Task info: {task_info}") - except Exception as info_error: - logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") - - evaluator_result.celery_task_id = task.id - db.commit() - logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") - except Exception as task_error: - logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) - # Still log that we created the result even if task trigger failed - logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") - logger.warning(f"Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") - - logger.info(f"✅ Created evaluator result {result_id} and triggered processing task") + logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") + + from app.workers.celery_app import celery_app + logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") + logger.info(f"Celery result backend: {celery_app.conf.result_backend}") + + if 'process_evaluator_result' not in celery_app.tasks: + logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") + logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") + else: + logger.info("✅ Task 'process_evaluator_result' is registered") + + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") + + try: + task_info = task.info + logger.info(f"Task info: {task_info}") + except Exception as info_error: + logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") + + evaluator_result.celery_task_id = task.id + db.commit() + logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") + except Exception as task_error: + logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) + logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") + logger.warning("Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") + else: + logger.info(f"Skipping post-call evaluation for result {result_id} (run_evaluation=false)") + + logger.info(f"✅ Created evaluator result {result_id}" + (" and triggered processing task" if run_evaluation else "")) except Exception as e: logger.error(f"❌ Error creating evaluator result: {e}", exc_info=True) @@ -814,6 +832,7 @@ def _extract_bearer(value: Optional[str]) -> Optional[str]: agent_id = request.query_params.get("agent_id") persona_id = request.query_params.get("persona_id") scenario_id = request.query_params.get("scenario_id") + run_evaluation = _parse_bool_query(request.query_params.get("run_evaluation"), default=False) # Determine which AI Provider to use based on agent configuration ai_provider = None @@ -959,6 +978,7 @@ def check_provider(provider_enum): agent_id=agent_id, persona_id=persona_id, scenario_id=scenario_id, + run_evaluation=run_evaluation, fallback_host=request.headers.get("host", f"localhost:{settings.PORT}"), fallback_scheme=( request.headers.get("x-forwarded-proto") diff --git a/app/cli.py b/app/cli.py index 7fbefbff..6fb5fb8c 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1600,7 +1600,17 @@ def usage_recompute( help="Merge generated pricing into app/config/models.json", ) @click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") -def usage_sync_litellm(local: bool, write_models: bool, stdout: bool): +@click.option( + "--import-missing-fireworks", + is_flag=True, + help="Add current Fireworks serverless chat models missing from models.json", +) +def usage_sync_litellm( + local: bool, + write_models: bool, + stdout: bool, + import_missing_fireworks: bool, +): """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" import subprocess import sys as sys_module @@ -1613,6 +1623,8 @@ def usage_sync_litellm(local: bool, write_models: bool, stdout: bool): cmd.append("--write-models") if stdout: cmd.append("--stdout") + if import_missing_fireworks: + cmd.append("--import-missing-fireworks") subprocess.run(cmd, check=True) diff --git a/app/config.py b/app/config.py index 8242409c..c66c3aa6 100644 --- a/app/config.py +++ b/app/config.py @@ -212,7 +212,9 @@ class Settings(BaseSettings): ] # Live telephony pipeline recording merge (dual-track → natural mono) - TELEPHONY_BOT_PLAYBACK_DELAY_MS: int = 400 + # Residual one-way carrier latency trim applied to the bot track at merge time. + # Keep at 0 unless measured on real calls; the recorders are already aligned. + TELEPHONY_BOT_PLAYBACK_DELAY_MS: int = 0 TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT: float = 0.35 # Judge Alignment (AlignEval-style hybrid integration). diff --git a/app/config/models.json b/app/config/models.json index 38f24cca..d0d4e0e8 100644 --- a/app/config/models.json +++ b/app/config/models.json @@ -1075,6 +1075,267 @@ "output_per_1m": 0.9 } }, + "deepseek-v4-flash-0731": { + "provider": "fireworks", + "model_type": "llm", + "description": "deepseek v4 flash 0731 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.14, + "output_per_1m": 0.28, + "cache_read_per_1m": 0.028 + } + }, + "glm-4p7": { + "provider": "fireworks", + "model_type": "llm", + "description": "glm 4p7 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.6, + "output_per_1m": 2.2, + "cache_read_per_1m": 0.3 + } + }, + "glm-5p1-fast": { + "provider": "fireworks", + "model_type": "llm", + "description": "glm 5p1 fast via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.8, + "output_per_1m": 8.8, + "cache_read_per_1m": 0.52 + } + }, + "glm-5p2": { + "provider": "fireworks", + "model_type": "llm", + "description": "glm 5p2 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.4, + "output_per_1m": 4.4, + "cache_read_per_1m": 0.14 + } + }, + "glm-5p2-fast": { + "provider": "fireworks", + "model_type": "llm", + "description": "glm 5p2 fast via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.1, + "output_per_1m": 6.6, + "cache_read_per_1m": 0.21 + } + }, + "glm-5p2-fast-us": { + "provider": "fireworks", + "model_type": "llm", + "description": "glm 5p2 fast us via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.1, + "output_per_1m": 6.6, + "cache_read_per_1m": 0.21 + } + }, + "kimi-k2p6-fast": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k2p6 fast via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.3 + } + }, + "kimi-k2p7-code": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k2p7 code via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.95, + "output_per_1m": 4.0, + "cache_read_per_1m": 0.19 + } + }, + "kimi-k2p7-code-fast": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k2p7 code fast via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 1.9, + "output_per_1m": 8.0, + "cache_read_per_1m": 0.38 + } + }, + "kimi-k3": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k3 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.0, + "output_per_1m": 15.0, + "cache_read_per_1m": 0.3 + } + }, + "kimi-k3-fast": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k3 fast via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 4.5, + "output_per_1m": 22.5, + "cache_read_per_1m": 0.45 + } + }, + "kimi-k3-us": { + "provider": "fireworks", + "model_type": "llm", + "description": "kimi k3 us via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 3.3, + "output_per_1m": 16.5, + "cache_read_per_1m": 0.33 + } + }, + "llama4-maverick-instruct-basic": { + "provider": "fireworks", + "model_type": "llm", + "description": "llama4 maverick instruct basic via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.22, + "output_per_1m": 0.88 + } + }, + "llama4-scout-instruct-basic": { + "provider": "fireworks", + "model_type": "llm", + "description": "llama4 scout instruct basic via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.15, + "output_per_1m": 0.6 + } + }, + "minimax-m2p1": { + "provider": "fireworks", + "model_type": "llm", + "description": "minimax m2p1 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 1.2, + "cache_read_per_1m": 0.03 + } + }, + "minimax-m3": { + "provider": "fireworks", + "model_type": "llm", + "description": "minimax m3 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.3, + "output_per_1m": 1.2, + "cache_read_per_1m": 0.06 + } + }, + "muse-glimmer-30b": { + "provider": "fireworks", + "model_type": "llm", + "description": "muse glimmer 30b via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.35, + "output_per_1m": 1.5, + "cache_read_per_1m": 0.04 + } + }, + "nemotron-3-ultra-nvfp4": { + "provider": "fireworks", + "model_type": "llm", + "description": "nemotron 3 ultra nvfp4 via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.6, + "output_per_1m": 2.4, + "cache_read_per_1m": 0.12 + } + }, + "nemotron-lightning-3p5-30b-a3b": { + "provider": "fireworks", + "model_type": "llm", + "description": "nemotron lightning 3p5 30b a3b via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.05, + "output_per_1m": 0.2, + "cache_read_per_1m": 0.01 + } + }, + "qwen3-coder-480b-a35b-instruct": { + "provider": "fireworks", + "model_type": "llm", + "description": "qwen3 coder 480b a35b instruct via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.45, + "output_per_1m": 1.8 + } + }, + "qwen3p7-plus": { + "provider": "fireworks", + "model_type": "llm", + "description": "qwen3p7 plus via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 0.4, + "output_per_1m": 1.6, + "cache_read_per_1m": 0.08 + } + }, + "qwen3p8-max": { + "provider": "fireworks", + "model_type": "llm", + "description": "qwen3p8 max via Fireworks", + "pricing": { + "source": "litellm_import", + "usage_kind": "llm", + "input_per_1m": 2.0, + "output_per_1m": 6.0, + "cache_read_per_1m": 0.25 + } + }, "google-speech-v2": { "provider": "google", "model_type": "stt", diff --git a/app/core/license.py b/app/core/license.py index 56fd1888..b15f4bdf 100644 --- a/app/core/license.py +++ b/app/core/license.py @@ -43,6 +43,11 @@ "description": "Bulk-import production call recordings via CSV and run batch evaluations on them.", "category": "evaluation", }, + "evaluation_clustering": { + "title": "Evaluation Failure Clustering", + "description": "Cluster failed evaluation runs from LLM rationales to surface recurring failure patterns.", + "category": "evaluation", + }, # --- Authentication features (gate pluggable auth providers) --- "oidc_sso": { "title": "Enterprise SSO (OIDC)", diff --git a/app/migrations/078_add_test_agent_template_to_agents.py b/app/migrations/078_add_test_agent_template_to_agents.py new file mode 100644 index 00000000..54855a02 --- /dev/null +++ b/app/migrations/078_add_test_agent_template_to_agents.py @@ -0,0 +1,28 @@ +""" +Migration: Add test_agent_template JSON column to agents. + +Stores structured test agent prompt sections and first-message configuration. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add test_agent_template to agents" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE agents + ADD COLUMN IF NOT EXISTS test_agent_template JSON + """)) + db.commit() + print("Added test_agent_template column to agents") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE agents + DROP COLUMN IF EXISTS test_agent_template + """)) + db.commit() + print("Dropped test_agent_template column from agents") diff --git a/app/migrations/079_persona_ambient_noise.py b/app/migrations/079_persona_ambient_noise.py new file mode 100644 index 00000000..38c91840 --- /dev/null +++ b/app/migrations/079_persona_ambient_noise.py @@ -0,0 +1,32 @@ +""" +Migration: Add persona ambient noise configuration columns. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add persona background noise source, preset, volume, and s3 key" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + ADD COLUMN IF NOT EXISTS background_noise_source VARCHAR(20) NOT NULL DEFAULT 'none', + ADD COLUMN IF NOT EXISTS background_noise_preset VARCHAR(50), + ADD COLUMN IF NOT EXISTS background_noise_volume DOUBLE PRECISION DEFAULT 0.22, + ADD COLUMN IF NOT EXISTS background_noise_s3_key VARCHAR + """)) + db.commit() + print("Added persona ambient noise columns") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + DROP COLUMN IF EXISTS background_noise_s3_key, + DROP COLUMN IF EXISTS background_noise_volume, + DROP COLUMN IF EXISTS background_noise_preset, + DROP COLUMN IF EXISTS background_noise_source + """)) + db.commit() + print("Dropped persona ambient noise columns") diff --git a/app/migrations/080_ambient_noise_library.py b/app/migrations/080_ambient_noise_library.py new file mode 100644 index 00000000..05bc069f --- /dev/null +++ b/app/migrations/080_ambient_noise_library.py @@ -0,0 +1,52 @@ +""" +Migration: Add reusable ambient noise library and persona asset reference. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add ambient_noise_assets table and persona background_noise_asset_id" + + +def upgrade(db: Session): + db.execute(text(""" + CREATE TABLE IF NOT EXISTS ambient_noise_assets ( + id UUID PRIMARY KEY, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + name VARCHAR(255) NOT NULL, + s3_key VARCHAR NOT NULL, + original_filename VARCHAR(255), + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + ) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_ambient_noise_assets_organization_id + ON ambient_noise_assets (organization_id) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_ambient_noise_assets_workspace_id + ON ambient_noise_assets (workspace_id) + """)) + db.execute(text(""" + ALTER TABLE personas + ADD COLUMN IF NOT EXISTS background_noise_asset_id UUID + REFERENCES ambient_noise_assets(id) ON DELETE SET NULL + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_personas_background_noise_asset_id + ON personas (background_noise_asset_id) + """)) + db.commit() + print("Added ambient_noise_assets table and persona background_noise_asset_id") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + DROP COLUMN IF EXISTS background_noise_asset_id + """)) + db.execute(text("DROP TABLE IF EXISTS ambient_noise_assets")) + db.commit() + print("Dropped ambient_noise_assets table and persona background_noise_asset_id") diff --git a/app/migrations/081_evaluator_suite_persona_scenario_unique.py b/app/migrations/081_evaluator_suite_persona_scenario_unique.py new file mode 100644 index 00000000..c0de10b2 --- /dev/null +++ b/app/migrations/081_evaluator_suite_persona_scenario_unique.py @@ -0,0 +1,28 @@ +""" +Migration: unique persona+scenario pairs per evaluator suite child row. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add unique index on evaluators (suite_id, persona_id, scenario_id)" + + +def upgrade(db: Session): + db.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS uq_evaluator_suite_persona_scenario + ON evaluators (suite_id, persona_id, scenario_id) + WHERE suite_id IS NOT NULL + AND persona_id IS NOT NULL + AND scenario_id IS NOT NULL + """)) + db.commit() + print("Added uq_evaluator_suite_persona_scenario index on evaluators") + + +def downgrade(db: Session): + db.execute(text(""" + DROP INDEX IF EXISTS uq_evaluator_suite_persona_scenario + """)) + db.commit() + print("Dropped uq_evaluator_suite_persona_scenario index on evaluators") diff --git a/app/migrations/082_evaluator_result_cluster_jobs.py b/app/migrations/082_evaluator_result_cluster_jobs.py new file mode 100644 index 00000000..cc6ae1b4 --- /dev/null +++ b/app/migrations/082_evaluator_result_cluster_jobs.py @@ -0,0 +1,71 @@ +""" +Migration: Cached failure-cluster jobs scoped to evaluator-result filters. + +Stores ``metric_clusters`` JSONB per workspace filter scope (agent / suite / scenario). +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = ( + "Add evaluator_result_cluster_jobs table for scoped evaluation-results " + "failure clustering." +) + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if _table_exists(db, "evaluator_result_cluster_jobs"): + print("evaluator_result_cluster_jobs already exists, skipping...") + return + + db.execute( + text( + """ + CREATE TABLE evaluator_result_cluster_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + scope_key VARCHAR(512) NOT NULL, + agent_id UUID NULL REFERENCES agents(id) ON DELETE SET NULL, + suite_id UUID NULL REFERENCES evaluator_suites(id) ON DELETE SET NULL, + scenario_id UUID NULL REFERENCES scenarios(id) ON DELETE SET NULL, + metric_clusters JSONB NULL, + celery_task_id VARCHAR NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, scope_key) + ) + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_evaluator_result_cluster_jobs_org + ON evaluator_result_cluster_jobs (organization_id) + """ + ) + ) + print("Created evaluator_result_cluster_jobs table") + + +def downgrade(db: Session): + if not _table_exists(db, "evaluator_result_cluster_jobs"): + print("evaluator_result_cluster_jobs missing, skipping...") + return + db.execute(text("DROP TABLE evaluator_result_cluster_jobs")) + print("Dropped evaluator_result_cluster_jobs table") diff --git a/app/models/database.py b/app/models/database.py index 92350139..d7159028 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -522,6 +522,7 @@ class Agent(Base): voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) prompt_variables = Column(JSON, nullable=True) + test_agent_template = Column(JSON, nullable=True) silence_hangup_secs = Column(Integer, nullable=False, server_default="15") created_at = Column(DateTime, server_default=func.now()) @@ -557,12 +558,41 @@ class Persona(Base): response_delay_ms = Column(Integer, nullable=True) max_turns = Column(Integer, nullable=True) allow_interruptions = Column(Boolean, nullable=True) + background_noise_source = Column(String(20), nullable=False, default="none") + background_noise_preset = Column(String(50), nullable=True) + background_noise_volume = Column(Float, nullable=True, default=0.22) + background_noise_s3_key = Column(String, nullable=True) + background_noise_asset_id = Column( + UUID(as_uuid=True), + ForeignKey("ambient_noise_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) created_at = Column(DateTime, server_default=func.now()) updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) created_by = Column(String) +class AmbientNoiseAsset(Base): + """Reusable ambient noise bed uploaded for test-agent personas.""" + __tablename__ = "ambient_noise_assets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + s3_key = Column(String, nullable=False) + original_filename = Column(String(255), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + class Scenario(Base): """Scenario - The conversation scenario/test case""" __tablename__ = "scenarios" @@ -818,7 +848,7 @@ class TestAgentConversation(Base): class EvaluatorSuite(Base): - """Evaluator suite — one agent + one persona + N scenario combinations.""" + """Evaluator suite — one agent + one or more personas × N scenario combinations.""" __tablename__ = "evaluator_suites" @@ -1106,6 +1136,31 @@ class EvaluatorResult(Base): created_by = Column(String, nullable=True) +class EvaluatorResultClusterJob(Base): + """Cached failure-cluster state for a filtered evaluator-results scope.""" + + __tablename__ = "evaluator_result_cluster_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + scope_key = Column(String(512), nullable=False) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + suite_id = Column(UUID(as_uuid=True), ForeignKey("evaluator_suites.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + metric_clusters = Column(JSON, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Enums moved to enums.py diff --git a/app/models/enums.py b/app/models/enums.py index 140ffe48..020acecf 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -81,12 +81,21 @@ class AccentEnum(str, enum.Enum): GERMAN = "german" NEUTRAL = "neutral" +class BackgroundNoiseSourceEnum(str, enum.Enum): + """Where persona ambient audio is loaded from.""" + NONE = "none" + PLATFORM = "platform" + CUSTOM = "custom" + + class BackgroundNoiseEnum(str, enum.Enum): - """Background noise options""" + """Platform ambient preset names.""" NONE = "none" OFFICE = "office" STREET = "street" + TRAFFIC = "traffic" CAFE = "cafe" + CONCERT = "concert" HOME = "home" CALL_CENTER = "call_center" diff --git a/app/models/schemas.py b/app/models/schemas.py index 40e988e9..c0e3e702 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -8,6 +8,7 @@ from app.models.enums import ( EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + BackgroundNoiseSourceEnum, IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, @@ -173,6 +174,37 @@ class ErrorResponse(BaseModel): # Enums moved to enums.py + + + +# Test agent template schemas (defined before AgentCreate/AgentUpdate) +class TestPromptSectionResponse(BaseModel): + """One canonical section of a generated test agent prompt.""" + key: str + title: str + content: str + + +class TestAgentFirstMessageResponse(BaseModel): + """Who speaks first on production vs test caller sides.""" + production_mode: str + production_message: Optional[str] = None + caller_mode: str + caller_message: Optional[str] = None + + +class TestAgentTemplateResponse(BaseModel): + """Structured test agent template stored on agents.""" + sections: List[TestPromptSectionResponse] + first_message: TestAgentFirstMessageResponse + + +class TestAgentTemplateInput(BaseModel): + """Structured test agent template for create/update.""" + sections: List[TestPromptSectionResponse] + first_message: TestAgentFirstMessageResponse + + # Agent Schemas class AgentCreate(BaseModel): """Schema for creating a new agent""" @@ -188,6 +220,7 @@ class AgentCreate(BaseModel): voice_ai_integration_id: Optional[UUID] = None voice_ai_agent_id: Optional[str] = None provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateInput] = None silence_hangup_secs: int = Field( default=15, ge=0, @@ -245,6 +278,7 @@ class AgentUpdate(BaseModel): voice_ai_integration_id: Optional[UUID] = None voice_ai_agent_id: Optional[str] = None provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateInput] = None prompt_variables: Optional[Dict[str, str]] = None silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) @@ -303,15 +337,6 @@ class AgentPhoneAssignmentCheckResponse(BaseModel): -class TestPromptSectionResponse(BaseModel): - """One canonical section of a generated test agent prompt.""" - key: str - title: str - content: str - - - - class GeneratedScenarioDraftResponse(BaseModel): """LLM-generated scenario draft before persistence.""" name: str @@ -339,6 +364,8 @@ class GenerateTestPromptRequest(BaseModel): class GenerateTestPromptResponse(BaseModel): sections: List[TestPromptSectionResponse] test_agent_prompt: str + first_message: TestAgentFirstMessageResponse + test_agent_template: TestAgentTemplateResponse provider: str model: str @@ -388,6 +415,8 @@ class GenerateTestSetupRequest(BaseModel): class GenerateTestSetupResponse(BaseModel): sections: List[TestPromptSectionResponse] test_agent_prompt: str + first_message: TestAgentFirstMessageResponse + test_agent_template: TestAgentTemplateResponse scenarios: List[GeneratedScenarioDraftResponse] provider: str model: str @@ -410,6 +439,7 @@ class AgentResponse(BaseModel): voice_ai_integration_id: Optional[UUID] voice_ai_agent_id: Optional[str] provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateResponse] = None prompt_variables: Optional[Dict[str, str]] = None silence_hangup_secs: int = Field( default=15, @@ -496,6 +526,10 @@ class PersonaCreate(BaseModel): response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) max_turns: Optional[int] = Field(None, ge=1, le=100) allow_interruptions: Optional[bool] = None + background_noise_source: BackgroundNoiseSourceEnum = BackgroundNoiseSourceEnum.NONE + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = Field(None, ge=0.05, le=0.60) + background_noise_asset_id: Optional[UUID] = None @model_validator(mode="after") def validate_tts_config(self): @@ -504,6 +538,28 @@ def validate_tts_config(self): validate_persona_tts_config(self.tts_provider, self.tts_config) return self + @model_validator(mode="after") + def validate_ambient_fields(self): + from app.services.personas.persona_ambient_noise import validate_persona_ambient_fields + + validated = validate_persona_ambient_fields( + source=self.background_noise_source.value, + preset=self.background_noise_preset, + volume=self.background_noise_volume, + s3_key=None, + asset_id=self.background_noise_asset_id, + ) + if validated["background_noise_source"] == BackgroundNoiseSourceEnum.CUSTOM.value: + if not self.background_noise_asset_id: + raise ValueError( + "Select an uploaded ambient bed from the Environment tab or upload one under Background Noise" + ) + self.background_noise_source = BackgroundNoiseSourceEnum(validated["background_noise_source"]) + self.background_noise_preset = validated["background_noise_preset"] + self.background_noise_volume = validated["background_noise_volume"] + self.background_noise_asset_id = validated["background_noise_asset_id"] + return self + class PersonaUpdate(BaseModel): """Schema for updating a persona""" @@ -520,6 +576,10 @@ class PersonaUpdate(BaseModel): response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) max_turns: Optional[int] = Field(None, ge=1, le=100) allow_interruptions: Optional[bool] = None + background_noise_source: Optional[BackgroundNoiseSourceEnum] = None + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = Field(None, ge=0.05, le=0.60) + background_noise_asset_id: Optional[UUID] = None @model_validator(mode="after") def validate_tts_config(self): @@ -546,6 +606,11 @@ class PersonaResponse(BaseModel): response_delay_ms: Optional[int] = None max_turns: Optional[int] = None allow_interruptions: Optional[bool] = None + background_noise_source: str = BackgroundNoiseSourceEnum.NONE.value + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = None + background_noise_s3_key: Optional[str] = None + background_noise_asset_id: Optional[UUID] = None created_at: datetime updated_at: datetime @@ -563,6 +628,21 @@ def convert_gender(cls, v): model_config = ConfigDict(from_attributes=True) +class AmbientNoiseAssetResponse(BaseModel): + id: UUID + name: str + s3_key: str + original_filename: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class AmbientNoiseAssetUpdateRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + + class PersonaCloneRequest(BaseModel): """Schema for cloning a persona""" name: Optional[str] = None @@ -1639,19 +1719,28 @@ class EvaluatorSuiteCombinationResponse(BaseModel): """One agent+persona+scenario combination inside a suite.""" id: UUID evaluator_id: str + persona_id: Optional[UUID] = None + persona_name: Optional[str] = None scenario_id: Optional[UUID] = None scenario_name: Optional[str] = None scenario_description: Optional[str] = None scenario_required_info: Optional[Any] = None +class EvaluatorSuitePersonaSummary(BaseModel): + """Persona referenced by a suite combination grid.""" + id: UUID + name: Optional[str] = None + + class EvaluatorSuiteCreate(BaseModel): """Schema for creating an evaluator suite.""" name: Optional[str] = None agent_id: UUID - persona_id: UUID + persona_id: Optional[UUID] = None + persona_ids: Optional[List[UUID]] = None scenario_ids: List[UUID] metric_ids: Optional[List[UUID]] = None llm_provider: Optional[ModelProvider] = None @@ -1683,6 +1772,8 @@ class EvaluatorSuiteResponse(BaseModel): name: Optional[str] = None agent_id: UUID persona_id: UUID + persona_ids: List[UUID] = Field(default_factory=list) + personas: List[EvaluatorSuitePersonaSummary] = Field(default_factory=list) agent_name: Optional[str] = None persona_name: Optional[str] = None agent_call_type: Optional[str] = None @@ -1710,6 +1801,16 @@ class EvaluatorSuiteAddScenariosRequest(BaseModel): scenario_ids: List[UUID] +class EvaluatorSuiteAddPersonasRequest(BaseModel): + """Schema for adding personas to an existing suite.""" + persona_ids: List[UUID] + + +class EvaluatorSuiteReplacePersonasRequest(BaseModel): + """Schema for replacing the persona set on an existing suite.""" + persona_ids: List[UUID] = Field(..., min_length=1) + + class RunEvaluatorSuiteRequest(BaseModel): @@ -1741,6 +1842,8 @@ class RunNextCombinationRequest(BaseModel): class RunNextCombinationResponse(BaseModel): """Schema for round-robin run response.""" evaluator_id: UUID + persona_id: Optional[UUID] = None + persona_name: Optional[str] = None scenario_id: Optional[UUID] = None scenario_name: str combination_index: int @@ -1757,6 +1860,8 @@ class RunNextCombinationResponse(BaseModel): class ChooseNextCombinationResponse(BaseModel): """Advance inbound round-robin without initiating a call or evaluation run.""" evaluator_id: UUID + persona_id: Optional[UUID] = None + persona_name: Optional[str] = None scenario_id: Optional[UUID] = None scenario_name: str combination_index: int diff --git a/app/services/audio/ambient_catalog.py b/app/services/audio/ambient_catalog.py new file mode 100644 index 00000000..5d6edd9e --- /dev/null +++ b/app/services/audio/ambient_catalog.py @@ -0,0 +1,257 @@ +"""Pluggable ambient asset catalog and persona resolution.""" + +from __future__ import annotations + +import os +from importlib import metadata +from pathlib import Path +from typing import Any, Optional, Protocol, runtime_checkable + +from loguru import logger + +from app.models.enums import BackgroundNoiseEnum, BackgroundNoiseSourceEnum +from app.services.audio.ambient_mixer import AmbientMixer, clamp_ambient_volume + +PRESET_DISPLAY_NAMES = { + BackgroundNoiseEnum.CAFE.value: "Cafe", + BackgroundNoiseEnum.TRAFFIC.value: "Traffic", + BackgroundNoiseEnum.STREET.value: "Street traffic", + BackgroundNoiseEnum.CONCERT.value: "Concert crowd", + BackgroundNoiseEnum.OFFICE.value: "Office", + BackgroundNoiseEnum.HOME.value: "Home", + BackgroundNoiseEnum.CALL_CENTER.value: "Call center", +} + + +def normalize_ambient_preset(name: Optional[str]) -> Optional[str]: + if not name: + return None + value = str(name).strip().lower() + if value == BackgroundNoiseEnum.NONE.value: + return None + if value == BackgroundNoiseEnum.STREET.value: + return BackgroundNoiseEnum.TRAFFIC.value + return value + + +@runtime_checkable +class AmbientAssetProvider(Protocol): + def list_presets(self) -> list[str]: + ... + + def load_wav(self, name: str) -> bytes: + ... + + +class EmptyAmbientAssetProvider: + def list_presets(self) -> list[str]: + return [] + + def load_wav(self, name: str) -> bytes: + raise FileNotFoundError(name) + + +class DirectoryAmbientAssetProvider: + """Load preset WAV/MP3/OGG files from EFFICIENTAI_AMBIENT_DIR.""" + + def __init__(self, directory: Optional[str] = None): + self._directory = Path(directory or os.getenv("EFFICIENTAI_AMBIENT_DIR", "")).expanduser() + + def list_presets(self) -> list[str]: + if not self._directory.is_dir(): + return [] + names: list[str] = [] + for path in sorted(self._directory.iterdir()): + if path.is_file() and path.suffix.lower() in {".wav", ".mp3", ".ogg", ".m4a", ".flac"}: + names.append(path.stem.lower()) + return names + + def load_wav(self, name: str) -> bytes: + normalized = normalize_ambient_preset(name) or name + if not self._directory.is_dir(): + raise FileNotFoundError(normalized) + for ext in (".wav", ".mp3", ".ogg", ".m4a", ".flac"): + candidate = self._directory / f"{normalized}{ext}" + if candidate.is_file(): + return candidate.read_bytes() + raise FileNotFoundError(normalized) + + +class EntryPointAmbientAssetProvider: + """Load presets from efficientai.ambient_assets entry points.""" + + GROUP = "efficientai.ambient_assets" + + def __init__(self): + self._provider: Optional[AmbientAssetProvider] = None + self._loaded = False + + def _ensure_provider(self) -> Optional[AmbientAssetProvider]: + if self._loaded: + return self._provider + self._loaded = True + try: + eps = metadata.entry_points(group=self.GROUP) + except TypeError: + eps = metadata.entry_points().get(self.GROUP, []) + for ep in eps: + try: + provider = ep.load() + if hasattr(provider, "list_presets") and hasattr(provider, "load_wav"): + self._provider = provider + logger.info("Loaded ambient asset provider from entry point {}", ep.name) + break + except Exception as exc: + logger.warning("Failed to load ambient asset entry point {}: {}", ep.name, exc) + return self._provider + + def list_presets(self) -> list[str]: + provider = self._ensure_provider() + return provider.list_presets() if provider else [] + + def load_wav(self, name: str) -> bytes: + provider = self._ensure_provider() + if not provider: + raise FileNotFoundError(name) + return provider.load_wav(name) + + +class ChainedAmbientAssetProvider: + def __init__(self, providers: list[AmbientAssetProvider]): + self._providers = providers + + def list_presets(self) -> list[str]: + seen: set[str] = set() + ordered: list[str] = [] + for provider in self._providers: + for preset in provider.list_presets(): + normalized = normalize_ambient_preset(preset) or preset + if normalized not in seen: + seen.add(normalized) + ordered.append(normalized) + return ordered + + def load_wav(self, name: str) -> bytes: + normalized = normalize_ambient_preset(name) or name + last_error: Optional[Exception] = None + for provider in self._providers: + try: + return provider.load_wav(normalized) + except FileNotFoundError as exc: + last_error = exc + raise FileNotFoundError(normalized) from last_error + + +_default_provider: Optional[ChainedAmbientAssetProvider] = None + + +def get_ambient_asset_provider() -> ChainedAmbientAssetProvider: + global _default_provider + if _default_provider is None: + _default_provider = ChainedAmbientAssetProvider( + [ + EntryPointAmbientAssetProvider(), + DirectoryAmbientAssetProvider(), + EmptyAmbientAssetProvider(), + ] + ) + return _default_provider + + +def list_ambient_presets() -> list[dict[str, str]]: + provider = get_ambient_asset_provider() + return [ + { + "id": preset, + "label": PRESET_DISPLAY_NAMES.get(preset, preset.replace("_", " ").title()), + } + for preset in provider.list_presets() + ] + + +def persona_ambient_source(persona: Any) -> str: + source = getattr(persona, "background_noise_source", None) or BackgroundNoiseSourceEnum.NONE.value + return str(source).strip().lower() or BackgroundNoiseSourceEnum.NONE.value + + +def persona_ambient_volume(persona: Any) -> float: + return clamp_ambient_volume(getattr(persona, "background_noise_volume", None)) + + +def persona_has_active_ambient(persona: Any) -> bool: + return persona_ambient_source(persona) != BackgroundNoiseSourceEnum.NONE.value + + +def _resolve_custom_ambient_s3_key(persona: Any) -> Optional[str]: + asset_id = getattr(persona, "background_noise_asset_id", None) + if asset_id: + try: + from app.database import SessionLocal + from app.models.database import AmbientNoiseAsset + + db = SessionLocal() + try: + row = db.query(AmbientNoiseAsset).filter(AmbientNoiseAsset.id == asset_id).first() + if row and row.s3_key: + return str(row.s3_key) + finally: + db.close() + except Exception as exc: + logger.warning("Failed to resolve ambient asset {}: {}", asset_id, exc) + legacy_key = getattr(persona, "background_noise_s3_key", None) + return str(legacy_key) if legacy_key else None + + +def _build_ambient_mixer_sync(persona: Any, sample_rate: int) -> Optional[AmbientMixer]: + """Blocking ambient mixer construction (I/O + decode). Run via asyncio.to_thread.""" + source = persona_ambient_source(persona) + volume = persona_ambient_volume(persona) + + if source == BackgroundNoiseSourceEnum.NONE.value: + return None + + if source == BackgroundNoiseSourceEnum.PLATFORM.value: + preset = normalize_ambient_preset(getattr(persona, "background_noise_preset", None)) + if not preset: + logger.warning("Persona {} has platform ambient source but no preset", getattr(persona, "id", "?")) + return None + provider = get_ambient_asset_provider() + try: + file_bytes = provider.load_wav(preset) + except FileNotFoundError: + logger.warning( + "Ambient preset {} is not available (install ambient asset pack or set EFFICIENTAI_AMBIENT_DIR)", + preset, + ) + return None + return AmbientMixer.from_pcm_bytes(file_bytes, sample_rate=sample_rate, volume=volume) + + if source == BackgroundNoiseSourceEnum.CUSTOM.value: + s3_key = _resolve_custom_ambient_s3_key(persona) + if not s3_key: + logger.warning( + "Persona {} has custom ambient source but no resolvable audio", + getattr(persona, "id", "?"), + ) + return None + try: + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + logger.warning("Blob storage disabled; cannot load custom ambient audio for persona {}", getattr(persona, "id", "?")) + return None + file_bytes = s3_service.download_file_by_key(str(s3_key)) + except Exception as exc: + logger.warning("Failed to load custom ambient audio for persona {}: {}", getattr(persona, "id", "?"), exc) + return None + return AmbientMixer.from_pcm_bytes(file_bytes, sample_rate=sample_rate, volume=volume) + + logger.warning("Unknown ambient source {} on persona {}", source, getattr(persona, "id", "?")) + return None + + +async def resolve_ambient_mixer(persona: Any, sample_rate: int) -> Optional[AmbientMixer]: + """Build an AmbientMixer for a persona at the given sample rate, or None.""" + import asyncio + + return await asyncio.to_thread(_build_ambient_mixer_sync, persona, sample_rate) diff --git a/app/services/audio/ambient_input_processor.py b/app/services/audio/ambient_input_processor.py new file mode 100644 index 00000000..b5bc3ec2 --- /dev/null +++ b/app/services/audio/ambient_input_processor.py @@ -0,0 +1,36 @@ +"""Mix persona ambient bed into inbound telephony caller audio.""" + +from __future__ import annotations + +from app.services.audio.ambient_mixer import AmbientBed + +_ambient_input_processor_class = None + + +def get_ambient_input_processor_class(): + """Return AmbientInputProcessor (lazy efficientai import).""" + global _ambient_input_processor_class + if _ambient_input_processor_class is not None: + return _ambient_input_processor_class + + from efficientai.frames.frames import AudioRawFrame + from efficientai.processors.frame_processor import FrameProcessor + + class AmbientInputProcessor(FrameProcessor): + """Overlay looping ambient bed on caller-side AudioRawFrame streams.""" + + def __init__(self, bed: AmbientBed): + super().__init__() + self._bed = bed + + async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + # Mix in place so InputAudioRawFrame keeps Frame metadata (pts, + # transport_source, etc.). Replacing with a bare AudioRawFrame mixin + # drops those fields and the frame can reach the output transport. + if isinstance(frame, AudioRawFrame) and frame.audio: + frame.audio = self._bed.mix_speech(frame.audio) + await self.push_frame(frame, direction) + + _ambient_input_processor_class = AmbientInputProcessor + return _ambient_input_processor_class diff --git a/app/services/audio/ambient_mic_pump.py b/app/services/audio/ambient_mic_pump.py new file mode 100644 index 00000000..801b5e7d --- /dev/null +++ b/app/services/audio/ambient_mic_pump.py @@ -0,0 +1,95 @@ +"""Continuous ambient mic feed for evaluator WebRTC bridges.""" + +from __future__ import annotations + +import asyncio +from typing import Awaitable, Callable, Optional + +from loguru import logger + +from app.services.audio.ambient_mixer import AmbientBed + + +class AmbientMicPump: + """ + Streams continuous ambient-only PCM while idle and mixed speech while active. + + Replaces ElevenLabs' zero-silence loop when a persona has background noise. + """ + + def __init__( + self, + bed: AmbientBed, + *, + sample_rate: int, + chunk_duration_ms: int = 20, + send_callback: Callable[[bytes], Awaitable[None]], + mark_speech_done: Optional[Callable[[], None]] = None, + ): + self._bed = bed + self._sample_rate = sample_rate + self._chunk_duration_ms = chunk_duration_ms + self._send_callback = send_callback + self._mark_speech_done = mark_speech_done + self._speaking = False + self._stop = asyncio.Event() + self._task: Optional[asyncio.Task] = None + + @property + def chunk_duration_ms(self) -> int: + return self._chunk_duration_ms + + def _chunk_samples(self) -> int: + return max(1, (self._sample_rate * self._chunk_duration_ms) // 1000) + + async def start(self): + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._idle_loop()) + logger.info( + "Ambient mic pump started (sample_rate={}, chunk_ms={})", + self._sample_rate, + self._chunk_duration_ms, + ) + + async def stop(self): + self._stop.set() + if self._task: + try: + await asyncio.wait_for(self._task, timeout=2.0) + except asyncio.TimeoutError: + self._task.cancel() + self._task = None + + async def _idle_loop(self): + chunk_samples = self._chunk_samples() + interval = self._chunk_duration_ms / 1000.0 + try: + while not self._stop.is_set(): + if self._speaking: + await asyncio.sleep(0.01) + continue + await self._send_callback(self._bed.chunk_bytes(chunk_samples)) + await asyncio.sleep(interval) + except asyncio.CancelledError: + pass + except Exception as exc: + logger.error("Ambient mic pump idle loop error: {}", exc) + + async def send_speech( + self, + audio_bytes: bytes, + stream_chunks: Callable[[bytes, Callable[[bytes], Awaitable[None]], int], Awaitable[None]], + ): + self._speaking = True + try: + async def mixed_callback(chunk: bytes): + mixed = self._bed.mix_speech(chunk) + await self._send_callback(mixed) + + await stream_chunks(audio_bytes, mixed_callback, self._chunk_duration_ms) + finally: + self._speaking = False + if self._mark_speech_done: + self._mark_speech_done() diff --git a/app/services/audio/ambient_mixer.py b/app/services/audio/ambient_mixer.py new file mode 100644 index 00000000..d9492a6f --- /dev/null +++ b/app/services/audio/ambient_mixer.py @@ -0,0 +1,171 @@ +"""Ambient background audio mixing for test-agent caller simulation.""" + +from __future__ import annotations + +import io +from typing import Optional + +import numpy as np +from loguru import logger + +from efficientai.audio.mixers.base_audio_mixer import BaseAudioMixer +from efficientai.frames.frames import MixerControlFrame + +DEFAULT_AMBIENT_VOLUME = 0.22 +MIN_AMBIENT_VOLUME = 0.05 +MAX_AMBIENT_VOLUME = 0.60 + + +def clamp_ambient_volume(volume: Optional[float]) -> float: + if volume is None: + return DEFAULT_AMBIENT_VOLUME + return float(max(MIN_AMBIENT_VOLUME, min(MAX_AMBIENT_VOLUME, volume))) + + +def resample_mono_int16(audio: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray: + if source_rate == target_rate or len(audio) == 0: + return audio.astype(np.int16, copy=False) + target_len = max(1, int(round(len(audio) * target_rate / source_rate))) + try: + from scipy.signal import resample + + resampled = resample(audio.astype(np.float64), target_len) + except Exception: + source_positions = np.linspace(0, len(audio) - 1, num=len(audio)) + target_positions = np.linspace(0, len(audio) - 1, num=target_len) + resampled = np.interp(target_positions, source_positions, audio.astype(np.float64)) + return np.clip(np.round(resampled), -32768, 32767).astype(np.int16) + + +def decode_audio_bytes_to_pcm_int16(file_bytes: bytes, target_sample_rate: int) -> np.ndarray: + """Decode arbitrary audio bytes to mono int16 PCM at target sample rate.""" + try: + import soundfile as sf + except ModuleNotFoundError as exc: + raise RuntimeError("soundfile is required for ambient audio decoding") from exc + + data, sample_rate = sf.read(io.BytesIO(file_bytes), dtype="float32", always_2d=False) + if data.ndim > 1: + data = np.mean(data, axis=1) + pcm = np.clip(data * 32767.0, -32768, 32767).astype(np.int16) + return resample_mono_int16(pcm, int(sample_rate), target_sample_rate) + + +class AmbientBed: + """Looping mono int16 bed used by bridge pumps and transport mixers.""" + + def __init__( + self, + bed_pcm: np.ndarray, + *, + volume: float = DEFAULT_AMBIENT_VOLUME, + loop: bool = True, + ): + self._bed = np.asarray(bed_pcm, dtype=np.int16) + if self._bed.size == 0: + raise ValueError("ambient bed PCM must not be empty") + self._volume = clamp_ambient_volume(volume) + self._loop = loop + self._pos = 0 + + @property + def volume(self) -> float: + return self._volume + + def chunk(self, num_samples: int) -> np.ndarray: + if num_samples <= 0: + return np.zeros(0, dtype=np.int16) + if not self._loop and self._pos >= len(self._bed): + return np.zeros(num_samples, dtype=np.int16) + + out = np.empty(num_samples, dtype=np.int16) + offset = 0 + while offset < num_samples: + if self._pos >= len(self._bed): + if not self._loop: + out[offset:] = 0 + break + self._pos = 0 + take = min(num_samples - offset, len(self._bed) - self._pos) + segment = self._bed[self._pos : self._pos + take] + out[offset : offset + take] = np.clip( + np.round(segment.astype(np.float64) * self._volume), + -32768, + 32767, + ).astype(np.int16) + self._pos += take + offset += take + return out + + def chunk_bytes(self, num_samples: int) -> bytes: + return self.chunk(num_samples).tobytes() + + def mix_speech(self, speech_bytes: bytes) -> bytes: + if not speech_bytes: + return speech_bytes + speech = np.frombuffer(speech_bytes, dtype=np.int16) + bed = self.chunk(len(speech)) + mixed = np.clip( + speech.astype(np.int32) + bed.astype(np.int32), + -32768, + 32767, + ).astype(np.int16) + return mixed.tobytes() + + def clone(self) -> "AmbientBed": + """Return an independent bed copy for recording without sharing playback position.""" + return AmbientBed( + self._bed.copy(), + volume=self._volume, + loop=self._loop, + ) + + +class AmbientMixer(BaseAudioMixer): + """Output-transport mixer that overlays a looping ambient bed on bot TTS audio.""" + + def __init__(self, bed: AmbientBed): + self._bed = bed + self._sample_rate = 0 + self._enabled = True + + @classmethod + def from_pcm_bytes( + cls, + file_bytes: bytes, + *, + sample_rate: int, + volume: Optional[float] = None, + ) -> "AmbientMixer": + pcm = decode_audio_bytes_to_pcm_int16(file_bytes, sample_rate) + return cls(AmbientBed(pcm, volume=clamp_ambient_volume(volume))) + + @classmethod + def from_pcm_array( + cls, + pcm: np.ndarray, + *, + volume: Optional[float] = None, + ) -> "AmbientMixer": + return cls(AmbientBed(np.asarray(pcm, dtype=np.int16), volume=clamp_ambient_volume(volume))) + + @property + def bed(self) -> AmbientBed: + return self._bed + + async def start(self, sample_rate: int): + self._sample_rate = sample_rate + + async def stop(self): + pass + + async def process_frame(self, frame: MixerControlFrame): + del frame + + async def mix(self, audio: bytes) -> bytes: + if not self._enabled: + return audio + if not audio: + samples = max(1, self._sample_rate // 50) if self._sample_rate else 320 + return self._bed.chunk_bytes(samples) + return self._bed.mix_speech(audio) diff --git a/app/services/audio/ambient_telephony.py b/app/services/audio/ambient_telephony.py new file mode 100644 index 00000000..0f7e2f92 --- /dev/null +++ b/app/services/audio/ambient_telephony.py @@ -0,0 +1,56 @@ +"""Direction-aware ambient placement for live telephony pipelines.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional + +from app.services.audio.ambient_catalog import persona_has_active_ambient, resolve_ambient_mixer +from app.services.audio.ambient_mixer import AmbientBed, AmbientMixer + + +@dataclass(frozen=True) +class TelephonyAmbientConfig: + """Resolved ambient wiring for a telephony call leg.""" + + output_mixer: Optional[AmbientMixer] = None + input_bed: Optional[AmbientBed] = None + + +async def resolve_ambient_for_telephony( + persona: Any, + *, + call_direction: str, + input_sample_rate: int, + output_sample_rate: int, + persona_speaks_via_tts: bool = False, +) -> TelephonyAmbientConfig: + """ + Place persona ambient on the correct telephony leg. + + Use ``persona_speaks_via_tts`` (simulation role), not session direction alone: + + - **Simulated customer/caller** (persona speech is TTS on this leg): mix onto + output toward the remote party — the agent hears caller speech + continuous + ambient live, same as web-bridge ``AmbientMicPump``. + - **Live PSTN caller → production agent** (inbound answer, human on the phone): + mix onto input only (STT + recording). Never attach ``audio_out_mixer`` here; + its idle loop streams ambient on the agent→caller downlink and breaks the call. + """ + if persona is None or not persona_has_active_ambient(persona): + return TelephonyAmbientConfig() + + if persona_speaks_via_tts: + mixer = await resolve_ambient_mixer(persona, output_sample_rate) + if mixer is None: + return TelephonyAmbientConfig() + return TelephonyAmbientConfig(output_mixer=mixer) + + direction = (call_direction or "outbound").strip().lower() + if direction == "inbound": + input_mixer = await resolve_ambient_mixer(persona, input_sample_rate) + if input_mixer is None: + return TelephonyAmbientConfig() + return TelephonyAmbientConfig(input_bed=input_mixer.bed) + + return TelephonyAmbientConfig() diff --git a/app/services/call_import_metric_clusters.py b/app/services/call_import_metric_clusters.py index e02e88cc..1459bb4d 100644 --- a/app/services/call_import_metric_clusters.py +++ b/app/services/call_import_metric_clusters.py @@ -37,9 +37,15 @@ compute_rca_summary, enrich_metric_cluster_groups, ) +from app.services.metric_cluster_rows import ( + MetricClusterSourceRow, + call_import_pair_to_cluster_row, + filter_cluster_rows_by_ids, +) from app.services.metric_failure_policy import ( effective_policies, is_metric_failure, + is_metric_failure_from_scores, policies_from_evaluation_raw, policy_has_failure_criteria, ) @@ -192,6 +198,23 @@ def filter_completed_row_pairs( return [(eval_row, source_row) for eval_row, source_row in pairs if str(eval_row.id) in allowed] +def filter_completed_source_rows( + source_rows: Sequence[MetricClusterSourceRow], + evaluation_row_ids: Optional[Sequence[UUID]], +) -> List[MetricClusterSourceRow]: + return filter_cluster_rows_by_ids(source_rows, evaluation_row_ids) + + +def pairs_to_source_rows( + evaluation: CallImportEvaluation, + completed_row_pairs: Sequence[Tuple[CallImportEvaluationRow, CallImportRow]], +) -> List[MetricClusterSourceRow]: + return [ + call_import_pair_to_cluster_row(evaluation, eval_row, source_row) + for eval_row, source_row in completed_row_pairs + ] + + def resolve_clustering_policies( evaluation: CallImportEvaluation, metrics: Sequence[Metric], @@ -214,6 +237,18 @@ def flagged_metric_names_for_row( source_row: CallImportRow, metrics: Sequence[Metric], policies: Dict[str, MetricFailurePolicy], +) -> List[str]: + return flagged_metric_names_for_source_row( + call_import_pair_to_cluster_row(evaluation, eval_row, source_row), + metrics, + policies, + ) + + +def flagged_metric_names_for_source_row( + source_row: MetricClusterSourceRow, + metrics: Sequence[Metric], + policies: Dict[str, MetricFailurePolicy], ) -> List[str]: names: List[str] = [] for metric in metrics: @@ -222,9 +257,7 @@ def flagged_metric_names_for_row( policy = policies.get(str(metric.id)) if policy is None: continue - if _build_flagged_row_payload( - evaluation, eval_row, source_row, metric, policy - ): + if _build_flagged_row_payload_from_source(source_row, metric, policy): names.append(metric.name) return names @@ -234,17 +267,24 @@ def list_eligible_cluster_rows( completed_row_pairs: Sequence[Tuple[CallImportEvaluationRow, CallImportRow]], metrics: Sequence[Metric], policies: Dict[str, MetricFailurePolicy], +) -> List[Dict[str, Any]]: + source_rows = pairs_to_source_rows(evaluation, completed_row_pairs) + return list_eligible_cluster_source_rows(source_rows, metrics, policies) + + +def list_eligible_cluster_source_rows( + source_rows: Sequence[MetricClusterSourceRow], + metrics: Sequence[Metric], + policies: Dict[str, MetricFailurePolicy], ) -> List[Dict[str, Any]]: items: List[Dict[str, Any]] = [] - for eval_row, source_row in completed_row_pairs: - flagged_names = flagged_metric_names_for_row( - evaluation, eval_row, source_row, metrics, policies - ) + for source_row in source_rows: + flagged_names = flagged_metric_names_for_source_row(source_row, metrics, policies) if not flagged_names: continue items.append( { - "evaluation_row_id": eval_row.id, + "evaluation_row_id": source_row.row_id, "conversation_id": source_row.conversation_id, "row_index": source_row.row_index, "flagged_metric_names": flagged_names, @@ -267,7 +307,19 @@ def _build_flagged_row_payload( metric: Metric, policy: MetricFailurePolicy, ) -> Optional[Dict[str, Any]]: - scores = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + return _build_flagged_row_payload_from_source( + call_import_pair_to_cluster_row(evaluation, eval_row, source_row), + metric, + policy, + ) + + +def _build_flagged_row_payload_from_source( + source_row: MetricClusterSourceRow, + metric: Metric, + policy: MetricFailurePolicy, +) -> Optional[Dict[str, Any]]: + scores = source_row.metric_scores if isinstance(source_row.metric_scores, dict) else {} entry = scores.get(str(metric.id)) derived_from_children = False if not isinstance(entry, dict): @@ -276,7 +328,7 @@ def _build_flagged_row_payload( if isinstance(entry, dict) and (entry.get("skipped") or entry.get("error")): return None - if not is_metric_failure(eval_row, metric, policy): + if not is_metric_failure_from_scores(scores, metric, policy): return None value = _score_value(entry, metric) if isinstance(entry, dict) else None @@ -324,8 +376,6 @@ def _build_flagged_row_payload( rationale = entry.get("rationale") if isinstance(entry, dict) else None if (not isinstance(rationale, str) or not rationale.strip()) and derived_from_children: - # Parent entry is absent in this legacy shape; surface one child rationale - # so cluster prompts still receive failure context. for child_entry_raw in scores.values(): if not isinstance(child_entry_raw, dict): continue @@ -342,14 +392,14 @@ def _build_flagged_row_payload( else "" ) return { - "conversation_id": source_row.conversation_id or str(source_row.id), - "evaluation_row_id": str(eval_row.id), + "conversation_id": source_row.conversation_id, + "evaluation_row_id": str(source_row.row_id), "row_index": source_row.row_index, "metric_name": metric.name, "metric_id": str(metric.id), "value": str(value)[:120], "rationale": rationale_text, - "transcript": _pick_transcript(evaluation, source_row)[:ROW_TRANSCRIPT_CHAR_CAP], + "transcript": source_row.transcript[:ROW_TRANSCRIPT_CHAR_CAP], } @@ -689,6 +739,37 @@ def estimate_metric_clusters_llm_calls( policies: Dict[str, MetricFailurePolicy], *, max_llm_calls: Optional[int] = None, +) -> Tuple[int, int]: + source_rows = pairs_to_source_rows(evaluation, completed_row_pairs) + return estimate_metric_clusters_llm_calls_for_source_rows( + evaluation.id, + metrics, + source_rows, + policies, + max_llm_calls=max_llm_calls, + ) + + +def _selected_evaluation_row_ids_from_raw(metric_clusters_raw: Any) -> List[str]: + if not isinstance(metric_clusters_raw, dict): + return [] + ids_raw = metric_clusters_raw.get("selected_evaluation_row_ids") + if not isinstance(ids_raw, list): + return [] + return [str(rid) for rid in ids_raw if rid] + + +def _selected_evaluation_row_ids(evaluation: CallImportEvaluation) -> List[str]: + return _selected_evaluation_row_ids_from_raw(evaluation.metric_clusters) + + +def estimate_metric_clusters_llm_calls_for_source_rows( + job_key: UUID, + metrics: Sequence[Metric], + source_rows: Sequence[MetricClusterSourceRow], + policies: Dict[str, MetricFailurePolicy], + *, + max_llm_calls: Optional[int] = None, ) -> Tuple[int, int]: """Return ``(flagged_metric_count, total_estimated_llm_calls)``.""" llm_budget = normalize_max_llm_calls(max_llm_calls) @@ -703,12 +784,10 @@ def estimate_metric_clusters_llm_calls( if policy is None or not policy_has_failure_criteria(policy, metric): continue flagged_count = 0 - for eval_row, source_row in completed_row_pairs: - if eval_row.status != "completed": + for source_row in source_rows: + if source_row.status != "completed": continue - if _build_flagged_row_payload( - evaluation, eval_row, source_row, metric, policy - ): + if _build_flagged_row_payload_from_source(source_row, metric, policy): flagged_count += 1 if flagged_count <= 0: continue @@ -717,31 +796,23 @@ def estimate_metric_clusters_llm_calls( _, num_batches = compute_extraction_plan( flagged_count, max_llm_calls=per_metric_cap ) - extraction_calls += num_batches + 1 # extraction batches + synthesis + extraction_calls += num_batches + 1 discovery = 1 if flagged_metric_count > 0 else 0 total = extraction_calls + discovery return flagged_metric_count, max(total, 1) -def _selected_evaluation_row_ids(evaluation: CallImportEvaluation) -> List[str]: - raw = evaluation.metric_clusters - if not isinstance(raw, dict): - return [] - ids_raw = raw.get("selected_evaluation_row_ids") - if not isinstance(ids_raw, list): - return [] - return [str(rid) for rid in ids_raw if rid] - - -def generate_metric_clusters( +def generate_metric_clusters_for_source_rows( db: Session, - evaluation: CallImportEvaluation, + *, + job_key: UUID, + metric_clusters_raw: Any, + completed_row_count: int, organization_id: UUID, provider: ModelProvider, model: str, - *, - completed_row_pairs: Sequence[Tuple[CallImportEvaluationRow, CallImportRow]], + source_rows: Sequence[MetricClusterSourceRow], metrics: Sequence[Metric], policies: Dict[str, MetricFailurePolicy], on_progress: Optional[ProgressCallback] = None, @@ -751,13 +822,11 @@ def generate_metric_clusters( """Run per-metric clustering + proactive discovery for internal diagnostics.""" llm_budget = normalize_max_llm_calls(max_llm_calls) quality_metrics = [m for m in metrics if _metric_is_quality(m)] - selected_row_ids = _selected_evaluation_row_ids(evaluation) - stored_policies, policy_source = policies_from_evaluation_raw( - evaluation.metric_clusters - ) + selected_row_ids = _selected_evaluation_row_ids_from_raw(metric_clusters_raw) + stored_policies, policy_source = policies_from_evaluation_raw(metric_clusters_raw) failure_policies = policies or stored_policies policies_updated_raw = None - raw_mc = evaluation.metric_clusters + raw_mc = metric_clusters_raw if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): try: policies_updated_raw = datetime.fromisoformat( @@ -772,16 +841,12 @@ def generate_metric_clusters( extractions_by_metric: Dict[str, List[Dict[str, Any]]] = {} metrics_by_id: Dict[str, Metric] = {str(m.id): m for m in metrics} total_flagged = 0 - analysed_calls = sum( - 1 - for eval_row, _source in completed_row_pairs - if eval_row.status == "completed" - ) + analysed_calls = sum(1 for row in source_rows if row.status == "completed") - flagged_metric_count, total_calls_estimate = estimate_metric_clusters_llm_calls( - evaluation, + flagged_metric_count, total_calls_estimate = estimate_metric_clusters_llm_calls_for_source_rows( + job_key, metrics, - completed_row_pairs, + source_rows, policies, max_llm_calls=llm_budget, ) @@ -813,7 +878,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: discovered_problems=[], error_message=METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=evaluation.completed_rows, + generated_at_completed_rows=completed_row_count, max_llm_calls=llm_budget, progress={ "completed_llm_calls": completed_calls, @@ -833,12 +898,10 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: if policy is None or not policy_has_failure_criteria(policy, metric): continue flagged_payloads: List[Dict[str, Any]] = [] - for eval_row, source_row in completed_row_pairs: - if eval_row.status != "completed": + for source_row in source_rows: + if source_row.status != "completed": continue - payload = _build_flagged_row_payload( - evaluation, eval_row, source_row, metric, policy - ) + payload = _build_flagged_row_payload_from_source(source_row, metric, policy) if payload: flagged_payloads.append(payload) all_flagged_samples.append(payload) @@ -852,7 +915,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: batch_size, num_batches = compute_extraction_plan( flagged_count, max_llm_calls=max(20, extraction_cap // max(len(quality_metrics), 1)) ) - rng = random.Random(f"{evaluation.id}:{metric.id}") + rng = random.Random(f"{job_key}:{metric.id}") shuffled = list(flagged_payloads) rng.shuffle(shuffled) batches = _batch_rows(shuffled, batch_size)[:num_batches] @@ -875,7 +938,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: except Exception as exc: # noqa: BLE001 logger.warning( "Metric cluster extraction failed for {} / {}: {}", - evaluation.id, + job_key, metric.id, exc, ) @@ -902,7 +965,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: except Exception as exc: # noqa: BLE001 logger.error( "Metric cluster synthesis failed for {} / {}: {}", - evaluation.id, + job_key, metric.id, exc, ) @@ -940,11 +1003,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: total_flagged=max(total_flagged, 1), ) except Exception as exc: # noqa: BLE001 - logger.warning( - "Proactive discovery failed for evaluation {}: {}", - evaluation.id, - exc, - ) + logger.warning("Proactive discovery failed for job {}: {}", job_key, exc) completed_calls += 1 if on_progress: on_progress(completed_calls, total_calls_estimate) @@ -979,9 +1038,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: for g in groups ] if discovered: - overview_parts.append( - f"{len(discovered)} proactively discovered theme(s)" - ) + overview_parts.append(f"{len(discovered)} proactively discovered theme(s)") return EvaluationMetricClustersState( status="completed", @@ -990,7 +1047,7 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: rca_summary=rca_summary, overview="; ".join(overview_parts) if overview_parts else None, generated_at=datetime.now(timezone.utc), - generated_at_completed_rows=evaluation.completed_rows, + generated_at_completed_rows=completed_row_count, max_llm_calls=llm_budget, progress={ "completed_llm_calls": completed_calls, @@ -1007,6 +1064,38 @@ def _check_cancelled() -> Optional[EvaluationMetricClustersState]: ) +def generate_metric_clusters( + db: Session, + evaluation: CallImportEvaluation, + organization_id: UUID, + provider: ModelProvider, + model: str, + *, + completed_row_pairs: Sequence[Tuple[CallImportEvaluationRow, CallImportRow]], + metrics: Sequence[Metric], + policies: Dict[str, MetricFailurePolicy], + on_progress: Optional[ProgressCallback] = None, + max_llm_calls: Optional[int] = None, + is_cancelled: Optional[CancelCheck] = None, +) -> EvaluationMetricClustersState: + source_rows = pairs_to_source_rows(evaluation, completed_row_pairs) + return generate_metric_clusters_for_source_rows( + db, + job_key=evaluation.id, + metric_clusters_raw=evaluation.metric_clusters, + completed_row_count=evaluation.completed_rows, + organization_id=organization_id, + provider=provider, + model=model, + source_rows=source_rows, + metrics=metrics, + policies=policies, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + is_cancelled=is_cancelled, + ) + + def metric_clusters_state_from_raw( raw: Any, *, diff --git a/app/services/evaluators/evaluator_helpers.py b/app/services/evaluators/evaluator_helpers.py index e786fedd..c3015467 100644 --- a/app/services/evaluators/evaluator_helpers.py +++ b/app/services/evaluators/evaluator_helpers.py @@ -205,7 +205,7 @@ def load_suite_combinations( Evaluator.organization_id == organization_id, Evaluator.workspace_id == workspace_id, ) - .order_by(Evaluator.scenario_id) + .order_by(Evaluator.persona_id, Evaluator.scenario_id) .all() ) diff --git a/app/services/evaluators/evaluator_result_metric_clusters.py b/app/services/evaluators/evaluator_result_metric_clusters.py new file mode 100644 index 00000000..86a1df86 --- /dev/null +++ b/app/services/evaluators/evaluator_result_metric_clusters.py @@ -0,0 +1,509 @@ +"""Failure clustering for filtered evaluator-result scopes.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import EvaluatorResult, EvaluatorResultClusterJob, Metric +from app.models.enums import EvaluatorResultStatus +from app.models.schemas import CallImportMetricAggregate, MetricFailurePolicy +from app.services.call_import_metric_clusters import ( + METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + _metric_is_quality, + filter_completed_source_rows, + list_eligible_cluster_source_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_from_raw, + metric_clusters_state_to_db, +) +from app.services.evaluators.evaluator_results_aggregate import compute_evaluator_results_aggregate +from app.services.evaluators.evaluator_results_query import ( + build_evaluator_results_query, + classify_display_status, +) +from app.services.metric_cluster_rows import ( + MetricClusterSourceRow, + build_evaluator_results_scope_key, + evaluator_result_to_cluster_row, +) +from app.services.metric_failure_policy import ( + build_failure_policy_previews, + effective_policies_from_raw, + has_clusterable_metrics_from_scores, + merge_failure_policies_into_raw, + policies_from_evaluation_raw, + validate_failure_policies_for_metrics, +) + + +def get_or_create_cluster_job( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: Optional[UUID] = None, + suite_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, +) -> EvaluatorResultClusterJob: + scope_key = build_evaluator_results_scope_key( + agent_id=agent_id, + suite_id=suite_id, + scenario_id=scenario_id, + ) + job = ( + db.query(EvaluatorResultClusterJob) + .filter( + EvaluatorResultClusterJob.organization_id == organization_id, + EvaluatorResultClusterJob.workspace_id == workspace_id, + EvaluatorResultClusterJob.scope_key == scope_key, + ) + .first() + ) + if job is not None: + return job + job = EvaluatorResultClusterJob( + organization_id=organization_id, + workspace_id=workspace_id, + scope_key=scope_key, + agent_id=agent_id, + suite_id=suite_id, + scenario_id=scenario_id, + ) + db.add(job) + db.flush() + return job + + +def load_completed_evaluator_results( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: Optional[UUID] = None, + suite_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, +) -> List[EvaluatorResult]: + query = build_evaluator_results_query( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=str(agent_id) if agent_id else None, + suite_id=str(suite_id) if suite_id else None, + scenario_id=str(scenario_id) if scenario_id else None, + playground=False, + ) + rows = query.all() + from app.services.live_entity_storage import hydrate_evaluator_results + + hydrate_evaluator_results(rows) + completed: List[EvaluatorResult] = [] + for row in rows: + if classify_display_status(row) == EvaluatorResultStatus.COMPLETED.value: + completed.append(row) + return completed + + +def _metrics_for_ids(db: Session, organization_id: UUID, metric_ids: Sequence[UUID]) -> List[Metric]: + if not metric_ids: + return [] + return ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.id.in_(list(metric_ids)), + ) + .all() + ) + + +def _child_names_by_parent( + db: Session, + organization_id: UUID, + parent_ids: Sequence[UUID], +) -> Dict[str, List[str]]: + if not parent_ids: + return {} + children = ( + db.query(Metric) + .filter( + Metric.organization_id == organization_id, + Metric.parent_metric_id.in_(list(parent_ids)), + ) + .all() + ) + out: Dict[str, List[str]] = {} + for child in children: + if not child.parent_metric_id: + continue + key = str(child.parent_metric_id) + out.setdefault(key, []).append(child.name) + for key in out: + out[key] = sorted(out[key]) + return out + + +def metrics_for_evaluator_result_clustering( + db: Session, + *, + organization_id: UUID, + results: Sequence[EvaluatorResult], +) -> List[Metric]: + aggregate_metric_ids: List[UUID] = [] + seen: set[UUID] = set() + for result in results: + scores = result.metric_scores if isinstance(result.metric_scores, dict) else {} + for metric_id_str in scores.keys(): + try: + metric_id = UUID(metric_id_str) + except ValueError: + continue + if metric_id in seen: + continue + seen.add(metric_id) + aggregate_metric_ids.append(metric_id) + + if not aggregate_metric_ids: + return [] + + aggregate_metrics = _metrics_for_ids(db, organization_id, aggregate_metric_ids) + by_id = {metric.id: metric for metric in aggregate_metrics} + + normalized_ids: List[UUID] = [] + normalized_seen: set[UUID] = set() + for metric_id in aggregate_metric_ids: + metric = by_id.get(metric_id) + target_id = ( + metric.parent_metric_id + if metric is not None and metric.parent_metric_id + else metric_id + ) + if target_id in normalized_seen: + continue + normalized_seen.add(target_id) + normalized_ids.append(target_id) + + metrics = _metrics_for_ids(db, organization_id, normalized_ids) + return [ + metric + for metric in metrics + if getattr(metric, "enabled", True) and _metric_is_quality(metric) + ] + + +def compute_aggregates_for_evaluator_results( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: Optional[UUID] = None, + suite_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, +) -> Tuple[List[CallImportMetricAggregate], int]: + """Return metric aggregates and completed row count for a filter scope.""" + if suite_id is not None: + agg = compute_evaluator_results_aggregate( + db, + organization_id=organization_id, + workspace_id=workspace_id, + suite_id=suite_id, + agent_id=agent_id, + scenario_id=scenario_id, + ) + return list(agg.metrics), agg.completed_rows + + if agent_id is not None and scenario_id is not None: + agg = compute_evaluator_results_aggregate( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + scenario_id=scenario_id, + ) + return list(agg.metrics), agg.completed_rows + + results = load_completed_evaluator_results( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + suite_id=suite_id, + scenario_id=scenario_id, + ) + metrics = metrics_for_evaluator_result_clustering( + db, organization_id=organization_id, results=results + ) + metric_ids = [m.id for m in metrics] + if not metric_ids: + return [], len(results) + + # Reuse aggregate builder via a synthetic narrow scope when possible. + if agent_id and not suite_id and not scenario_id and results: + scenario_ids = {r.scenario_id for r in results if r.scenario_id} + if len(scenario_ids) == 1: + only_scenario = next(iter(scenario_ids)) + agg = compute_evaluator_results_aggregate( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + scenario_id=only_scenario, + ) + return list(agg.metrics), agg.completed_rows + + # Workspace-wide or mixed scope: build aggregates from loaded rows inline. + from collections import defaultdict + + from app.models.schemas import CallImportMetricHistogramBucket, CallImportMetricValueCount + + metric_values: Dict[str, Dict[str, Any]] = defaultdict( + lambda: { + "numeric": [], + "categories": defaultdict(int), + "count": 0, + "skipped": 0, + "errors": 0, + "name": None, + "type": None, + } + ) + for row in results: + scores = row.metric_scores if isinstance(row.metric_scores, dict) else {} + for metric_id_str, entry in scores.items(): + if not isinstance(entry, dict): + continue + bucket = metric_values[metric_id_str] + bucket["count"] += 1 + if entry.get("metric_name"): + bucket["name"] = entry.get("metric_name") + if entry.get("type"): + bucket["type"] = entry.get("type") + if entry.get("skipped"): + bucket["skipped"] += 1 + continue + if entry.get("error"): + bucket["errors"] += 1 + continue + value = entry.get("value") + mtype = (entry.get("type") or "").lower() + if mtype in ("number", "rating") and isinstance(value, (int, float)): + bucket["numeric"].append(float(value)) + elif mtype == "boolean": + label = "true" if value in (True, "true", 1, "1") else "false" + bucket["categories"][label] += 1 + else: + text = str(value) if value is not None else "—" + bucket["categories"][text] += 1 + + metric_meta = {str(m.id): m for m in metrics} + aggregates: List[CallImportMetricAggregate] = [] + for metric_id_str, data in sorted(metric_values.items(), key=lambda kv: kv[0]): + meta = metric_meta.get(metric_id_str) + numeric = data["numeric"] + value_counts = [ + CallImportMetricValueCount(label=label, count=count) + for label, count in sorted(data["categories"].items()) + ] + aggregates.append( + CallImportMetricAggregate( + metric_id=metric_id_str, + metric_name=data["name"] or (meta.name if meta else metric_id_str), + metric_type=data["type"] or (meta.metric_type if meta else None), + metric_category=getattr(meta, "metric_category", "quality") if meta else "quality", + count=data["count"], + skipped=data["skipped"], + errors=data["errors"], + mean=sum(numeric) / len(numeric) if numeric else None, + value_counts=value_counts, + histogram=[], + ) + ) + return aggregates, len(results) + + +def clustering_context_for_job( + db: Session, + job: EvaluatorResultClusterJob, +) -> Tuple[ + List[Metric], + List[CallImportMetricAggregate], + Dict[str, MetricFailurePolicy], + Literal["inferred", "user"], + Dict[str, List[str]], + List[MetricClusterSourceRow], + int, +]: + results = load_completed_evaluator_results( + db, + organization_id=job.organization_id, + workspace_id=job.workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + source_rows = [evaluator_result_to_cluster_row(r) for r in results] + metrics = metrics_for_evaluator_result_clustering( + db, organization_id=job.organization_id, results=results + ) + aggregates, completed_count = compute_aggregates_for_evaluator_results( + db, + organization_id=job.organization_id, + workspace_id=job.workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + parent_ids = [ + m.id + for m in metrics + if getattr(m, "selection_mode", None) and not getattr(m, "parent_metric_id", None) + ] + child_names_by_parent = _child_names_by_parent( + db, job.organization_id, parent_ids + ) + policies, source = effective_policies_from_raw( + job.metric_clusters, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + return metrics, aggregates, policies, source, child_names_by_parent, source_rows, completed_count + + +def metric_clusters_payload(job: EvaluatorResultClusterJob) -> Optional[Any]: + if job.metric_clusters is None: + return None + completed = 0 + # Stale detection uses stored completed count when present. + raw = job.metric_clusters if isinstance(job.metric_clusters, dict) else {} + if isinstance(raw, dict): + completed = int(raw.get("generated_at_completed_rows") or 0) + return metric_clusters_state_from_raw(job.metric_clusters, completed_rows=completed) + + +def resolve_source_row_selection( + db: Session, + job: EvaluatorResultClusterJob, + *, + evaluation_row_ids: Optional[List[UUID]] = None, + row_limit: Optional[int] = None, + policies: Optional[Dict[str, MetricFailurePolicy]] = None, +) -> Tuple[List[MetricClusterSourceRow], List[str]]: + metrics, _aggregates, default_policies, _source, _child_map, source_rows, _ = ( + clustering_context_for_job(db, job) + ) + active_policies = policies or default_policies + eligible = list_eligible_cluster_source_rows(source_rows, metrics, active_policies) + eligible_ordered_ids = [str(item["evaluation_row_id"]) for item in eligible] + eligible_id_set = set(eligible_ordered_ids) + + if evaluation_row_ids is None and row_limit is not None: + selected_ids = eligible_ordered_ids[:row_limit] + filtered = filter_completed_source_rows( + source_rows, [UUID(rid) for rid in selected_ids] + ) + return filtered, selected_ids + + if evaluation_row_ids is None: + selected_ids = eligible_ordered_ids + filtered = filter_completed_source_rows( + source_rows, [UUID(rid) for rid in selected_ids] + ) + return filtered, selected_ids + + requested = {str(rid) for rid in evaluation_row_ids} + completed_id_set = {str(row.row_id) for row in source_rows} + unknown = sorted(requested - completed_id_set) + if unknown: + raise ValueError( + "One or more evaluation_row_ids are missing or not completed: " + + ", ".join(unknown[:5]) + + ("…" if len(unknown) > 5 else "") + ) + not_eligible = sorted(requested - eligible_id_set) + if not_eligible: + raise ValueError( + "Each selected row must have at least one flagged quality metric. " + "Ineligible row(s): " + + ", ".join(not_eligible[:5]) + + ("…" if len(not_eligible) > 5 else "") + ) + selected_ids = sorted(requested) + filtered = filter_completed_source_rows(source_rows, evaluation_row_ids) + return filtered, selected_ids + + +def has_clusterable_evaluator_results( + db: Session, + job: EvaluatorResultClusterJob, + policies: Dict[str, MetricFailurePolicy], +) -> bool: + results = load_completed_evaluator_results( + db, + organization_id=job.organization_id, + workspace_id=job.workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + metrics = metrics_for_evaluator_result_clustering( + db, organization_id=job.organization_id, results=results + ) + + class _RowShim: + def __init__(self, result: EvaluatorResult): + self.status = "completed" + self.metric_scores = result.metric_scores + + return has_clusterable_metrics_from_scores( + metrics, + policies, + [_RowShim(r) for r in results], + ) + + +def apply_metric_clusters_cancel(job: EvaluatorResultClusterJob) -> bool: + raw = job.metric_clusters + if not isinstance(raw, dict): + return False + if (raw.get("status") or "").lower() != "running": + return False + progress = raw.get("progress") if isinstance(raw.get("progress"), dict) else {} + job.metric_clusters = { + **raw, + "status": "cancelled", + "error_message": METRIC_CLUSTERS_CANCELLED_BY_USER_ERROR, + "progress": progress, + "celery_task_id": None, + } + return True + + +def failure_policies_response_for_job(db: Session, job: EvaluatorResultClusterJob): + metrics, aggregates, policies, source, child_names_by_parent, _rows, _ = ( + clustering_context_for_job(db, job) + ) + previews = build_failure_policy_previews( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + effective=policies, + ) + updated_at = None + raw_mc = job.metric_clusters + if isinstance(raw_mc, dict) and raw_mc.get("failure_policies_updated_at"): + try: + updated_at = datetime.fromisoformat(str(raw_mc["failure_policies_updated_at"])) + except ValueError: + updated_at = None + from app.models.schemas import MetricFailurePoliciesResponse + + return MetricFailurePoliciesResponse( + previews=previews, + policies=policies, + source=source, + updated_at=updated_at, + ) diff --git a/app/services/evaluators/evaluator_results_overview.py b/app/services/evaluators/evaluator_results_overview.py index 576634dc..c3934ff8 100644 --- a/app/services/evaluators/evaluator_results_overview.py +++ b/app/services/evaluators/evaluator_results_overview.py @@ -76,16 +76,19 @@ def build_evaluator_results_overview( workspace_id: UUID, agent_id: Optional[UUID] = None, suite_id: Optional[UUID] = None, + since: Optional[datetime] = None, + until: Optional[datetime] = None, ) -> EvaluatorResultsOverviewResponse: - rows = ( - db.query(EvaluatorResult) - .filter( - EvaluatorResult.organization_id == organization_id, - EvaluatorResult.workspace_id == workspace_id, - EvaluatorResult.evaluator_id.isnot(None), - ) - .all() + query = db.query(EvaluatorResult).filter( + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + EvaluatorResult.evaluator_id.isnot(None), ) + if since is not None: + query = query.filter(EvaluatorResult.timestamp >= since) + if until is not None: + query = query.filter(EvaluatorResult.timestamp <= until) + rows = query.all() evaluator_ids: Set[UUID] = {r.evaluator_id for r in rows if r.evaluator_id} evaluators = ( @@ -148,6 +151,18 @@ def build_evaluator_results_overview( for sid, sc in suite_counts.items(): meta = suite_meta.get(sid) if meta and meta.agent_id == aid: + scenario_summaries: List[EvaluatorResultsScenarioSummary] = [] + for (suite_key, scen_id), scen_counts in scenario_counts.items(): + if suite_key != sid: + continue + scenario_summaries.append( + EvaluatorResultsScenarioSummary( + scenario_id=scen_id, + scenario_name=scenario_names.get(scen_id, "Scenario"), + counts=scen_counts.to_schema(), + ) + ) + scenario_summaries.sort(key=lambda s: s.scenario_name.lower()) agent_suites.append( EvaluatorResultsSuiteSummary( suite_id=sid, @@ -155,6 +170,7 @@ def build_evaluator_results_overview( agent_id=meta.agent_id, persona_id=meta.persona_id, counts=sc.to_schema(), + scenarios=scenario_summaries or None, ) ) agent_suites.sort(key=lambda s: (s.suite_name or "").lower()) @@ -172,6 +188,18 @@ def build_evaluator_results_overview( for sid, sc in suite_counts.items(): meta = suite_meta.get(sid) if meta and meta.agent_id == agent_id: + scenario_summaries = [] + for (suite_key, scen_id), scen_counts in scenario_counts.items(): + if suite_key != sid: + continue + scenario_summaries.append( + EvaluatorResultsScenarioSummary( + scenario_id=scen_id, + scenario_name=scenario_names.get(scen_id, "Scenario"), + counts=scen_counts.to_schema(), + ) + ) + scenario_summaries.sort(key=lambda s: s.scenario_name.lower()) agent_suites.append( EvaluatorResultsSuiteSummary( suite_id=sid, @@ -179,6 +207,7 @@ def build_evaluator_results_overview( agent_id=meta.agent_id, persona_id=meta.persona_id, counts=sc.to_schema(), + scenarios=scenario_summaries or None, ) ) agent_suites.sort(key=lambda s: (s.suite_name or "").lower()) diff --git a/app/services/evaluators/evaluator_results_query.py b/app/services/evaluators/evaluator_results_query.py index f4e6b79c..82f50085 100644 --- a/app/services/evaluators/evaluator_results_query.py +++ b/app/services/evaluators/evaluator_results_query.py @@ -9,11 +9,12 @@ from sqlalchemy.orm import Session, Query from sqlalchemy import and_, or_ -from app.models.database import EvaluatorResult, Evaluator, Agent, VoiceBundle, Scenario +from app.models.database import EvaluatorResult, Evaluator, Agent, VoiceBundle, Scenario, Persona from app.models.schemas import ( AgentResponse, EvaluatorResultResponse, ScenarioResponse, + PersonaResponse, EvaluatorResponse, ) from app.services.evaluators.evaluator_result_status import ( @@ -67,9 +68,12 @@ def build_evaluator_results_query( ) if playground is True: - query = query.filter(EvaluatorResult.evaluator_id.is_(None)) if test_agents_only is True: + # Voice-bundle playground test agents (not Retell/Vapi). Includes runs + # that auto-link an evaluator when persona + scenario are selected. query = query.filter(EvaluatorResult.provider_platform.is_(None)) + else: + query = query.filter(EvaluatorResult.evaluator_id.is_(None)) elif playground is False: query = query.filter(EvaluatorResult.evaluator_id.isnot(None)) else: @@ -166,6 +170,12 @@ def serialize_evaluator_result_row( if scenario: scenario_data = ScenarioResponse.model_validate(scenario) + persona_data = None + if result.persona_id: + persona = db.query(Persona).filter(Persona.id == result.persona_id).first() + if persona: + persona_data = PersonaResponse.model_validate(persona) + result_dict: Dict[str, Any] = { "id": result.id, "result_id": result.result_id, @@ -193,6 +203,7 @@ def serialize_evaluator_result_row( "updated_at": result.updated_at, "created_by": result.created_by, "scenario": scenario_data, + "persona": persona_data, "evaluator": evaluator_stub, } diff --git a/app/services/evaluators/evaluator_suite_service.py b/app/services/evaluators/evaluator_suite_service.py index aec3aade..6d45d747 100644 --- a/app/services/evaluators/evaluator_suite_service.py +++ b/app/services/evaluators/evaluator_suite_service.py @@ -1,7 +1,7 @@ """Evaluator suite business logic.""" import random -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Set, Tuple from uuid import UUID from fastapi import HTTPException @@ -26,6 +26,7 @@ from app.models.schemas import ( EvaluatorSuiteCombinationResponse, EvaluatorSuiteCreate, + EvaluatorSuitePersonaSummary, EvaluatorSuiteResponse, EvaluatorSuiteUpdate, ) @@ -46,21 +47,83 @@ def _agent_suite_count( ) +def _resolve_create_persona_ids(data: EvaluatorSuiteCreate) -> List[UUID]: + if data.persona_ids: + persona_ids = list(dict.fromkeys(data.persona_ids)) + if not persona_ids: + raise HTTPException(status_code=400, detail="persona_ids must contain at least one persona") + return persona_ids + if data.persona_id: + return [data.persona_id] + raise HTTPException(status_code=400, detail="Either persona_id or persona_ids is required") + + +def _distinct_ids_in_order(values: List[Optional[UUID]]) -> List[UUID]: + seen: Set[UUID] = set() + ordered: List[UUID] = [] + for value in values: + if value and value not in seen: + seen.add(value) + ordered.append(value) + return ordered + + +def _suite_persona_ids(combinations: List[Evaluator], primary_persona_id: UUID) -> List[UUID]: + combo_persona_ids = _distinct_ids_in_order([c.persona_id for c in combinations]) + if primary_persona_id in combo_persona_ids: + return [primary_persona_id] + [pid for pid in combo_persona_ids if pid != primary_persona_id] + return combo_persona_ids + + +def _suite_scenario_ids(combinations: List[Evaluator]) -> List[UUID]: + return _distinct_ids_in_order([c.scenario_id for c in combinations]) + + +def _load_personas_by_id(db: Session, persona_ids: List[UUID]) -> Dict[UUID, Persona]: + if not persona_ids: + return {} + personas = db.query(Persona).filter(Persona.id.in_(persona_ids)).all() + return {p.id: p for p in personas} + + +def _validate_personas_for_agent( + db: Session, + agent: Agent, + persona_ids: List[UUID], + organization_id: UUID, + workspace_id: UUID, +) -> List[Persona]: + personas = db.query(Persona).filter( + and_( + Persona.id.in_(persona_ids), + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ) + ).all() + if len(personas) != len(set(persona_ids)): + raise HTTPException(status_code=404, detail="One or more personas not found") + personas_by_id = {p.id: p for p in personas} + ordered = [personas_by_id[pid] for pid in persona_ids] + for persona in ordered: + validate_agent_persona_tts(db, agent, persona) + return ordered + + def _build_suite_response( db: Session, suite: EvaluatorSuite, combinations: Optional[List[Evaluator]] = None, ) -> EvaluatorSuiteResponse: if combinations is None: - combinations = ( - db.query(Evaluator) - .filter(Evaluator.suite_id == suite.id) - .order_by(Evaluator.scenario_id) - .all() + combinations = load_suite_combinations( + db, suite.id, suite.organization_id, suite.workspace_id ) agent = db.query(Agent).filter(Agent.id == suite.agent_id).first() - persona = db.query(Persona).filter(Persona.id == suite.persona_id).first() + persona_ids = _suite_persona_ids(combinations, suite.persona_id) + personas_by_id = _load_personas_by_id(db, persona_ids) + primary_persona = personas_by_id.get(suite.persona_id) + scenario_ids = [c.scenario_id for c in combinations if c.scenario_id] scenarios_by_id = {} if scenario_ids: @@ -70,10 +133,13 @@ def _build_suite_response( combo_responses = [] for combo in combinations: scenario = scenarios_by_id.get(combo.scenario_id) if combo.scenario_id else None + combo_persona = personas_by_id.get(combo.persona_id) if combo.persona_id else None combo_responses.append( EvaluatorSuiteCombinationResponse( id=combo.id, evaluator_id=combo.evaluator_id, + persona_id=combo.persona_id, + persona_name=combo_persona.name if combo_persona else None, scenario_id=combo.scenario_id, scenario_name=scenario.name if scenario else None, scenario_description=scenario.description if scenario else None, @@ -81,14 +147,24 @@ def _build_suite_response( ) ) + persona_summaries = [ + EvaluatorSuitePersonaSummary( + id=pid, + name=personas_by_id[pid].name if pid in personas_by_id else None, + ) + for pid in persona_ids + ] + return EvaluatorSuiteResponse( id=suite.id, organization_id=suite.organization_id, name=suite.name, agent_id=suite.agent_id, persona_id=suite.persona_id, + persona_ids=persona_ids, + personas=persona_summaries, agent_name=agent.name if agent else None, - persona_name=persona.name if persona else None, + persona_name=primary_persona.name if primary_persona else None, agent_call_type=getattr(agent, "call_type", None) if agent else None, agent_call_medium=getattr(agent, "call_medium", None) if agent else None, metric_ids=suite.metric_ids, @@ -146,38 +222,105 @@ def _validate_scenarios_for_agent( def _create_child_evaluators( db: Session, suite: EvaluatorSuite, + persona_ids: List[UUID], scenario_ids: List[UUID], validated_metric_ids: Optional[List[str]], ) -> List[Evaluator]: evaluators = [] - for scenario_id in scenario_ids: - evaluator_id = generate_unique_evaluator_id(db) - evaluator = Evaluator( - evaluator_id=evaluator_id, - organization_id=suite.organization_id, - workspace_id=suite.workspace_id, - suite_id=suite.id, - name=suite.name, - agent_id=suite.agent_id, - persona_id=suite.persona_id, - scenario_id=scenario_id, - metric_ids=validated_metric_ids, - llm_provider=suite.llm_provider, - llm_model=suite.llm_model, - llm_config=suite.llm_config, - tags=suite.tags, - ) - db.add(evaluator) - evaluators.append(evaluator) + for persona_id in persona_ids: + for scenario_id in scenario_ids: + evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=evaluator_id, + organization_id=suite.organization_id, + workspace_id=suite.workspace_id, + suite_id=suite.id, + name=suite.name, + agent_id=suite.agent_id, + persona_id=persona_id, + scenario_id=scenario_id, + metric_ids=validated_metric_ids, + llm_provider=suite.llm_provider, + llm_model=suite.llm_model, + llm_config=suite.llm_config, + tags=suite.tags, + ) + db.add(evaluator) + evaluators.append(evaluator) return evaluators +def _delete_combination_rows(db: Session, combos: List[Evaluator]) -> None: + for combo in combos: + db.query(EvaluatorResult).filter(EvaluatorResult.evaluator_id == combo.id).update( + {EvaluatorResult.evaluator_id: None}, synchronize_session=False + ) + db.delete(combo) + + +def _sync_suite_grid( + db: Session, + suite: EvaluatorSuite, + persona_ids: List[UUID], + scenario_ids: List[UUID], +) -> List[Evaluator]: + if not persona_ids: + raise HTTPException(status_code=400, detail="At least one persona is required") + if not scenario_ids: + raise HTTPException(status_code=400, detail="At least one scenario is required") + + existing = load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + desired_pairs = {(pid, sid) for pid in persona_ids for sid in scenario_ids} + + to_delete = [c for c in existing if (c.persona_id, c.scenario_id) not in desired_pairs] + if to_delete: + _delete_combination_rows(db, to_delete) + db.flush() + + still_existing = { + (c.persona_id, c.scenario_id) + for c in load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + } + missing_pairs = [ + (pid, sid) + for pid in persona_ids + for sid in scenario_ids + if (pid, sid) not in still_existing + ] + + if missing_pairs: + for persona_id, scenario_id in missing_pairs: + evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=evaluator_id, + organization_id=suite.organization_id, + workspace_id=suite.workspace_id, + suite_id=suite.id, + name=suite.name, + agent_id=suite.agent_id, + persona_id=persona_id, + scenario_id=scenario_id, + metric_ids=suite.metric_ids, + llm_provider=suite.llm_provider, + llm_model=suite.llm_model, + llm_config=suite.llm_config, + tags=suite.tags, + ) + db.add(evaluator) + + suite.persona_id = persona_ids[0] + db.flush() + return load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + + def create_evaluator_suite( db: Session, organization_id: UUID, workspace_id: UUID, data: EvaluatorSuiteCreate, ) -> EvaluatorSuiteResponse: + persona_ids = _resolve_create_persona_ids(data) + agent = db.query(Agent).filter( and_( Agent.id == data.agent_id, @@ -188,15 +331,7 @@ def create_evaluator_suite( if not agent: raise HTTPException(status_code=404, detail="Agent not found") - persona = db.query(Persona).filter( - and_( - Persona.id == data.persona_id, - Persona.organization_id == organization_id, - Persona.workspace_id == workspace_id, - ) - ).first() - if not persona: - raise HTTPException(status_code=404, detail="Persona not found") + _validate_personas_for_agent(db, agent, persona_ids, organization_id, workspace_id) scenarios = db.query(Scenario).filter( and_( @@ -210,8 +345,8 @@ def create_evaluator_suite( _validate_scenarios_for_agent(scenarios, data.agent_id) - validate_agent_persona_tts(db, agent, persona) validated_metric_ids = validate_metric_ids(db, organization_id, data.metric_ids) + scenario_ids = list(dict.fromkeys(data.scenario_ids)) existing_for_agent = ( db.query(EvaluatorSuite) @@ -227,7 +362,7 @@ def create_evaluator_suite( workspace_id=workspace_id, name=data.name, agent_id=data.agent_id, - persona_id=data.persona_id, + persona_id=persona_ids[0], metric_ids=validated_metric_ids, llm_provider=data.llm_provider.value if data.llm_provider else None, llm_model=data.llm_model, @@ -240,7 +375,9 @@ def create_evaluator_suite( db.add(suite) db.flush() - evaluators = _create_child_evaluators(db, suite, list(dict.fromkeys(data.scenario_ids)), validated_metric_ids) + evaluators = _create_child_evaluators( + db, suite, persona_ids, scenario_ids, validated_metric_ids + ) db.commit() db.refresh(suite) for ev in evaluators: @@ -325,7 +462,11 @@ def add_scenarios_to_suite( _validate_scenarios_for_agent(scenarios, suite.agent_id) - _create_child_evaluators(db, suite, new_ids, suite.metric_ids) + persona_ids = _suite_persona_ids(existing, suite.persona_id) + if not persona_ids: + persona_ids = [suite.persona_id] + + _create_child_evaluators(db, suite, persona_ids, new_ids, suite.metric_ids) db.commit() db.refresh(suite) return _build_suite_response(db, suite) @@ -336,29 +477,120 @@ def remove_scenario_from_suite( suite: EvaluatorSuite, scenario_id: UUID, ) -> EvaluatorSuiteResponse: - combo = ( + combos = ( db.query(Evaluator) .filter( Evaluator.suite_id == suite.id, Evaluator.scenario_id == scenario_id, ) - .first() + .all() ) - if not combo: + if not combos: raise HTTPException(status_code=404, detail="Scenario combination not found in suite") - remaining = ( - db.query(Evaluator) - .filter(Evaluator.suite_id == suite.id, Evaluator.id != combo.id) - .count() - ) - if remaining == 0: + remaining_scenario_ids = { + c.scenario_id + for c in load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + if c.scenario_id and c.scenario_id != scenario_id + } + if not remaining_scenario_ids: raise HTTPException(status_code=400, detail="Cannot remove the last scenario from a suite") - db.query(EvaluatorResult).filter(EvaluatorResult.evaluator_id == combo.id).update( - {EvaluatorResult.evaluator_id: None}, synchronize_session=False + _delete_combination_rows(db, combos) + db.commit() + db.refresh(suite) + return _build_suite_response(db, suite) + + +def add_personas_to_suite( + db: Session, + suite: EvaluatorSuite, + persona_ids: List[UUID], +) -> EvaluatorSuiteResponse: + if not persona_ids: + return _build_suite_response(db, suite) + + existing = load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + current_persona_ids = _suite_persona_ids(existing, suite.persona_id) + new_persona_ids = [pid for pid in persona_ids if pid not in current_persona_ids] + if not new_persona_ids: + return _build_suite_response(db, suite) + + agent = db.query(Agent).filter(Agent.id == suite.agent_id).first() + if not agent: + raise HTTPException(status_code=404, detail="Agent not found") + + _validate_personas_for_agent( + db, agent, new_persona_ids, suite.organization_id, suite.workspace_id + ) + + scenario_ids = _suite_scenario_ids(existing) + if not scenario_ids: + raise HTTPException(status_code=400, detail="Suite has no scenarios") + + _create_child_evaluators(db, suite, new_persona_ids, scenario_ids, suite.metric_ids) + db.commit() + db.refresh(suite) + return _build_suite_response(db, suite) + + +def replace_personas_in_suite( + db: Session, + suite: EvaluatorSuite, + persona_ids: List[UUID], +) -> EvaluatorSuiteResponse: + persona_ids = list(dict.fromkeys(persona_ids)) + if not persona_ids: + raise HTTPException(status_code=400, detail="At least one persona is required") + + agent = db.query(Agent).filter(Agent.id == suite.agent_id).first() + if not agent: + raise HTTPException(status_code=404, detail="Agent not found") + + _validate_personas_for_agent( + db, agent, persona_ids, suite.organization_id, suite.workspace_id ) - db.delete(combo) + + existing = load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + scenario_ids = _suite_scenario_ids(existing) + if not scenario_ids: + raise HTTPException(status_code=400, detail="Suite has no scenarios") + + combinations = _sync_suite_grid(db, suite, persona_ids, scenario_ids) + db.commit() + db.refresh(suite) + return _build_suite_response(db, suite, combinations) + + +def remove_persona_from_suite( + db: Session, + suite: EvaluatorSuite, + persona_id: UUID, +) -> EvaluatorSuiteResponse: + existing = load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + current_persona_ids = _suite_persona_ids(existing, suite.persona_id) + if persona_id not in current_persona_ids: + raise HTTPException(status_code=404, detail="Persona not found in suite") + + if len(current_persona_ids) <= 1: + raise HTTPException(status_code=400, detail="Cannot remove the last persona from a suite") + + combos = ( + db.query(Evaluator) + .filter( + Evaluator.suite_id == suite.id, + Evaluator.persona_id == persona_id, + ) + .all() + ) + _delete_combination_rows(db, combos) + + remaining = load_suite_combinations(db, suite.id, suite.organization_id, suite.workspace_id) + remaining_persona_ids = _suite_persona_ids(remaining, suite.persona_id) + if remaining_persona_ids: + if suite.persona_id not in remaining_persona_ids: + suite.persona_id = remaining_persona_ids[0] + db.commit() db.refresh(suite) return _build_suite_response(db, suite) diff --git a/app/services/media_urls.py b/app/services/media_urls.py index 6ef55adf..5ab87c68 100644 --- a/app/services/media_urls.py +++ b/app/services/media_urls.py @@ -67,6 +67,7 @@ def build_voice_agent_ws_url( agent_id: Optional[str] = None, persona_id: Optional[str] = None, scenario_id: Optional[str] = None, + run_evaluation: bool = False, fallback_host: Optional[str] = None, fallback_scheme: str = "http", ) -> str: @@ -86,5 +87,7 @@ def build_voice_agent_ws_url( query += f"&persona_id={quote(persona_id)}" if scenario_id: query += f"&scenario_id={quote(scenario_id)}" + if run_evaluation: + query += "&run_evaluation=true" return f"{base}{settings.API_V1_PREFIX}/voice-agent/ws?{query}" diff --git a/app/services/metric_cluster_rows.py b/app/services/metric_cluster_rows.py new file mode 100644 index 00000000..0e3b80bf --- /dev/null +++ b/app/services/metric_cluster_rows.py @@ -0,0 +1,103 @@ +"""Generic row adapter for metric failure clustering.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence +from uuid import UUID + +from app.models.database import ( + CallImportEvaluation, + CallImportEvaluationRow, + CallImportRow, + EvaluatorResult, +) +from app.services.call_import_user_insights import _pick_transcript +from app.services.evaluators.evaluator_results_query import classify_display_status + +ROW_TRANSCRIPT_CHAR_CAP = 3000 + + +@dataclass(frozen=True) +class MetricClusterSourceRow: + """Unified shape for call-import and evaluator-result clustering rows.""" + + row_id: UUID + conversation_id: str + row_index: Optional[int] + metric_scores: Dict[str, Any] + status: str + transcript: str + + +def transcript_from_evaluator_result(result: EvaluatorResult) -> str: + text = (result.transcription or "").strip() + if text: + return text[:ROW_TRANSCRIPT_CHAR_CAP] + segments = result.speaker_segments + if isinstance(segments, list) and segments: + parts: List[str] = [] + for seg in segments: + if not isinstance(seg, dict): + continue + speaker = str(seg.get("speaker") or "Speaker").strip() + seg_text = str(seg.get("text") or "").strip() + if seg_text: + parts.append(f"{speaker}: {seg_text}") + joined = "\n".join(parts).strip() + if joined: + return joined[:ROW_TRANSCRIPT_CHAR_CAP] + return "" + + +def evaluator_result_to_cluster_row(result: EvaluatorResult) -> MetricClusterSourceRow: + display_status = classify_display_status(result) + return MetricClusterSourceRow( + row_id=result.id, + conversation_id=result.result_id, + row_index=None, + metric_scores=result.metric_scores if isinstance(result.metric_scores, dict) else {}, + status=display_status, + transcript=transcript_from_evaluator_result(result), + ) + + +def call_import_pair_to_cluster_row( + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow, +) -> MetricClusterSourceRow: + return MetricClusterSourceRow( + row_id=eval_row.id, + conversation_id=source_row.conversation_id or str(source_row.id), + row_index=source_row.row_index, + metric_scores=eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {}, + status=str(eval_row.status or ""), + transcript=_pick_transcript(evaluation, source_row)[:ROW_TRANSCRIPT_CHAR_CAP], + ) + + +def filter_cluster_rows_by_ids( + rows: Sequence[MetricClusterSourceRow], + row_ids: Optional[Sequence[UUID]], +) -> List[MetricClusterSourceRow]: + if not row_ids: + return list(rows) + allowed = {str(rid) for rid in row_ids} + return [row for row in rows if str(row.row_id) in allowed] + + +def build_evaluator_results_scope_key( + *, + agent_id: Optional[UUID] = None, + suite_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, +) -> str: + parts: List[str] = [] + if agent_id: + parts.append(f"agent:{agent_id}") + if suite_id: + parts.append(f"suite:{suite_id}") + if scenario_id: + parts.append(f"scenario:{scenario_id}") + return "|".join(parts) if parts else "all" diff --git a/app/services/metric_failure_policy.py b/app/services/metric_failure_policy.py index c40c5fc6..281013bc 100644 --- a/app/services/metric_failure_policy.py +++ b/app/services/metric_failure_policy.py @@ -261,6 +261,15 @@ def is_metric_failure( return score_matches_failure_policy(scores, metric, policy) +def is_metric_failure_from_scores( + metric_scores: Dict[str, Any], + metric: Metric, + policy: MetricFailurePolicy, +) -> bool: + scores = metric_scores if isinstance(metric_scores, dict) else {} + return score_matches_failure_policy(scores, metric, policy) + + def policies_from_evaluation_raw(raw: Any) -> Tuple[Dict[str, MetricFailurePolicy], str]: if not isinstance(raw, dict): return {}, "inferred" @@ -312,6 +321,26 @@ def build_inferred_policies( return policies +def effective_policies_from_raw( + metric_clusters_raw: Any, + metrics: Sequence[Metric], + aggregates: Sequence[CallImportMetricAggregate], + *, + child_names_by_parent: Optional[Dict[str, List[str]]] = None, +) -> Tuple[Dict[str, MetricFailurePolicy], Literal["inferred", "user"]]: + stored, source = policies_from_evaluation_raw(metric_clusters_raw) + if source == "user" and stored: + return stored, "user" + inferred = build_inferred_policies( + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + if stored and source == "inferred": + return stored, "inferred" + return inferred, "inferred" + + def effective_policies( evaluation: CallImportEvaluation, metrics: Sequence[Metric], @@ -602,6 +631,43 @@ def merge_clustering_policies( return merged +def merge_clustering_policies_from_raw( + submitted: Optional[Dict[str, MetricFailurePolicy]], + metric_clusters_raw: Any, + metrics: Sequence[Metric], + aggregates: Sequence[CallImportMetricAggregate], + *, + child_names_by_parent: Optional[Dict[str, List[str]]] = None, +) -> Dict[str, MetricFailurePolicy]: + """Like ``merge_clustering_policies`` but reads policies from arbitrary JSON raw.""" + effective, _source = effective_policies_from_raw( + metric_clusters_raw, + metrics, + aggregates, + child_names_by_parent=child_names_by_parent, + ) + agg_by_id = {str(a.metric_id): a for a in aggregates} + merged: Dict[str, MetricFailurePolicy] = {} + for metric in metrics: + mid = str(metric.id) + agg = agg_by_id.get(mid) + row_counts: Dict[str, int] = {} + if agg: + for vc in agg.value_counts or []: + row_counts[vc.label] = vc.count + if submitted is not None and mid in submitted: + merged[mid] = submitted[mid] + continue + base = effective.get(mid) or suggest_failure_policy( + metric, + observed_labels=list(row_counts.keys()), + child_names=(child_names_by_parent or {}).get(mid), + numeric_mean=agg.mean if agg else None, + ) + merged[mid] = prune_policy_to_observed_rows(base, metric, row_counts) + return merged + + def has_clusterable_metrics( metrics: Sequence[Metric], policies: Dict[str, MetricFailurePolicy], @@ -620,6 +686,31 @@ def has_clusterable_metrics( return False +def has_clusterable_metrics_from_scores( + metrics: Sequence[Metric], + policies: Dict[str, MetricFailurePolicy], + rows: Sequence[Any], + *, + status_attr: str = "status", + scores_attr: str = "metric_scores", +) -> bool: + """True when at least one completed row matches a metric failure policy.""" + for row in rows: + status = getattr(row, status_attr, None) + if status != "completed": + continue + scores = getattr(row, scores_attr, None) + if not isinstance(scores, dict): + scores = {} + for metric in metrics: + policy = policies.get(str(metric.id)) + if policy is None or not policy_has_failure_criteria(policy, metric): + continue + if is_metric_failure_from_scores(scores, metric, policy): + return True + return False + + def validate_failure_policies_for_metrics( policies: Dict[str, MetricFailurePolicy], metrics: Sequence[Metric], diff --git a/app/services/personas/ambient_library.py b/app/services/personas/ambient_library.py new file mode 100644 index 00000000..ce3d17aa --- /dev/null +++ b/app/services/personas/ambient_library.py @@ -0,0 +1,49 @@ +"""Workspace ambient noise library helpers.""" + +from __future__ import annotations + +import re +from typing import Any, Optional +from uuid import UUID, uuid4 + +from app.services.audio.ambient_mixer import decode_audio_bytes_to_pcm_int16 +from app.services.personas.persona_ambient_noise import ( + ALLOWED_AMBIENT_EXTENSIONS, + MAX_AMBIENT_UPLOAD_BYTES, +) + + +def sanitize_ambient_name(name: Optional[str], fallback: str) -> str: + raw = (name or fallback or "Ambient bed").strip() + cleaned = re.sub(r"\s+", " ", raw) + return cleaned[:255] or "Ambient bed" + + +def ambient_library_s3_key(organization_id: Any, asset_id: Any, extension: str) -> str: + from app.services.storage.s3_service import s3_service + + ext = extension.lower().lstrip(".") + return ( + f"{s3_service.prefix}organizations/{organization_id}/ambient-library/" + f"{asset_id}.{ext}" + ) + + +def validate_ambient_upload_bytes(file_bytes: bytes, *, filename: str) -> str: + extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if extension not in ALLOWED_AMBIENT_EXTENSIONS: + raise ValueError( + f"Unsupported ambient audio format. Allowed: {', '.join(sorted(ALLOWED_AMBIENT_EXTENSIONS))}" + ) + if not file_bytes: + raise ValueError("Uploaded file is empty") + if len(file_bytes) > MAX_AMBIENT_UPLOAD_BYTES: + raise ValueError( + f"Ambient audio must be at most {MAX_AMBIENT_UPLOAD_BYTES // (1024 * 1024)} MB" + ) + decode_audio_bytes_to_pcm_int16(file_bytes, 16000) + return extension + + +def new_ambient_asset_id() -> UUID: + return uuid4() diff --git a/app/services/personas/configured_tts_providers.py b/app/services/personas/configured_tts_providers.py new file mode 100644 index 00000000..b2acb1d5 --- /dev/null +++ b/app/services/personas/configured_tts_providers.py @@ -0,0 +1,58 @@ +"""Resolve TTS provider keys configured for an organization.""" + +from __future__ import annotations + +from typing import Set +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import AIProvider, Integration +from app.models.enums import ModelProvider +from app.services.ai.model_config_service import model_config_service + + +def _tts_capable_provider_keys() -> Set[str]: + keys: Set[str] = set() + for provider_enum in ModelProvider: + try: + tts_models = model_config_service.get_models_by_type(provider_enum, "tts") + except Exception: + tts_models = [] + if tts_models: + keys.add(provider_enum.value) + return keys + + +def get_configured_tts_provider_keys(organization_id: UUID, db: Session) -> Set[str]: + """Return provider keys with active credentials and TTS capability.""" + tts_capable = _tts_capable_provider_keys() + active_keys: Set[str] = set() + + ai_providers = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, # noqa: E712 + ) + .all() + ) + for ap in ai_providers: + pval = (ap.provider or "").lower() + if pval in tts_capable: + active_keys.add(pval) + + integrations = ( + db.query(Integration) + .filter( + Integration.organization_id == organization_id, + Integration.is_active == True, # noqa: E712 + ) + .all() + ) + for integ in integrations: + pval = (integ.platform or "").lower() + if pval in tts_capable: + active_keys.add(pval) + + return active_keys diff --git a/app/services/personas/persona_ambient_noise.py b/app/services/personas/persona_ambient_noise.py new file mode 100644 index 00000000..3bf88cdc --- /dev/null +++ b/app/services/personas/persona_ambient_noise.py @@ -0,0 +1,129 @@ +"""Validation and normalization for persona ambient noise settings.""" + +from __future__ import annotations + +from typing import Any, Optional + +from app.core.usage_entitlement import has_enterprise_entitlement +from app.models.enums import BackgroundNoiseEnum, BackgroundNoiseSourceEnum +from app.services.audio.ambient_catalog import ( + get_ambient_asset_provider, + normalize_ambient_preset, +) +from app.services.audio.ambient_mixer import ( + MAX_AMBIENT_VOLUME, + MIN_AMBIENT_VOLUME, + clamp_ambient_volume, +) + +ALLOWED_AMBIENT_EXTENSIONS = {"wav", "mp3", "ogg", "m4a", "flac"} +MAX_AMBIENT_UPLOAD_BYTES = 10 * 1024 * 1024 + + +def normalize_ambient_source(value: Optional[str]) -> str: + if not value: + return BackgroundNoiseSourceEnum.NONE.value + normalized = str(value).strip().lower() + try: + return BackgroundNoiseSourceEnum(normalized).value + except ValueError as exc: + raise ValueError( + f"background_noise_source must be one of: " + f"{', '.join(s.value for s in BackgroundNoiseSourceEnum)}" + ) from exc + + +def validate_ambient_preset(preset: Optional[str]) -> Optional[str]: + normalized = normalize_ambient_preset(preset) + if not normalized: + raise ValueError("background_noise_preset is required for platform ambient source") + known = {item.value for item in BackgroundNoiseEnum if item != BackgroundNoiseEnum.NONE} + known.add(BackgroundNoiseEnum.TRAFFIC.value) + provider_presets = set(get_ambient_asset_provider().list_presets()) + allowed = known | provider_presets + if normalized not in allowed: + raise ValueError(f"Unknown ambient preset: {normalized}") + return normalized + + +def validate_persona_ambient_fields( + *, + source: Optional[str], + preset: Optional[str], + volume: Optional[float], + s3_key: Optional[str], + asset_id: Optional[Any] = None, + organization_id: Any = None, + workspace_id: Any = None, + require_custom_file: bool = False, + db=None, +) -> dict[str, Any]: + normalized_source = normalize_ambient_source(source) + normalized_volume = clamp_ambient_volume(volume) + if normalized_volume < MIN_AMBIENT_VOLUME or normalized_volume > MAX_AMBIENT_VOLUME: + raise ValueError( + f"background_noise_volume must be between {MIN_AMBIENT_VOLUME} and {MAX_AMBIENT_VOLUME}" + ) + + normalized_preset = None + resolved_asset_id = None + if normalized_source == BackgroundNoiseSourceEnum.PLATFORM.value: + normalized_preset = validate_ambient_preset(preset) + elif preset: + normalized_preset = normalize_ambient_preset(preset) + + if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value: + if organization_id is not None and not has_enterprise_entitlement(organization_id): + raise ValueError( + "Custom ambient audio requires a valid EfficientAI Enterprise license" + ) + resolved_asset_id = None + if asset_id: + if db is None: + resolved_asset_id = asset_id + else: + if db is None: + raise ValueError("Internal error resolving ambient asset") + from app.models.database import AmbientNoiseAsset + + query = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + ) + if workspace_id is not None: + query = query.filter(AmbientNoiseAsset.workspace_id == workspace_id) + row = query.first() + if not row: + raise ValueError("Selected ambient library asset was not found") + resolved_asset_id = row.id + s3_key = None + elif require_custom_file and not s3_key: + raise ValueError("Select an uploaded ambient bed or upload one in the Background Noise tab") + if s3_key and not str(s3_key).strip(): + raise ValueError("background_noise_s3_key must not be empty") + + if normalized_source == BackgroundNoiseSourceEnum.NONE.value: + return { + "background_noise_source": BackgroundNoiseSourceEnum.NONE.value, + "background_noise_preset": None, + "background_noise_volume": normalized_volume, + "background_noise_s3_key": None, + "background_noise_asset_id": None, + } + + return { + "background_noise_source": normalized_source, + "background_noise_preset": normalized_preset, + "background_noise_volume": normalized_volume, + "background_noise_s3_key": s3_key if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value and not asset_id else None, + "background_noise_asset_id": resolved_asset_id if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value else None, + } + + +def persona_ambient_s3_key(organization_id: Any, persona_id: Any, extension: str) -> str: + from app.services.storage.s3_service import s3_service + + ext = extension.lower().lstrip(".") + return ( + f"{s3_service.prefix}organizations/{organization_id}/personas/{persona_id}/ambient.{ext}" + ) diff --git a/app/services/personas/persona_prompt_generation.py b/app/services/personas/persona_prompt_generation.py index 25c044a1..f13e4d32 100644 --- a/app/services/personas/persona_prompt_generation.py +++ b/app/services/personas/persona_prompt_generation.py @@ -120,17 +120,22 @@ def generate_persona_prompt_from_agent( if source == "test_agent": source_prompt = test_agent_prompt source_label = "test agent prompt" + if not source_prompt.strip(): + raise ValueError("Selected agent has no test agent prompt to generate from") elif source == "agent": source_prompt = agent_prompt or test_agent_prompt source_label = "agent / production prompt" + if not source_prompt.strip(): + raise ValueError( + "Selected agent has no production prompt or test agent prompt to generate from" + ) else: source_prompt = test_agent_prompt or agent_prompt source_label = "agent prompt" - - if not source_prompt.strip(): - raise ValueError( - "Selected agent has no test agent prompt or production prompt to generate from" - ) + if not source_prompt.strip(): + raise ValueError( + "Selected agent has no test agent prompt or production prompt to generate from" + ) messages = [ {"role": "system", "content": GENERATE_PERSONA_PROMPT_SYSTEM}, diff --git a/app/services/telephony/vobiz_agent_context.py b/app/services/telephony/vobiz_agent_context.py index 9a6970bc..758ba28a 100644 --- a/app/services/telephony/vobiz_agent_context.py +++ b/app/services/telephony/vobiz_agent_context.py @@ -39,6 +39,16 @@ class VobizAgentContext: llm_api_key: Optional[str] +@dataclass +class VobizTelephonyRunParams: + """Voice pipeline parameters for a live Vobiz media session.""" + + system_instruction: Optional[str] + persona_speaks_via_tts: bool = False + caller_speaks_first: bool = True + caller_opening_text: Optional[str] = None + + def _resolve_api_key_for_provider(db: Session, organization_id: UUID, provider: ModelProvider) -> Optional[str]: provider_value = provider.value if hasattr(provider, "value") else provider @@ -263,6 +273,73 @@ def resolve_vobiz_agent_context( ) +def resolve_vobiz_telephony_run_params( + db: Session, + *, + context: VobizAgentContext, + call_direction: str, + persona_id: Optional[str] = None, + scenario_id: Optional[str] = None, + evaluator_id: Optional[str] = None, +) -> VobizTelephonyRunParams: + """ + Choose production-agent vs simulated-customer prompts for telephony. + + Phone evaluator outbound runs simulate the persona/customer on the media leg + (customer receiving or participating in a call). Live inbound answers with a + human PSTN caller keep the production agent prompt and input-side ambient only. + """ + direction = (call_direction or "outbound").strip().lower() + if direction != "outbound" or not evaluator_id or not persona_id or not scenario_id: + return VobizTelephonyRunParams(system_instruction=context.system_instruction) + + persona = context.persona + if persona is None: + return VobizTelephonyRunParams(system_instruction=context.system_instruction) + + try: + scenario_uuid = UUID(scenario_id) + except ValueError: + return VobizTelephonyRunParams(system_instruction=context.system_instruction) + + scenario_query = db.query(Scenario).filter( + Scenario.id == scenario_uuid, + Scenario.organization_id == context.organization_id, + ) + if context.workspace_id is not None: + scenario_query = scenario_query.filter(Scenario.workspace_id == context.workspace_id) + scenario = scenario_query.first() + if scenario is None: + return VobizTelephonyRunParams(system_instruction=context.system_instruction) + + from app.services.testing.test_agent_simulation_prompt import build_live_test_agent_system_prompt + from app.services.testing.test_agent_template import ( + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, + ) + + first_message_config = resolve_first_message_from_agent(context.agent) + scenario_first_message = None + if scenario.required_info and isinstance(scenario.required_info, dict): + scenario_first_message = scenario.required_info.get("first_message") + + return VobizTelephonyRunParams( + system_instruction=build_live_test_agent_system_prompt( + context.agent, + persona, + scenario, + ), + persona_speaks_via_tts=True, + caller_speaks_first=should_caller_speak_first(first_message_config), + caller_opening_text=resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona.name or "Test Caller", + scenario_first_message=scenario_first_message, + ), + ) + + def vobiz_webhook_base_url() -> str: """Public telephony edge base URL (webhooks + default carrier WebSocket host). diff --git a/app/services/testing/agent_test_setup_generation.py b/app/services/testing/agent_test_setup_generation.py index ed4f0704..a1940120 100644 --- a/app/services/testing/agent_test_setup_generation.py +++ b/app/services/testing/agent_test_setup_generation.py @@ -13,23 +13,19 @@ from app.models.enums import ModelProvider from app.services.ai.llm_service import llm_service - -CANONICAL_SECTION_KEYS: tuple[str, ...] = ( - "purpose", - "behavior", - "expected_interactions", - "personality_traits", - "constraints", +from app.services.testing.test_agent_template import ( + CANONICAL_SECTION_KEYS, + CANONICAL_SECTION_TITLES, + TestAgentFirstMessage, + TestAgentPromptSection, + TestAgentTemplate, + assemble_test_agent_prompt, + derive_caller_first_message, + normalize_first_message, + normalize_sections, + template_from_generation, ) -CANONICAL_SECTION_TITLES: dict[str, str] = { - "purpose": "Purpose", - "behavior": "Behavior", - "expected_interactions": "Expected Interactions", - "personality_traits": "Personality Traits", - "constraints": "Constraints", -} - SCENARIO_DESCRIPTION_SECTIONS: tuple[str, ...] = ( "### Background (2-3 sentences)", "### Caller Intent (1-2 sentences)", @@ -40,23 +36,41 @@ GENERATE_TEST_PROMPT_SYSTEM = ( "You are an expert at generating synthetic voice AI test agents.\n\n" - "You will receive the system prompt of a production voice AI agent.\n\n" - "Your task is to generate the foundational system prompt for the complementary test agent.\n\n" + "You will receive the system prompt of a production voice AI agent (the assistant that answers calls).\n\n" + "Your task is to generate the foundational configuration for the complementary test caller " + "that will call this production agent during evaluation.\n\n" "Do NOT generate a specific persona or scenario. Those are supplied separately.\n\n" - "Instead, generate instructions that define how the synthetic caller should generally behave " - "when interacting with this production agent.\n\n" - "The generated prompt should:\n\n" - "- Infer the complementary role from the production agent.\n" - "- Describe the test agent's general responsibilities.\n" - "- Define conversational behaviour.\n" - "- Define how information should be disclosed.\n" - "- Define how questions should be answered.\n" - "- Encourage realistic, human conversation.\n" - "- Avoid mentioning testing, QA, automation, prompts or evaluation.\n" - "- Assume persona-specific details and scenario context will be injected later.\n" - "- Leave placeholders where appropriate for persona and scenario.\n\n" - "The prompt should be reusable across many personas and scenarios.\n" - "Return only the generated system prompt." + "Return ONLY valid JSON with this exact shape:\n" + "{\n" + ' "sections": [\n' + ' {"key": "complementary_goal", "title": "Role and Goal", "content": "..."},\n' + ' {"key": "talking_style", "title": "Talking Style", "content": "..."},\n' + ' {"key": "questions_to_ask", "title": "Questions to Ask", "content": "..."},\n' + ' {"key": "information_to_relay", "title": "Information to Relay", "content": "..."},\n' + ' {"key": "constraints", "title": "Constraints", "content": "..."}\n' + " ],\n" + ' "first_message": {\n' + ' "production_mode": "assistant_speaks_first" | "assistant_waits_for_user" | ' + '"assistant_speaks_first_model_generated",\n' + ' "production_message": "static greeting text or null"\n' + " }\n" + "}\n\n" + "Section guidance:\n" + "- complementary_goal: what the caller should ultimately achieve, inverted from the production agent's goals/criteria\n" + "- talking_style: how callers of this production agent typically speak (pacing, tone, phone realism)\n" + "- questions_to_ask: typical questions this caller type would ask the production agent\n" + "- information_to_relay: facts/details the caller should be ready to provide when asked\n" + "- constraints: boundaries (don't dump everything at once, don't mention testing/QA, stay realistic)\n\n" + "First message guidance:\n" + "- Infer who speaks first on the production side from greeting/opening instructions in the production prompt\n" + "- production_message: include the static greeting when production_mode is assistant_speaks_first, else null\n" + "- Do not include caller opening lines in sections; caller behavior is derived from production_mode\n\n" + "Rules:\n" + "- Avoid mentioning testing, QA, automation, prompts, or evaluation in section content\n" + "- Assume persona-specific details and scenario context will be injected later\n" + "- Leave placeholders where appropriate for persona and scenario\n" + "- The template must be reusable across many personas and scenarios\n" + "- Return only JSON, no markdown wrapper, no explanation" ) GENERATE_SCENARIOS_SYSTEM = ( @@ -66,17 +80,16 @@ ) -@dataclass -class AgentTestPromptSection: - key: str - title: str - content: str +# Re-export for backward compatibility in tests/imports +AgentTestPromptSection = TestAgentPromptSection @dataclass class TestPromptGenerationResult: - sections: List[AgentTestPromptSection] + sections: List[TestAgentPromptSection] test_agent_prompt: str + first_message: TestAgentFirstMessage + test_agent_template: TestAgentTemplate provider: str model: str @@ -99,11 +112,20 @@ def _strip_llm_text_wrapper(text: str) -> str: """Strip optional markdown fences and trim LLM preamble from plain-text responses.""" cleaned = (text or "").strip() if cleaned.startswith("```"): - cleaned = re.sub(r"^```(?:markdown|md|text)?\s*\n?", "", cleaned) + cleaned = re.sub(r"^```(?:markdown|md|text|json)?\s*\n?", "", cleaned) cleaned = re.sub(r"\n?```\s*$", "", cleaned) return cleaned.strip() +def _extract_json_object(text: str) -> Dict[str, Any]: + cleaned = _strip_llm_text_wrapper(text) + start = cleaned.find("{") + end = cleaned.rfind("}") + if start == -1 or end == -1 or end <= start: + raise ValueError("LLM response did not contain a JSON object") + return json.loads(cleaned[start : end + 1]) + + def _extract_json_array(text: str) -> List[Any]: cleaned = text.strip() if cleaned.startswith("```"): @@ -116,81 +138,6 @@ def _extract_json_array(text: str) -> List[Any]: return json.loads(cleaned[start : end + 1]) -def assemble_test_agent_prompt(sections: Sequence[AgentTestPromptSection]) -> str: - """Deterministically assemble canonical sections into markdown.""" - by_key = {section.key: section for section in sections} - ordered: list[AgentTestPromptSection] = [] - for key in CANONICAL_SECTION_KEYS: - section = by_key.get(key) - if section is None: - ordered.append( - AgentTestPromptSection( - key=key, - title=CANONICAL_SECTION_TITLES[key], - content="Not specified in source prompt.", - ) - ) - else: - ordered.append(section) - - parts: list[str] = [] - for section in ordered: - title = section.title.strip() or CANONICAL_SECTION_TITLES.get(section.key, section.key) - content = (section.content or "").strip() or "Not specified in source prompt." - parts.append(f"## {title}\n\n{content}") - return "\n\n".join(parts) - - -def _normalize_sections(raw_sections: Any) -> List[AgentTestPromptSection]: - if not isinstance(raw_sections, list): - raise ValueError("LLM response sections must be a list") - - by_key: dict[str, AgentTestPromptSection] = {} - for item in raw_sections: - if not isinstance(item, dict): - continue - key = str(item.get("key") or "").strip() - if key not in CANONICAL_SECTION_KEYS: - continue - title = str(item.get("title") or CANONICAL_SECTION_TITLES[key]).strip() - content = str(item.get("content") or "").strip() - by_key[key] = AgentTestPromptSection(key=key, title=title, content=content) - - sections: list[AgentTestPromptSection] = [] - for key in CANONICAL_SECTION_KEYS: - if key in by_key: - sections.append(by_key[key]) - else: - sections.append( - AgentTestPromptSection( - key=key, - title=CANONICAL_SECTION_TITLES[key], - content="Not specified in source prompt.", - ) - ) - return sections - - -def _normalize_scenario_drafts(raw: Any) -> List[ScenarioDraft]: - if not isinstance(raw, list): - raise ValueError("LLM response scenarios must be a list") - - drafts: list[ScenarioDraft] = [] - for item in raw: - if not isinstance(item, dict): - continue - name = str(item.get("name") or "").strip() - description = str(item.get("description") or "").strip() - if not name or not description: - continue - goal = item.get("goal") - goal_str = str(goal).strip() if goal else None - drafts.append(ScenarioDraft(name=name, description=description, goal=goal_str or None)) - if not drafts: - raise ValueError("LLM response did not contain valid scenario drafts") - return drafts - - def build_test_prompt_user_message( *, production_prompt: str, @@ -255,6 +202,26 @@ def build_scenario_generation_user_message( return "\n".join(parts) +def _parse_generation_payload(text: str) -> tuple[List[TestAgentPromptSection], TestAgentFirstMessage]: + try: + payload = _extract_json_object(text) + except (ValueError, json.JSONDecodeError) as exc: + logger.warning(f"[AgentTestSetup] Failed to parse test prompt JSON: {exc}") + raise ValueError("Could not parse generated test agent template from LLM response") from exc + + sections = normalize_sections(payload.get("sections")) + first_message_raw = payload.get("first_message") + if isinstance(first_message_raw, dict): + production_mode = str(first_message_raw.get("production_mode") or "").strip() + production_message = first_message_raw.get("production_message") + production_message_str = str(production_message).strip() if production_message else None + first_message = derive_caller_first_message(production_mode, production_message_str) + else: + first_message = normalize_first_message(first_message_raw) + + return sections, first_message + + def generate_test_prompt_from_production( production_prompt: str, *, @@ -269,7 +236,7 @@ def generate_test_prompt_from_production( llm_config: Optional[Dict[str, Any]] = None, credential_id: Optional[UUID] = None, ) -> TestPromptGenerationResult: - """Stage 1: generate foundational test agent prompt from production prompt.""" + """Stage 1: generate foundational test agent template from production prompt.""" if not production_prompt.strip(): raise ValueError("Production prompt is required") @@ -298,13 +265,18 @@ def generate_test_prompt_from_production( credential_id=credential_id, ) - test_agent_prompt = _strip_llm_text_wrapper(result["text"]) - if not test_agent_prompt: + sections, first_message = _parse_generation_payload(result["text"]) + test_agent_prompt = assemble_test_agent_prompt(sections) + if not test_agent_prompt.strip(): raise ValueError("LLM response did not contain a test agent prompt") + template = template_from_generation(sections, first_message) + return TestPromptGenerationResult( - sections=[], + sections=sections, test_agent_prompt=test_agent_prompt, + first_message=first_message, + test_agent_template=template, provider=llm_provider.value, model=llm_model, ) @@ -369,3 +341,23 @@ def generate_scenarios_from_test_prompt( provider=llm_provider.value, model=llm_model, ) + + +def _normalize_scenario_drafts(raw: Any) -> List[ScenarioDraft]: + if not isinstance(raw, list): + raise ValueError("LLM response scenarios must be a list") + + drafts: list[ScenarioDraft] = [] + for item in raw: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + description = str(item.get("description") or "").strip() + if not name or not description: + continue + goal = item.get("goal") + goal_str = str(goal).strip() if goal else None + drafts.append(ScenarioDraft(name=name, description=description, goal=goal_str or None)) + if not drafts: + raise ValueError("LLM response did not contain valid scenario drafts") + return drafts diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index b13b78b1..05e0b33f 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -30,6 +30,11 @@ compose_test_agent_simulation_prompt, scenario_goal_from_required_info, ) +from app.services.testing.test_agent_template import ( + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, +) from app.workers.celery_app import process_evaluator_result_task @@ -310,6 +315,8 @@ async def _connect_and_bridge_with_webrtc( webrtc_bridge = None test_agent = None + ambient_mic_pump = None + turn_gate = None # Helper function to update status async def update_status(new_status: str, event: str = None, error: str = None): @@ -596,10 +603,17 @@ def resolve_api_key_for_provider( test_agent = None else: scenario_goal = scenario_goal_from_required_info(scenario) - first_message = f"Hello, this is {persona.name} calling." - + first_message_config = resolve_first_message_from_agent(agent) + scenario_first_message = None if scenario.required_info and isinstance(scenario.required_info, dict): - first_message = scenario.required_info.get("first_message", first_message) + scenario_first_message = scenario.required_info.get("first_message") + + first_message = resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona.name or "Test Caller", + scenario_first_message=scenario_first_message, + ) + caller_speaks_first = should_caller_speak_first(first_message_config) persona_description = build_persona_description_for_bridge(persona) effective_max_turns = resolve_persona_max_turns(persona) @@ -623,7 +637,8 @@ def resolve_api_key_for_provider( persona_description=persona_description, scenario_description=getattr(scenario, "description", None) or scenario.name or "Test call scenario", scenario_goal=scenario_goal, - first_message=first_message, + first_message=first_message or "", + caller_speaks_first=caller_speaks_first, llm_api_key=llm_api_key, llm_temperature=getattr(persona, "llm_temperature", None), llm_max_tokens=getattr(persona, "llm_max_tokens", None), @@ -660,61 +675,93 @@ def touch_voice_activity() -> None: nonlocal last_voice_activity last_voice_activity = time.monotonic() - # Start recording - await webrtc_bridge.start_recording() - - logger.info("[Bridge WebRTC] ✅ Recording started") - if test_agent: - # Set up callbacks to connect test agent with voice provider + # Wire turn gate before any slow I/O so Vapi/Retell events are not stalled. chunk_ms = 40 if provider_platform == "vapi" else 20 + ambient_mic_pump = None + + from app.services.audio.ambient_catalog import resolve_ambient_mixer + from app.services.audio.ambient_mic_pump import AmbientMicPump + from app.services.webrtc_bridge.production_turn_gate import ProductionTurnGate async def send_audio_chunks(audio: bytes): """Stream audio to voice provider in real-time chunks.""" touch_voice_activity() - await test_agent.stream_audio_chunks(audio, webrtc_bridge.receive_audio_from_test_agent, chunk_duration_ms=chunk_ms) - # Tell ElevenLabs bridge that we're done sending real audio - # so the background silence stream can resume immediately. - if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done"): - webrtc_bridge.mark_user_audio_done() + if ambient_mic_pump: + await ambient_mic_pump.send_speech( + audio, + test_agent.stream_audio_chunks, + ) + else: + await test_agent.stream_audio_chunks( + audio, + webrtc_bridge.receive_audio_from_test_agent, + chunk_duration_ms=chunk_ms, + ) + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done"): + webrtc_bridge.mark_user_audio_done() + + async def flush_held_transcript(transcript: str): + """Run test-agent LLM+TTS only after VAD confirms production audio is quiet.""" + logger.info( + f"[Bridge WebRTC] VAD gate flush — processing held transcript from " + f"{provider_platform}: {transcript[:50]}..." + ) + test_agent.agent_is_talking = False + turn_gate.set_outbound_active(True) + try: + audio = await test_agent.process_agent_transcript(transcript) + if audio: + logger.info( + f"[Bridge WebRTC] Streaming {len(audio)} bytes of audio to {provider_platform}..." + ) + await send_audio_chunks(audio) + logger.info("[Bridge WebRTC] Audio streaming complete") + else: + logger.warning( + f"[Bridge WebRTC] No audio generated for held transcript — test agent silent " + f"(TTS={test_agent.config.tts_provider}, turn={test_agent.turn_count})" + ) + finally: + turn_gate.set_outbound_active(False) + + turn_gate = ProductionTurnGate( + on_flush=flush_held_transcript, + stop_secs=( + 0.5 + if provider_platform == "vapi" + else 0.25 if provider_platform == "elevenlabs" + else 1.0 + ), + flush_on_vad_quiet=provider_platform != "elevenlabs", + ) + await turn_gate.start() async def on_transcript_received(transcript: str): - """When voice agent finishes speaking, process with test agent.""" + """Hold provider text until inbound audio VAD confirms silence.""" touch_voice_activity() - logger.info(f"[Bridge WebRTC] Received transcript from {provider_platform}: {transcript[:50]}...") - audio = await test_agent.process_agent_transcript(transcript) - if audio: - logger.info(f"[Bridge WebRTC] Streaming {len(audio)} bytes of audio to {provider_platform}...") - await send_audio_chunks(audio) - logger.info("[Bridge WebRTC] Audio streaming complete") - else: - logger.warning(f"[Bridge WebRTC] ⚠️ No audio generated for transcript — test agent silent this turn " - f"(TTS={test_agent.config.tts_provider}, turn={test_agent.turn_count})") + logger.info( + f"[Bridge WebRTC] Holding transcript from {provider_platform}: {transcript[:50]}..." + ) + await turn_gate.hold_transcript(transcript) + + async def on_audio_received(pcm: bytes): + """Feed production-agent PCM into the VAD turn gate.""" + touch_voice_activity() + await turn_gate.ingest_audio(pcm, source_rate=sample_rate) async def on_agent_start_talking(): """Voice AI agent started speaking -- test agent should wait.""" touch_voice_activity() logger.info(f"[Bridge WebRTC] {provider_platform} agent started speaking") test_agent.agent_is_talking = True + await turn_gate.on_production_start_talking() async def on_agent_stop_talking(): - """Voice AI agent stopped speaking -- test agent can respond.""" - logger.info(f"[Bridge WebRTC] {provider_platform} agent stopped speaking") - test_agent.agent_is_talking = False - - # Process any transcript that was queued while agent was talking - pending = test_agent._pending_transcript - if pending: - test_agent._pending_transcript = None - logger.info(f"[Bridge WebRTC] Processing pending transcript after agent stopped: {pending[:50]}...") - audio = await test_agent.process_agent_transcript(pending) - if audio: - logger.info(f"[Bridge WebRTC] Streaming {len(audio)} bytes of audio to {provider_platform}...") - await send_audio_chunks(audio) - logger.info("[Bridge WebRTC] Audio streaming complete") - else: - logger.warning(f"[Bridge WebRTC] ⚠️ No audio generated for pending transcript — test agent silent " - f"(TTS={test_agent.config.tts_provider}, turn={test_agent.turn_count})") + """Provider stop signal — VAD gate may flush via fallback.""" + touch_voice_activity() + logger.info(f"[Bridge WebRTC] {provider_platform} agent stop signal (VAD gate owns flush)") + await turn_gate.on_provider_stop_talking() async def on_call_should_end(): """Test agent decided to end the call.""" @@ -722,18 +769,51 @@ async def on_call_should_end(): webrtc_bridge.is_bridging = False webrtc_bridge.on_transcript_received = on_transcript_received + webrtc_bridge.on_audio_received = on_audio_received webrtc_bridge.on_agent_start_talking = on_agent_start_talking webrtc_bridge.on_agent_stop_talking = on_agent_stop_talking test_agent.on_call_should_end = on_call_should_end - # ElevenLabs agents have a built-in greeting; we skip the test - # agent's first message and let the ElevenLabs agent speak first. - # The background silence loop (started at connection time) simulates - # a live microphone so ElevenLabs' VAD activates normally. - if provider_platform == "elevenlabs": + if hasattr(webrtc_bridge, "replay_buffered_turn_events"): + await webrtc_bridge.replay_buffered_turn_events() + + logger.info("[Bridge WebRTC] Turn gate and provider callbacks wired") + + async def _start_ambient_mic_pump(): + nonlocal ambient_mic_pump + try: + ambient_mixer = await resolve_ambient_mixer(persona, sample_rate) + if not ambient_mixer: + return + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "suppress_background_silence"): + webrtc_bridge.suppress_background_silence() + mark_done = ( + webrtc_bridge.mark_user_audio_done + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done") + else None + ) + ambient_mic_pump = AmbientMicPump( + ambient_mixer.bed, + sample_rate=sample_rate, + chunk_duration_ms=chunk_ms, + send_callback=webrtc_bridge.receive_audio_from_test_agent, + mark_speech_done=mark_done, + ) + await ambient_mic_pump.start() + except Exception as exc: + logger.warning(f"[Bridge WebRTC] Ambient mic pump setup failed: {exc}") + + asyncio.create_task( + _start_ambient_mic_pump(), + name=f"ambient-mic-pump-{provider_platform}", + ) + + # Honor template first-message config; ElevenLabs agents often greet first. + if provider_platform == "elevenlabs" and not caller_speaks_first: logger.info("[Bridge WebRTC] ElevenLabs: waiting for agent greeting (background silence stream active)") + elif not caller_speaks_first: + logger.info("[Bridge WebRTC] Test caller waiting for production agent to speak first") else: - # Retell / Vapi: test agent initiates the conversation logger.info("[Bridge WebRTC] Sending test agent's first message...") first_audio = await test_agent.generate_first_message() if first_audio: @@ -741,13 +821,19 @@ async def on_call_should_end(): await send_audio_chunks(first_audio) logger.info(f"[Bridge WebRTC] ✅ First message sent to {provider_platform}") else: - logger.error(f"[Bridge WebRTC] ⚠️ First message TTS returned no audio — test agent will be silent! " - f"Check TTS provider ({test_agent.config.tts_provider}) config and ffmpeg availability.") + logger.error( + f"[Bridge WebRTC] ⚠️ First message TTS returned no audio — test agent will be silent! " + f"Check TTS provider ({test_agent.config.tts_provider}) config and ffmpeg availability." + ) logger.info("[Bridge WebRTC] ✅ Test agent connected, conversation starting...") else: logger.info("[Bridge WebRTC] Running without test agent - provider will handle the call") + # Start recording after callbacks are live (does not block the event loop). + await webrtc_bridge.start_recording() + logger.info("[Bridge WebRTC] ✅ Recording started") + # Wait for the call to end call_timeout = 300 # 5 minutes max call duration start_time = asyncio.get_event_loop().time() @@ -800,6 +886,10 @@ async def on_call_should_end(): await update_status(EvaluatorResultStatus.FAILED.value, "call_error", str(e)) finally: # Cleanup + if turn_gate: + await turn_gate.stop() + if ambient_mic_pump: + await ambient_mic_pump.stop() if webrtc_bridge: await webrtc_bridge.disconnect() if test_agent: diff --git a/app/services/testing/test_agent_simulation_prompt.py b/app/services/testing/test_agent_simulation_prompt.py index ac1315ab..ddc63393 100644 --- a/app/services/testing/test_agent_simulation_prompt.py +++ b/app/services/testing/test_agent_simulation_prompt.py @@ -6,6 +6,7 @@ from typing import Any, Optional, Sequence, TYPE_CHECKING from app.models.database import Agent, Persona, Scenario +from app.services.testing.test_agent_template import SPOKEN_IDENTITY_GUARDRAIL if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -79,6 +80,24 @@ def format_scenario_prompt(scenario: Scenario) -> str: return "\n".join(parts) if parts else "General test call scenario" +def append_persona_identity_to_caller_prompt(caller_template: str, persona_name: str) -> str: + """Append explicit caller identity so the LLM speaks as the selected persona.""" + name = (persona_name or "").strip() + base = (caller_template or "").strip() + if not name: + return base or "Simulate a natural caller for the scenario below." + identity = ( + f"## Caller identity\n\n" + f"You are {name}. Introduce yourself as {name} and stay in character as {name} " + f"for the entire call." + ) + if not base: + return identity + if name.lower() in base.lower(): + return f"{base}\n\n{identity}" + return f"{base}\n\n{identity}" + + def compose_test_agent_simulation_prompt(agent: Agent, scenario: Scenario) -> str: """Merge core agent prompt and active scenario into test_agent_simulation_prompt.""" agent_name = (agent.name or "Voice AI Agent").strip() @@ -151,23 +170,26 @@ def build_test_agent_system_prompt( ) -> str: """Full caller LLM system prompt: simulation core + persona + instructions.""" under_test_name = (agent_name or agent.name or "Voice AI Agent").strip() + persona_name = (persona.name or "Caller").strip() effective_max_turns = max_turns if max_turns is not None else resolve_persona_max_turns(persona) simulation = compose_test_agent_simulation_prompt(agent, scenario) persona_block = format_persona_block( persona, persona_description=persona_description or build_persona_description_for_bridge(persona), ) - return f"""You are simulating a caller in a voice conversation. Your role is to test a voice AI agent. + return f"""You are {persona_name}, a real person on a phone call. -TEST AGENT SIMULATION PROMPT +{SPOKEN_IDENTITY_GUARDRAIL} + +CONTEXT (for your eyes only — do not read aloud) {simulation} PERSONA {persona_block} INSTRUCTIONS: -1. You are CALLING the voice AI agent described in the test agent simulation prompt -2. Stay in character as the persona described +1. You are CALLING the voice AI agent described above +2. Stay in character as {persona_name} at all times 3. Follow the scenario and work toward the goal 4. Speak naturally as if on a phone call 5. Keep responses concise (1-3 sentences) for natural conversation flow @@ -180,6 +202,72 @@ def build_test_agent_system_prompt( After {effective_max_turns} exchanges, wrap up the conversation politely.""" +def build_live_test_agent_system_prompt( + agent: Agent, + persona: Persona, + scenario: Scenario, + *, + max_turns: Optional[int] = None, + persona_description: Optional[str] = None, +) -> str: + """System prompt for live playground calls: caller template + persona + scenario. + + The human plays the production agent; the voice bundle simulates the caller. + """ + agent_name = (agent.name or "Voice AI Agent").strip() + persona_name = (persona.name or "Caller").strip() + effective_max_turns = max_turns if max_turns is not None else resolve_persona_max_turns(persona) + caller_template = append_persona_identity_to_caller_prompt( + strip_scenario_reference_appendix(agent.description or "").strip(), + persona_name, + ) + + production_context = (getattr(agent, "provider_prompt", None) or "").strip() + persona_block = format_persona_block( + persona, + persona_description=persona_description or build_persona_description_for_bridge(persona), + ) + scenario_block = format_scenario_prompt(scenario) + if persona_name: + scenario_block = f"{scenario_block}\n\nPlay this scenario as {persona_name}." + + parts = [ + f"You are {persona_name}, a real person on a phone call.", + SPOKEN_IDENTITY_GUARDRAIL, + "", + "CALLER PROMPT", + caller_template, + "", + "PERSONA", + persona_block, + "", + "SCENARIO", + scenario_block, + ] + if production_context: + parts.extend( + [ + "", + "PRODUCTION AGENT CONTEXT (what you are testing)", + production_context, + ] + ) + parts.extend( + [ + "", + "INSTRUCTIONS:", + f"1. You are {persona_name} — speak only as this person", + "2. Follow the scenario and work toward the goal", + "3. Speak naturally as if on a phone call", + "4. Keep responses concise (1-3 sentences)", + "5. Respond ONLY with spoken words — no stage directions or markdown", + f"6. The human on the other end is the production agent ({agent_name})", + f"7. After about {effective_max_turns} exchanges, wrap up politely.", + ] + ) + return "\n".join(parts) + + def format_scenarios_reference_appendix(scenarios: Sequence[Scenario]) -> str: """Markdown appendix listing linked scenarios using @scenario{uuid} reference tokens.""" if not scenarios: diff --git a/app/services/testing/test_agent_template.py b/app/services/testing/test_agent_template.py new file mode 100644 index 00000000..ecb419e6 --- /dev/null +++ b/app/services/testing/test_agent_template.py @@ -0,0 +1,249 @@ +"""Structured test agent template: sections, first-message config, and assembly.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Sequence + +CANONICAL_SECTION_KEYS: tuple[str, ...] = ( + "complementary_goal", + "talking_style", + "questions_to_ask", + "information_to_relay", + "constraints", +) + +CANONICAL_SECTION_TITLES: dict[str, str] = { + "complementary_goal": "Role and Goal", + "talking_style": "Talking Style", + "questions_to_ask": "Questions to Ask", + "information_to_relay": "Information to Relay", + "constraints": "Constraints", +} + +PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST = "assistant_speaks_first" +PRODUCTION_MODE_ASSISTANT_WAITS = "assistant_waits_for_user" +PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL = "assistant_speaks_first_model_generated" + +PRODUCTION_MODE_LABELS: dict[str, str] = { + PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST: "Assistant speaks first", + PRODUCTION_MODE_ASSISTANT_WAITS: "Assistant waits for user", + PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL: "Assistant speaks first with model-generated message", +} + +VALID_PRODUCTION_MODES: frozenset[str] = frozenset(PRODUCTION_MODE_LABELS) + +CALLER_MODE_WAIT = "wait" +CALLER_MODE_SPEAK_FIRST = "speak_first" + +DEFAULT_CALLER_MESSAGE = "Hello, I'm calling because I need some help." + +SPOKEN_IDENTITY_GUARDRAIL = ( + "Never say you are a test agent, AI, bot, simulator, or that you are testing or evaluating anything. " + "Never mention prompts, scenarios, personas, or automation. Speak only as the named caller." +) + + +def ensure_opening_includes_persona_name(opening: str, persona_name: str) -> str: + """Ensure the caller's first spoken line identifies as the persona when missing.""" + text = (opening or "").strip() + name = (persona_name or "").strip() + if not name: + return text + if text and name.lower() in text.lower(): + return text + if not text: + return f"Hello, this is {name} calling." + return f"Hi, this is {name}. {text}" + + +@dataclass +class TestAgentPromptSection: + key: str + title: str + content: str + + +@dataclass +class TestAgentFirstMessage: + production_mode: str = PRODUCTION_MODE_ASSISTANT_WAITS + production_message: Optional[str] = None + caller_mode: str = CALLER_MODE_SPEAK_FIRST + caller_message: Optional[str] = DEFAULT_CALLER_MESSAGE + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass +class TestAgentTemplate: + sections: List[TestAgentPromptSection] + first_message: TestAgentFirstMessage + + def to_dict(self) -> Dict[str, Any]: + return { + "sections": [ + {"key": s.key, "title": s.title, "content": s.content} + for s in self.sections + ], + "first_message": self.first_message.to_dict(), + } + + +def default_first_message() -> TestAgentFirstMessage: + """Legacy default: production waits, caller speaks first (matches Retell/Vapi bridge).""" + return TestAgentFirstMessage( + production_mode=PRODUCTION_MODE_ASSISTANT_WAITS, + production_message=None, + caller_mode=CALLER_MODE_SPEAK_FIRST, + caller_message=DEFAULT_CALLER_MESSAGE, + ) + + +def derive_caller_first_message(production_mode: str, production_message: Optional[str] = None) -> TestAgentFirstMessage: + """Invert production who-speaks-first into complementary caller behavior.""" + mode = production_mode if production_mode in VALID_PRODUCTION_MODES else PRODUCTION_MODE_ASSISTANT_WAITS + prod_msg = (production_message or "").strip() or None + + if mode in (PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST, PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL): + return TestAgentFirstMessage( + production_mode=mode, + production_message=prod_msg if mode == PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST else None, + caller_mode=CALLER_MODE_WAIT, + caller_message=None, + ) + + return TestAgentFirstMessage( + production_mode=PRODUCTION_MODE_ASSISTANT_WAITS, + production_message=None, + caller_mode=CALLER_MODE_SPEAK_FIRST, + caller_message=DEFAULT_CALLER_MESSAGE, + ) + + +def assemble_test_agent_prompt(sections: Sequence[TestAgentPromptSection]) -> str: + """Deterministically assemble canonical sections into markdown.""" + by_key = {section.key: section for section in sections} + parts: list[str] = [] + for key in CANONICAL_SECTION_KEYS: + section = by_key.get(key) + title = (section.title.strip() if section else "") or CANONICAL_SECTION_TITLES[key] + content = (section.content.strip() if section and section.content else "") or "Not specified in source prompt." + parts.append(f"## {title}\n\n{content}") + return "\n\n".join(parts) + + +def empty_prompt_sections() -> List[TestAgentPromptSection]: + return [ + TestAgentPromptSection(key=key, title=CANONICAL_SECTION_TITLES[key], content="") + for key in CANONICAL_SECTION_KEYS + ] + + +def normalize_sections(raw_sections: Any) -> List[TestAgentPromptSection]: + if not isinstance(raw_sections, list): + raise ValueError("LLM response sections must be a list") + + by_key: dict[str, TestAgentPromptSection] = {} + for item in raw_sections: + if not isinstance(item, dict): + continue + key = str(item.get("key") or "").strip() + if key not in CANONICAL_SECTION_KEYS: + continue + title = str(item.get("title") or CANONICAL_SECTION_TITLES[key]).strip() + content = str(item.get("content") or "").strip() + by_key[key] = TestAgentPromptSection(key=key, title=title, content=content) + + sections: list[TestAgentPromptSection] = [] + for key in CANONICAL_SECTION_KEYS: + if key in by_key: + sections.append(by_key[key]) + else: + sections.append( + TestAgentPromptSection( + key=key, + title=CANONICAL_SECTION_TITLES[key], + content="Not specified in source prompt.", + ) + ) + return sections + + +def normalize_first_message(raw: Any) -> TestAgentFirstMessage: + if not isinstance(raw, dict): + return default_first_message() + + production_mode = str(raw.get("production_mode") or PRODUCTION_MODE_ASSISTANT_WAITS).strip() + production_message = raw.get("production_message") + production_message_str = str(production_message).strip() if production_message else None + + derived = derive_caller_first_message(production_mode, production_message_str) + + caller_mode = str(raw.get("caller_mode") or derived.caller_mode).strip() + if caller_mode not in (CALLER_MODE_WAIT, CALLER_MODE_SPEAK_FIRST): + caller_mode = derived.caller_mode + + caller_message = raw.get("caller_message") + caller_message_str = str(caller_message).strip() if caller_message else derived.caller_message + + return TestAgentFirstMessage( + production_mode=derived.production_mode, + production_message=derived.production_message, + caller_mode=caller_mode, + caller_message=caller_message_str if caller_mode == CALLER_MODE_SPEAK_FIRST else None, + ) + + +def parse_test_agent_template(raw: Any) -> Optional[TestAgentTemplate]: + if not isinstance(raw, dict): + return None + sections_raw = raw.get("sections") + if not isinstance(sections_raw, list): + return None + sections = normalize_sections(sections_raw) + first_message = normalize_first_message(raw.get("first_message")) + return TestAgentTemplate(sections=sections, first_message=first_message) + + +def template_from_generation( + sections: Sequence[TestAgentPromptSection], + first_message: TestAgentFirstMessage, +) -> TestAgentTemplate: + return TestAgentTemplate(sections=list(sections), first_message=first_message) + + +def resolve_first_message_from_agent(agent: Any) -> TestAgentFirstMessage: + """Read first-message config from agent row, with legacy default.""" + raw = getattr(agent, "test_agent_template", None) + parsed = parse_test_agent_template(raw) + if parsed is not None: + return parsed.first_message + return default_first_message() + + +def resolve_caller_opening_text( + *, + first_message: TestAgentFirstMessage, + persona_name: str, + scenario_first_message: Optional[str] = None, +) -> Optional[str]: + """Return caller opening line when caller speaks first; None when caller waits.""" + if first_message.caller_mode == CALLER_MODE_WAIT: + return None + + name = (persona_name or "Test Caller").strip() + opening: Optional[str] = None + + if scenario_first_message and str(scenario_first_message).strip(): + opening = str(scenario_first_message).strip() + elif first_message.caller_message and str(first_message.caller_message).strip(): + opening = str(first_message.caller_message).strip() + else: + opening = f"Hello, this is {name} calling." + + return ensure_opening_includes_persona_name(opening, name) + + +def should_caller_speak_first(first_message: TestAgentFirstMessage) -> bool: + return first_message.caller_mode == CALLER_MODE_SPEAK_FIRST diff --git a/app/services/voice_agent/audio_recorder.py b/app/services/voice_agent/audio_recorder.py index 1599dcb9..243219c8 100644 --- a/app/services/voice_agent/audio_recorder.py +++ b/app/services/voice_agent/audio_recorder.py @@ -4,12 +4,19 @@ import time import wave +from typing import TYPE_CHECKING, Literal, Optional import numpy as np from loguru import logger +if TYPE_CHECKING: + from app.services.audio.ambient_mixer import AmbientBed + _audio_recorder_class = None +# Safety cap: refuse to pad more than 30 minutes in one gap (bad clock / hung call). +_MAX_WALL_CLOCK_PAD_SAMPLES = 30 * 60 * 8000 + def get_audio_recorder_class(): """Return the AudioRecorder FrameProcessor subclass (lazy efficientai import).""" @@ -17,8 +24,15 @@ def get_audio_recorder_class(): if _audio_recorder_class is not None: return _audio_recorder_class - from efficientai.frames.frames import AudioRawFrame, CancelFrame, EndFrame - from efficientai.processors.frame_processor import FrameProcessor + from efficientai.frames.frames import ( + AudioRawFrame, + BotStartedSpeakingFrame, + CancelFrame, + EndFrame, + InputAudioRawFrame, + OutputAudioRawFrame, + ) + from efficientai.processors.frame_processor import FrameDirection, FrameProcessor class AudioRecorder(FrameProcessor): def __init__( @@ -28,6 +42,8 @@ def __init__( target_sample_rate: int = 24000, recorder_name: str = "AudioRecorder", alignment_mode: str = "wall_clock", + capture: Literal["input", "output"] = "input", + ambient_bed: Optional["AmbientBed"] = None, ): super().__init__() self.filename = filename @@ -35,6 +51,8 @@ def __init__( self.target_sample_rate = target_sample_rate self.recorder_name = recorder_name self.alignment_mode = alignment_mode + self.capture = capture + self.ambient_bed = ambient_bed self.wave_file = None self.params_set = False self.sample_rate = 0 @@ -43,6 +61,9 @@ def __init__( self.audio_frames_received = 0 self.last_frame_time = None self.total_samples_written = 0 + # "playout" mode only: wall-clock time of the utterance we are about to + # write, captured from BotStartedSpeakingFrame. + self._pending_anchor_time = None def _resample_audio( self, audio_bytes: bytes, in_rate: int, out_rate: int, num_channels: int @@ -76,11 +97,81 @@ def _write_audio(self, audio_to_write: bytes, num_channels: int) -> None: self.wave_file.writeframes(audio_to_write) self.total_samples_written += num_samples + def _prepare_audio_bytes(self, audio_bytes: bytes) -> bytes: + if not audio_bytes: + return audio_bytes + if self.ambient_bed is not None: + return self.ambient_bed.mix_speech(audio_bytes) + return audio_bytes + + def _pad_bytes(self, num_samples: int, num_channels: int) -> bytes: + if num_samples <= 0: + return b"" + if self.ambient_bed is not None: + bed_mono = self.ambient_bed.chunk_bytes(num_samples) + if num_channels == 1: + return bed_mono + bed_arr = np.frombuffer(bed_mono, dtype=np.int16) + return np.repeat(bed_arr, num_channels).astype(np.int16).tobytes() + return b"\x00" * (num_samples * num_channels * 2) + + def _write_wall_clock_pad(self, current_time: float) -> None: + elapsed_time = current_time - self.start_time + expected_samples = int(elapsed_time * self.sample_rate) + if expected_samples <= self.total_samples_written: + return + + samples_to_pad = expected_samples - self.total_samples_written + max_pad = max( + _MAX_WALL_CLOCK_PAD_SAMPLES, + self.sample_rate * 60, + ) + if samples_to_pad > max_pad: + logger.warning( + "{} skipping excessive wall-clock pad: {} samples (cap {})", + self.recorder_name, + samples_to_pad, + max_pad, + ) + samples_to_pad = max_pad + + chunk_size = self.sample_rate + remaining = samples_to_pad + while remaining > 0: + write_samples = min(remaining, chunk_size) + pad_bytes = self._pad_bytes(write_samples, self.num_channels) + self.wave_file.writeframes(pad_bytes) + self.total_samples_written += write_samples + remaining -= write_samples + + def _should_capture(self, frame: AudioRawFrame, direction: FrameDirection) -> bool: + if direction != FrameDirection.DOWNSTREAM: + return False + if self.capture == "input": + return isinstance(frame, InputAudioRawFrame) + return isinstance(frame, OutputAudioRawFrame) + + def _close_wave_file(self, *, trailing_pad: bool = False) -> None: + if not self.wave_file: + return + try: + if ( + trailing_pad + and self.alignment_mode in ("wall_clock", "playout") + and self.params_set + ): + self._write_wall_clock_pad(time.time()) + except Exception as e: + logger.error(f"Error writing trailing pad for {self.recorder_name}: {e}") + finally: + self.wave_file.close() + self.wave_file = None + async def process_frame(self, frame, direction): await super().process_frame(frame, direction) self.frames_received += 1 - if isinstance(frame, AudioRawFrame): + if isinstance(frame, AudioRawFrame) and self._should_capture(frame, direction): self.audio_frames_received += 1 if not self.wave_file: try: @@ -113,36 +204,46 @@ async def process_frame(self, frame, direction): f"got {frame.num_channels}ch. Skipping frame." ) elif self.alignment_mode == "stream": + audio_to_write = self._prepare_audio_bytes(audio_to_write) + self._write_audio(audio_to_write, self.num_channels) + self.last_frame_time = current_time + elif self.alignment_mode == "playout": + # Anchor each utterance to the wall-clock moment the bot + # actually started speaking, then write its audio + # contiguously. Frames reach us faster than real time + # (the transport drains its send queue at ~2x), so padding + # per-frame against the wall clock would compress the + # track and pull bot speech earlier than it was heard. + anchor = self._pending_anchor_time + if anchor is None and self.total_samples_written == 0: + anchor = current_time + if anchor is not None: + self._write_wall_clock_pad(anchor) + self._pending_anchor_time = None + audio_to_write = self._prepare_audio_bytes(audio_to_write) self._write_audio(audio_to_write, self.num_channels) self.last_frame_time = current_time else: - elapsed_time = current_time - self.start_time - expected_samples = int(elapsed_time * self.sample_rate) - if expected_samples > self.total_samples_written: - samples_to_pad = expected_samples - self.total_samples_written - if samples_to_pad <= self.sample_rate: - silence_bytes = b"\x00" * ( - samples_to_pad * self.num_channels * 2 - ) - self.wave_file.writeframes(silence_bytes) - self.total_samples_written += samples_to_pad - + self._write_wall_clock_pad(current_time) + audio_to_write = self._prepare_audio_bytes(audio_to_write) self._write_audio(audio_to_write, self.num_channels) self.last_frame_time = current_time except Exception as e: logger.error(f"Error writing audio frame: {e}") + elif isinstance(frame, BotStartedSpeakingFrame): + # Emitted by the output transport as it drains its send queue, i.e. + # at playout time rather than TTS generation time. + if self.alignment_mode == "playout": + self._pending_anchor_time = time.time() + elif isinstance(frame, (EndFrame, CancelFrame)): - if self.wave_file: - self.wave_file.close() - self.wave_file = None + self._close_wave_file(trailing_pad=True) await self.push_frame(frame, direction) async def cleanup(self): - if self.wave_file: - self.wave_file.close() - self.wave_file = None + self._close_wave_file(trailing_pad=True) _audio_recorder_class = AudioRecorder return _audio_recorder_class diff --git a/app/services/voice_agent/bot_fast_api.py b/app/services/voice_agent/bot_fast_api.py index 3bcd92a7..72541b11 100644 --- a/app/services/voice_agent/bot_fast_api.py +++ b/app/services/voice_agent/bot_fast_api.py @@ -98,7 +98,7 @@ def _get_imports(): """ -async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None): +async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None, persona=None, call_direction: str = "outbound", persona_speaks_via_tts: bool = False): """ Run the voice agent bot with the provided Google API key. @@ -132,6 +132,27 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str transport_out_sample_rate = resolve_websocket_audio_out_sample_rate_hz( telephony_mode=telephony_mode, ) + ambient_mixer = None + ambient_input_processor = None + if persona is not None and telephony_mode: + from app.services.audio.ambient_telephony import resolve_ambient_for_telephony + from app.services.audio.ambient_input_processor import get_ambient_input_processor_class + + ambient_config = await resolve_ambient_for_telephony( + persona, + call_direction=call_direction, + input_sample_rate=transport_in_sample_rate, + output_sample_rate=transport_out_sample_rate, + persona_speaks_via_tts=persona_speaks_via_tts, + ) + ambient_mixer = ambient_config.output_mixer + if ambient_config.input_bed is not None: + AmbientInputProcessor = get_ambient_input_processor_class() + ambient_input_processor = AmbientInputProcessor(ambient_config.input_bed) + elif persona is not None: + from app.services.audio.ambient_catalog import resolve_ambient_mixer + + ambient_mixer = await resolve_ambient_mixer(persona, transport_out_sample_rate) ws_transport = imports["FastAPIWebsocketTransport"]( websocket=websocket_client, params=imports["FastAPIWebsocketParams"]( @@ -142,6 +163,7 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str audio_out_sample_rate=transport_out_sample_rate if telephony_mode else None, vad_analyzer=imports["SileroVADAnalyzer"](), serializer=transport_serializer, + audio_out_mixer=ambient_mixer, ), ) @@ -199,20 +221,25 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str # Use a common start time for synchronization start_time = time.time() - recorder_alignment = "stream" if telephony_mode else "wall_clock" + recording_ambient_bed = None + if telephony_mode and ambient_mixer is not None: + recording_ambient_bed = ambient_mixer.bed.clone() user_recorder = AudioRecorder( user_audio_path, start_time, target_sample_rate=recorder_sample_rate, recorder_name="UserAudioRecorder", - alignment_mode=recorder_alignment, + alignment_mode="wall_clock", + capture="input", ) bot_recorder = AudioRecorder( bot_audio_path, start_time, target_sample_rate=recorder_sample_rate, recorder_name="BotAudioRecorder", - alignment_mode=recorder_alignment, + alignment_mode="playout", + capture="output", + ambient_bed=recording_ambient_bed, ) from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor @@ -240,6 +267,8 @@ async def on_silence_hangup(): if telephony_mode: pipeline_processors = [ws_transport.input()] + if ambient_input_processor: + pipeline_processors.append(ambient_input_processor) if silence_hangup_processor: pipeline_processors.append(silence_hangup_processor) pipeline_processors.extend([user_recorder, context_aggregator.user()]) @@ -260,8 +289,8 @@ async def on_silence_hangup(): if agent_transcript_processor: pipeline_processors.append(agent_transcript_processor) pipeline_processors.extend([ - bot_recorder, ws_transport.output(), + bot_recorder, context_aggregator.assistant(), ]) pipeline = imports["Pipeline"](pipeline_processors) @@ -309,8 +338,8 @@ async def on_client_disconnected(transport, client): pipeline_steps.append(usage_recorder) pipeline_steps.extend( [ - bot_recorder, ws_transport.output(), + bot_recorder, context_aggregator.assistant(), ] ) @@ -360,6 +389,9 @@ async def on_client_disconnected(transport, client): organization_id=organization_id, evaluator_id=evaluator_id, result_id=result_id, + call_direction=call_direction if telephony_mode else None, + user_audio_frames=user_recorder.audio_frames_received, + bot_audio_frames=bot_recorder.audio_frames_received, ) # Extract conversation transcript from the LLM context diff --git a/app/services/voice_agent/llm_voice_providers.py b/app/services/voice_agent/llm_voice_providers.py new file mode 100644 index 00000000..59bec721 --- /dev/null +++ b/app/services/voice_agent/llm_voice_providers.py @@ -0,0 +1,339 @@ +"""Live voice pipeline LLM provider registry and service factory. + +Mirrors the LLM-capable providers exposed in Voice Bundles / Integrations +(see ``app/services/judge_alignment/model_catalog.py``). +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Callable, Dict, Optional + +from loguru import logger + +# Providers selectable for the LLM leg of STT+LLM+TTS voice bundles. +LLM_VOICE_PROVIDER_KEYS = frozenset( + { + "openai", + "anthropic", + "google", + "xai", + "fireworks", + "cohere", + "mistral", + "meta", + "together", + "perplexity", + "azure", + "aws", + "openrouter", + "custom", + "sarvam", + } +) + +_DEFAULT_LLM_MODELS: Dict[str, str] = { + "openai": "gpt-4.1", + "google": "gemini-2.5-flash", + "anthropic": "claude-sonnet-4.6", + "xai": "grok-3-beta", + "fireworks": "accounts/fireworks/models/llama-v3p1-8b-instruct", + "cohere": "command-r-plus-08-2024", + "mistral": "mistral-small-latest", + "meta": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "together": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "perplexity": "sonar", + "azure": "gpt-4.1", + "aws": "amazon.nova-lite-v1:0", + "openrouter": "openai/gpt-4o-2024-11-20", + "custom": "gpt-4o-mini", + "sarvam": "sarvam-30b", +} + +_ENV_KEYS: Dict[str, str] = { + "openai": "OPENAI_API_KEY", + "google": "GOOGLE_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "xai": "XAI_API_KEY", + "fireworks": "FIREWORKS_API_KEY", + "cohere": "COHERE_API_KEY", + "mistral": "MISTRAL_API_KEY", + "meta": "TOGETHER_API_KEY", + "together": "TOGETHER_API_KEY", + "perplexity": "PERPLEXITY_API_KEY", + "azure": "AZURE_OPENAI_API_KEY", + "aws": "AWS_ACCESS_KEY_ID", + "openrouter": "OPENROUTER_API_KEY", + "custom": "OPENAI_API_KEY", + "sarvam": "SARVAM_API_KEY", +} + + +def normalize_llm_model(provider: str, model: str) -> str: + """Normalize catalog model ids for provider-specific APIs.""" + provider_key = (provider or "").strip().lower() + if not model: + return model + if provider_key == "fireworks" and not model.startswith("accounts/"): + return f"accounts/fireworks/models/{model}" + if provider_key == "azure": + from app.services.ai.llm_service import _azure_deployment_name + + return _azure_deployment_name(model) + return model + + +def default_llm_model(provider: str) -> str: + return _DEFAULT_LLM_MODELS.get((provider or "").strip().lower(), "gpt-4.1") + + +def llm_env_key(provider: str) -> str: + return _ENV_KEYS.get((provider or "").strip().lower(), "OPENAI_API_KEY") + + +def _parse_aws_credentials(api_key: str) -> Dict[str, Any]: + """Parse AWS credential JSON or fall back to access key + env secret.""" + try: + parsed = json.loads(api_key) + if isinstance(parsed, dict): + access = ( + parsed.get("aws_access_key_id") + or parsed.get("access_key_id") + or parsed.get("aws_access_key") + ) + secret = ( + parsed.get("aws_secret_access_key") + or parsed.get("secret_access_key") + or parsed.get("aws_secret_key") + ) + return { + "aws_access_key": access, + "aws_secret_key": secret, + "aws_session_token": parsed.get("aws_session_token") + or parsed.get("session_token"), + "aws_region": parsed.get("aws_region") + or parsed.get("region") + or os.getenv("AWS_REGION", "us-east-1"), + } + except (json.JSONDecodeError, TypeError): + pass + return { + "aws_access_key": api_key, + "aws_secret_key": os.getenv("AWS_SECRET_ACCESS_KEY"), + "aws_region": os.getenv("AWS_REGION", "us-east-1"), + } + + +def get_llm_provider_registry(get_service: Callable[[str], Any]) -> Dict[str, Dict[str, Any]]: + """Build the LLM provider registry used by ``run_voice_bundle_fastapi``.""" + + def _openai_factory(api_key, model, params=None, base_url=None): + kwargs: Dict[str, Any] = {"api_key": api_key, "model": normalize_llm_model("openai", model)} + if params: + kwargs["params"] = params + if base_url: + kwargs["base_url"] = base_url + return get_service("OpenAILLMService")(**kwargs) + + registry: Dict[str, Dict[str, Any]] = {} + + for provider in sorted(LLM_VOICE_PROVIDER_KEYS): + env_key = llm_env_key(provider) + default_model = default_llm_model(provider) + + if provider == "openai": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, _f=_openai_factory: _f( + api_key, model, params + ), + } + elif provider == "google": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("GoogleLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "anthropic": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("AnthropicLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "fireworks": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("FireworksLLMService")( + api_key=api_key, + model=normalize_llm_model("fireworks", model), + **({"params": params} if params else {}), + ), + } + elif provider == "xai": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("GrokLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "mistral": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("MistralLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider in ("together", "meta"): + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("TogetherLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "perplexity": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("PerplexityLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "openrouter": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenRouterLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "aws": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("AWSBedrockLLMService")( + model=model, + params=params, + **_parse_aws_credentials(api_key), + ), + } + elif provider == "cohere": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenAILLMService")( + api_key=api_key, + model=model, + base_url="https://api.cohere.com/compatibility/v1", + **({"params": params} if params else {}), + ), + } + elif provider == "sarvam": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenAILLMService")( + api_key=api_key, + model=model, + base_url="https://api.sarvam.ai/v1", + **({"params": params} if params else {}), + ), + } + elif provider == "custom": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, base_url=None, _f=_openai_factory: _f( + api_key, model, params, base_url=base_url + ), + "supports_base_url": True, + } + elif provider == "azure": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + # Azure is instantiated with endpoint metadata in run_voice_bundle_fastapi. + "factory": lambda api_key, model, params=None, _f=_openai_factory: _f( + api_key, normalize_llm_model("azure", model), params + ), + } + + return registry + + +def resolve_voice_llm_base_url(db, organization_id, voice_bundle, llm_provider) -> Optional[str]: + """Resolve an OpenAI-compatible base URL for custom / gateway-routed LLM legs.""" + provider_key = ( + llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) + ).lower() + if provider_key != "custom": + return None + + from app.services.credentials import resolve_ai_provider + + ai_provider = resolve_ai_provider( + provider_key, + db, + organization_id, + credential_id=getattr(voice_bundle, "llm_credential_id", None), + ) + if not ai_provider: + return None + + base_url = getattr(ai_provider, "gateway_base_url", None) + if base_url and str(base_url).strip(): + return str(base_url).strip() + return None + + +def instantiate_llm_service( + provider: str, + *, + get_service: Callable[[str], Any], + api_key: str, + model: str, + params: Optional[Any] = None, + base_url: Optional[str] = None, +): + """Instantiate a streaming LLM service for the live voice pipeline.""" + registry = get_llm_provider_registry(get_service) + provider_key = (provider or "").strip().lower() + cfg = registry.get(provider_key) + if cfg is None: + supported = ", ".join(sorted(registry.keys())) + raise ValueError( + f"Unsupported LLM provider '{provider_key}'. Supported providers: {supported}" + ) + + factory = cfg["factory"] + if provider_key == "custom" or cfg.get("supports_base_url"): + return factory(api_key, model, params, base_url=base_url) + if base_url: + logger.debug( + "Ignoring llm base_url for provider '{}' (not OpenAI-compatible custom routing)", + provider_key, + ) + return factory(api_key, model, params) diff --git a/app/services/voice_agent/utils/audio_merge.py b/app/services/voice_agent/utils/audio_merge.py index eb19468d..6f50febc 100644 --- a/app/services/voice_agent/utils/audio_merge.py +++ b/app/services/voice_agent/utils/audio_merge.py @@ -16,6 +16,9 @@ def merge_and_upload_audio( organization_id: str = None, evaluator_id: str = None, result_id: str = None, + call_direction: str | None = None, + user_audio_frames: int | None = None, + bot_audio_frames: int | None = None, ): """ Merge user and bot telephony recordings with alignment analysis, upload mono WAV to S3, @@ -28,6 +31,13 @@ def merge_and_upload_audio( if os.path.exists(user_audio_path) and os.path.exists(bot_audio_path): user_size = os.path.getsize(user_audio_path) bot_size = os.path.getsize(bot_audio_path) + logger.info( + "Recording merge input user_bytes={} bot_bytes={} user_audio_frames={} bot_audio_frames={}", + user_size, + bot_size, + user_audio_frames, + bot_audio_frames, + ) if user_size > 100 and bot_size > 100: merged_fd, merged_path = tempfile.mkstemp(suffix=".wav") @@ -44,6 +54,7 @@ def merge_and_upload_audio( user_audio_path, bot_audio_path, output_path=merged_path, + call_direction=call_direction, ) except Exception as merge_exc: logger.error("Telephony aligned merge failed: {}", merge_exc, exc_info=True) @@ -81,6 +92,15 @@ def merge_and_upload_audio( evaluator_id, result_id, ) + elif bot_size > 100 and user_size <= 100: + logger.info("User track empty; uploading bot/outbound track only") + s3_key_result, duration_result = _upload_single_track( + bot_audio_path, + call_start_time, + organization_id, + evaluator_id, + result_id, + ) else: logger.warning("Recorded audio files are too small, skipping merge/upload.") elif os.path.exists(user_audio_path) and os.path.getsize(user_audio_path) > 100: diff --git a/app/services/voice_agent/utils/telephony_audio_align.py b/app/services/voice_agent/utils/telephony_audio_align.py index 8ece8e7e..f2904c7c 100644 --- a/app/services/voice_agent/utils/telephony_audio_align.py +++ b/app/services/voice_agent/utils/telephony_audio_align.py @@ -5,7 +5,7 @@ import wave from dataclasses import dataclass from enum import Enum -from typing import Tuple +from typing import Optional, Tuple import numpy as np from loguru import logger @@ -25,6 +25,7 @@ class TelephonyTrackAnalysis: bot_delay_samples: int correlation_peak: float correlation_lag_samples: int + leak_peak: float user_sample_rate: int user_duration_samples: int bot_duration_samples: int @@ -55,6 +56,33 @@ def _rms_envelope(samples: np.ndarray, frame_size: int) -> np.ndarray: return np.sqrt(np.mean(frames * frames, axis=1)) +def _bot_speech_leak_correlation( + user: np.ndarray, + bot: np.ndarray, + *, + sample_rate: int, +) -> float: + """Correlation on frames where the bot track is active (echo / bleed detection).""" + frame = max(1, sample_rate // 50) + user_env = _rms_envelope(user, frame) + bot_env = _rms_envelope(bot, frame) + min_len = min(len(user_env), len(bot_env)) + if min_len < 4: + return 0.0 + user_env = user_env[:min_len] + bot_env = bot_env[:min_len] + active = bot_env > max(float(bot_env.max()) * 0.2, 1.0) + if not np.any(active): + return 0.0 + active_user = user_env[active] - user_env[active].mean() + active_bot = bot_env[active] - bot_env[active].mean() + user_norm = np.linalg.norm(active_user) + bot_norm = np.linalg.norm(active_bot) + if user_norm < 1e-6 or bot_norm < 1e-6: + return 0.0 + return float(np.dot(active_user, active_bot) / (user_norm * bot_norm)) + + def estimate_bot_lag_samples( user: np.ndarray, bot: np.ndarray, @@ -62,7 +90,13 @@ def estimate_bot_lag_samples( sample_rate: int, max_lag_ms: int = 4000, ) -> Tuple[int, float]: - """Return (lag_samples, normalized_peak) where positive lag delays bot to align with user.""" + """Return (lag_samples, normalized_peak) of the envelope cross-correlation peak. + + DIAGNOSTIC ONLY -- do not use this lag to align the tracks. The user and bot + envelopes are anti-correlated in a turn-taking conversation, so the argmax is + the lag that best superimposes bot speech onto user speech, i.e. it maximises + overlap. It is meaningful only when both tracks carry the same signal (echo). + """ if len(user) < sample_rate // 10 or len(bot) < sample_rate // 20: return 0, 0.0 @@ -98,9 +132,15 @@ def analyze_dual_tracks( bot_samples: np.ndarray, *, sample_rate: int, + call_direction: Optional[str] = None, ) -> TelephonyTrackAnalysis: corr_lag, peak = estimate_bot_lag_samples(user_samples, bot_samples, sample_rate=sample_rate) - threshold = float(getattr(settings, "TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT", 0.35)) + + leak_peak = _bot_speech_leak_correlation( + user_samples, + bot_samples, + sample_rate=sample_rate, + ) if len(bot_samples) < sample_rate // 20: return TelephonyTrackAnalysis( @@ -108,33 +148,27 @@ def analyze_dual_tracks( bot_delay_samples=0, correlation_peak=peak, correlation_lag_samples=corr_lag, + leak_peak=leak_peak, user_sample_rate=sample_rate, user_duration_samples=len(user_samples), bot_duration_samples=len(bot_samples), reason="bot_track_too_short", ) - if peak >= threshold: - return TelephonyTrackAnalysis( - strategy=TelephonyMergeStrategy.USER_ONLY, - bot_delay_samples=0, - correlation_peak=peak, - correlation_lag_samples=corr_lag, - user_sample_rate=sample_rate, - user_duration_samples=len(user_samples), - bot_duration_samples=len(bot_samples), - reason="bot_energy_on_inbound_leg", - ) - - default_delay_ms = int(getattr(settings, "TELEPHONY_BOT_PLAYBACK_DELAY_MS", 400)) - default_delay_samples = int(sample_rate * default_delay_ms / 1000) - bot_delay = max(0, corr_lag + default_delay_samples) + # Both recorders now timestamp against the same wall clock at true playout + # time (the bot recorder sits downstream of the transport output and anchors + # each utterance to BotStartedSpeaking), so the tracks are already aligned. + # The only legitimate shift left is a measured residual carrier latency; + # corr_lag is deliberately NOT applied -- see estimate_bot_lag_samples. + residual_delay_ms = int(getattr(settings, "TELEPHONY_BOT_PLAYBACK_DELAY_MS", 0)) + bot_delay = max(0, int(sample_rate * residual_delay_ms / 1000)) return TelephonyTrackAnalysis( strategy=TelephonyMergeStrategy.ALIGNED_MIX, bot_delay_samples=bot_delay, correlation_peak=peak, correlation_lag_samples=corr_lag, + leak_peak=leak_peak, user_sample_rate=sample_rate, user_duration_samples=len(user_samples), bot_duration_samples=len(bot_samples), @@ -175,6 +209,7 @@ def merge_telephony_tracks_to_mono( bot_audio_path: str, *, output_path: str, + call_direction: Optional[str] = None, ) -> Tuple[TelephonyTrackAnalysis, float]: user, user_rate = read_wav_mono(user_audio_path) bot, bot_rate = read_wav_mono(bot_audio_path) @@ -193,20 +228,30 @@ def merge_telephony_tracks_to_mono( bot.astype(np.float32), ).astype(np.int16) - analysis = analyze_dual_tracks(user, bot, sample_rate=user_rate) + analysis = analyze_dual_tracks(user, bot, sample_rate=user_rate, call_direction=call_direction) logger.info( "Telephony merge analysis strategy={} reason={} corr_peak={:.3f} corr_lag_samples={} " - "bot_delay_samples={} user_samples={} bot_samples={}", + "leak_peak={:.3f} bot_delay_samples={} user_samples={} bot_samples={}", analysis.strategy.value, analysis.reason, analysis.correlation_peak, analysis.correlation_lag_samples, + analysis.leak_peak, analysis.bot_delay_samples, analysis.user_duration_samples, analysis.bot_duration_samples, ) + leak_threshold = float(getattr(settings, "TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT", 0.35)) + if analysis.leak_peak >= leak_threshold: + logger.warning( + "Telephony merge: inbound user track appears to already contain bot speech " + "(leak_peak={:.3f}). The carrier is likely mixing the agent leg back into the " + "inbound stream; summing the bot track will duplicate it.", + analysis.leak_peak, + ) + if analysis.strategy == TelephonyMergeStrategy.USER_ONLY: merged = user else: diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index edebc4d0..907d898a 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -12,15 +12,10 @@ import os import time -import wave -import uuid -import tempfile from dotenv import load_dotenv from loguru import logger -from app.services.storage.s3_service import s3_service - load_dotenv(override=True) @@ -39,7 +34,6 @@ def _get_core_imports(): from efficientai.processors.aggregators.llm_context import LLMContext from efficientai.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from efficientai.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor - from efficientai.processors.audio.audio_buffer_processor import AudioBufferProcessor from efficientai.runner.types import RunnerArguments from efficientai.runner.utils import create_transport from efficientai.serializers.protobuf import ProtobufFrameSerializer @@ -61,7 +55,6 @@ def _get_core_imports(): "RTVIConfig": RTVIConfig, "RTVIObserver": RTVIObserver, "RTVIProcessor": RTVIProcessor, - "AudioBufferProcessor": AudioBufferProcessor, "RunnerArguments": RunnerArguments, "create_transport": create_transport, "ProtobufFrameSerializer": ProtobufFrameSerializer, @@ -153,6 +146,30 @@ def _get_service(service_name: str): elif service_name == "AzureLLMService": from efficientai.services.azure.llm import AzureLLMService service_class = AzureLLMService + elif service_name == "AnthropicLLMService": + from efficientai.services.anthropic.llm import AnthropicLLMService + service_class = AnthropicLLMService + elif service_name == "FireworksLLMService": + from efficientai.services.fireworks.llm import FireworksLLMService + service_class = FireworksLLMService + elif service_name == "GrokLLMService": + from efficientai.services.grok.llm import GrokLLMService + service_class = GrokLLMService + elif service_name == "MistralLLMService": + from efficientai.services.mistral.llm import MistralLLMService + service_class = MistralLLMService + elif service_name == "TogetherLLMService": + from efficientai.services.together.llm import TogetherLLMService + service_class = TogetherLLMService + elif service_name == "PerplexityLLMService": + from efficientai.services.perplexity.llm import PerplexityLLMService + service_class = PerplexityLLMService + elif service_name == "OpenRouterLLMService": + from efficientai.services.openrouter.llm import OpenRouterLLMService + service_class = OpenRouterLLMService + elif service_name == "AWSBedrockLLMService": + from efficientai.services.aws.llm import AWSBedrockLLMService + service_class = AWSBedrockLLMService # Optional: Smart Turn Analyzer elif service_name == "LocalSmartTurnAnalyzerV3": @@ -347,40 +364,10 @@ def _instantiate_tts_service( def _get_llm_providers(): - """Get LLM provider registry with truly lazy-loaded service classes. - - Each provider's SDK is only loaded when that provider is actually used. - """ - return { - "openai": { - "env_key": "OPENAI_API_KEY", - "default_model": "gpt-4.1", - "factory": lambda api_key, model, params=None: _get_service("OpenAILLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - "google": { - "env_key": "GOOGLE_API_KEY", - "default_model": "gemini-2.5-flash", - "factory": lambda api_key, model, params=None: _get_service("GoogleLLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - "azure": { - "env_key": "AZURE_OPENAI_API_KEY", - "default_model": "gpt-4.1", - # Azure is instantiated with endpoint metadata in run_voice_bundle_fastapi. - "factory": lambda api_key, model, params=None: _get_service("OpenAILLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - } + """Get LLM provider registry with truly lazy-loaded service classes.""" + from app.services.voice_agent.llm_voice_providers import get_llm_provider_registry + + return get_llm_provider_registry(_get_service) DEFAULT_STT_PROVIDER = None @@ -522,14 +509,19 @@ async def run_voice_bundle_fastapi( tts_api_key: str | None = None, llm_api_key: str | None = None, llm_endpoint_url: str | None = None, + llm_base_url: str | None = None, serializer=None, telephony_mode: bool = False, call_short_id: str | None = None, silence_hangup_secs: float | None = None, + caller_speaks_first: bool = True, + caller_opening_text: str | None = None, + call_direction: str = "outbound", + persona_speaks_via_tts: bool = False, ): """ Run the STT+LLM+TTS voice bundle pipeline over a FastAPI WebSocket. - Uses AudioBufferProcessor for proper conversation audio recording. + Records dual-track WAV via AudioRecorder and merges/uploads after the call. """ # Lazy load all efficientai dependencies imports = _get_imports() @@ -542,9 +534,6 @@ async def run_voice_bundle_fastapi( duration_result = None transcript_text = None conversation_turns = [] - - # Storage for audio data from the buffer processor - recorded_audio_data = {"audio": None, "sample_rate": None, "num_channels": None} # Resolve STT provider config from the registry stt_provider_value = _resolve_provider(voice_bundle, "stt_provider", DEFAULT_STT_PROVIDER) @@ -607,6 +596,27 @@ async def run_voice_bundle_fastapi( transport_out_sample_rate = resolve_websocket_audio_out_sample_rate_hz( telephony_mode=telephony_mode, ) + ambient_mixer = None + ambient_input_processor = None + if persona is not None and telephony_mode: + from app.services.audio.ambient_telephony import resolve_ambient_for_telephony + from app.services.audio.ambient_input_processor import get_ambient_input_processor_class + + ambient_config = await resolve_ambient_for_telephony( + persona, + call_direction=call_direction, + input_sample_rate=transport_in_sample_rate, + output_sample_rate=transport_out_sample_rate, + persona_speaks_via_tts=persona_speaks_via_tts, + ) + ambient_mixer = ambient_config.output_mixer + if ambient_config.input_bed is not None: + AmbientInputProcessor = get_ambient_input_processor_class() + ambient_input_processor = AmbientInputProcessor(ambient_config.input_bed) + elif persona is not None: + from app.services.audio.ambient_catalog import resolve_ambient_mixer + + ambient_mixer = await resolve_ambient_mixer(persona, transport_out_sample_rate) ws_transport = imports["FastAPIWebsocketTransport"]( websocket=websocket_client, params=imports["FastAPIWebsocketParams"]( @@ -617,6 +627,7 @@ async def run_voice_bundle_fastapi( audio_out_sample_rate=transport_out_sample_rate, vad_analyzer=imports["SileroVADAnalyzer"](params=imports["VADParams"](stop_secs=0.2)), serializer=transport_serializer, + audio_out_mixer=ambient_mixer, ), ) @@ -696,7 +707,16 @@ async def run_voice_bundle_fastapi( params=llm_params, ) else: - llm = llm_cfg["factory"](api_key=llm_api_key, model=llm_model, params=llm_params) + from app.services.voice_agent.llm_voice_providers import instantiate_llm_service + + llm = instantiate_llm_service( + llm_provider_value, + get_service=_get_service, + api_key=llm_api_key, + model=llm_model, + params=llm_params, + base_url=llm_base_url, + ) # Build context with provided system instruction or a default base_instruction = ( @@ -715,77 +735,46 @@ async def run_voice_bundle_fastapi( "or special Unicode characters. Use only plain spoken words." ) - messages = [ - { - "role": "system", - "content": base_instruction, - }, - { - "role": "user", - "content": "Start by greeting the user warmly and introducing yourself based on the system instruction.", - }, - ] + messages = [{"role": "system", "content": base_instruction}] + if caller_speaks_first: + bootstrap = ( + f"Say your opening line now, in character, word for word if possible: " + f"\"{caller_opening_text.strip()}\"" + if caller_opening_text and caller_opening_text.strip() + else "Introduce yourself using your name from the system prompt, then begin the scenario naturally." + ) + messages.append({"role": "user", "content": bootstrap}) context = imports["LLMContext"](messages) context_aggregator = imports["LLMContextAggregatorPair"](context) - use_aligned_recorders = telephony_mode - user_recorder = None - bot_recorder = None - user_audio_path = None - bot_audio_path = None - recording_start_time = time.time() + from app.services.voice_agent.audio_recorder import get_audio_recorder_class + from app.services.voice_agent.telephony_recording_paths import telephony_recording_temp_path - if use_aligned_recorders: - from app.services.voice_agent.audio_recorder import get_audio_recorder_class - from app.services.voice_agent.telephony_recording_paths import telephony_recording_temp_path - - AudioRecorder = get_audio_recorder_class() - user_audio_path = telephony_recording_temp_path(suffix=".wav") - bot_audio_path = telephony_recording_temp_path(suffix=".wav") - user_recorder = AudioRecorder( - user_audio_path, - recording_start_time, - target_sample_rate=tts_sample_rate, - recorder_name="UserAudioRecorder", - alignment_mode="stream", - ) - bot_recorder = AudioRecorder( - bot_audio_path, - recording_start_time, - target_sample_rate=tts_sample_rate, - recorder_name="BotAudioRecorder", - alignment_mode="stream", - ) - audio_buffer_input = None - audio_buffer_output = None - input_audio_chunks = [] - output_audio_chunks = [] - else: - audio_buffer_input = imports["AudioBufferProcessor"]( - sample_rate=tts_sample_rate, - num_channels=1, - ) - audio_buffer_output = imports["AudioBufferProcessor"]( - sample_rate=tts_sample_rate, - num_channels=1, - ) - input_audio_chunks = [] - output_audio_chunks = [] - - @audio_buffer_input.event_handler("on_audio_data") - async def on_input_audio_data(buffer, audio, sample_rate, num_channels): - logger.debug(f"Input AudioBuffer captured {len(audio)} bytes") - if audio and len(audio) > 0: - input_audio_chunks.append(audio) - - @audio_buffer_output.event_handler("on_audio_data") - async def on_output_audio_data(buffer, audio, sample_rate, num_channels): - logger.debug(f"Output AudioBuffer captured {len(audio)} bytes") - if audio and len(audio) > 0: - output_audio_chunks.append(audio) - recorded_audio_data["sample_rate"] = sample_rate - recorded_audio_data["num_channels"] = num_channels + AudioRecorder = get_audio_recorder_class() + user_audio_path = telephony_recording_temp_path(suffix=".wav") + bot_audio_path = telephony_recording_temp_path(suffix=".wav") + recording_start_time = time.time() + recording_ambient_bed = None + if ambient_mixer is not None: + recording_ambient_bed = ambient_mixer.bed.clone() + user_recorder = AudioRecorder( + user_audio_path, + recording_start_time, + target_sample_rate=tts_sample_rate, + recorder_name="UserAudioRecorder", + alignment_mode="wall_clock", + capture="input", + ) + bot_recorder = AudioRecorder( + bot_audio_path, + recording_start_time, + target_sample_rate=tts_sample_rate, + recorder_name="BotAudioRecorder", + alignment_mode="playout", + capture="output", + ambient_bed=recording_ambient_bed, + ) pipeline_task_ref: list = [] @@ -809,16 +798,12 @@ async def on_silence_hangup(): audio_out_sample_rate=transport_out_sample_rate, ) - if use_aligned_recorders: - pipeline_processors = [ws_transport.input()] - if silence_hangup_processor: - pipeline_processors.append(silence_hangup_processor) - pipeline_processors.extend([user_recorder, stt]) - else: - pipeline_processors = [ws_transport.input()] - if silence_hangup_processor: - pipeline_processors.append(silence_hangup_processor) - pipeline_processors.extend([audio_buffer_input, stt]) + pipeline_processors = [ws_transport.input()] + if ambient_input_processor: + pipeline_processors.append(ambient_input_processor) + if silence_hangup_processor: + pipeline_processors.append(silence_hangup_processor) + pipeline_processors.extend([user_recorder, stt]) if telephony_mode and call_short_id: from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor @@ -857,8 +842,8 @@ async def on_silence_hangup(): if usage_recorder: pipeline_processors.append(usage_recorder) pipeline_processors.extend([ - bot_recorder if use_aligned_recorders else audio_buffer_output, ws_transport.output(), + bot_recorder, context_aggregator.assistant(), ]) @@ -874,10 +859,8 @@ async def on_silence_hangup(): @ws_transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info("Vobiz telephony client connected via WebSocket (voice bundle)") - if not use_aligned_recorders: - await audio_buffer_input.start_recording() - await audio_buffer_output.start_recording() - await task.queue_frames([imports["LLMRunFrame"]()]) + if caller_speaks_first: + await task.queue_frames([imports["LLMRunFrame"]()]) @ws_transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): @@ -887,13 +870,15 @@ async def on_client_disconnected(transport, client): rtvi = imports["RTVIProcessor"](config=imports["RTVIConfig"](config=[])) rtvi_processors = [ws_transport.input()] + if ambient_input_processor: + rtvi_processors.append(ambient_input_processor) if silence_hangup_processor: rtvi_processors.append(silence_hangup_processor) rtvi_processors.extend([ - audio_buffer_input, + user_recorder, + rtvi, stt, context_aggregator.user(), - rtvi, llm, tts, ]) @@ -907,8 +892,8 @@ async def on_client_disconnected(transport, client): if usage_recorder: rtvi_processors.append(usage_recorder) rtvi_processors.extend([ - audio_buffer_output, ws_transport.output(), + bot_recorder, context_aggregator.assistant(), ]) pipeline = imports["Pipeline"](rtvi_processors) @@ -923,10 +908,10 @@ async def on_client_disconnected(transport, client): @rtvi.event_handler("on_client_ready") async def on_client_ready(rtvi): await rtvi.set_bot_ready() - await audio_buffer_input.start_recording() - await audio_buffer_output.start_recording() - logger.info("AudioBufferProcessors started recording (input + output)") - await task.queue_frames([imports["LLMRunFrame"]()]) + if caller_speaks_first: + await task.queue_frames([imports["LLMRunFrame"]()]) + else: + logger.info("Caller waits for production agent — skipping initial LLM run") @ws_transport.event_handler("on_client_connected") async def on_client_connected(transport, client): @@ -978,66 +963,43 @@ async def on_client_disconnected(transport, client): conversation_turns = [] transcript_text = None - if use_aligned_recorders: - await user_recorder.cleanup() - await bot_recorder.cleanup() + await user_recorder.cleanup() + await bot_recorder.cleanup() - if telephony_mode and call_short_id: - try: - from app.workers.celery_app import finalize_telephony_recording_task - - finalize_telephony_recording_task.delay( - call_short_id=call_short_id, - user_audio_path=user_audio_path, - bot_audio_path=bot_audio_path, - call_start_time=call_start_time, - organization_id=organization_id, - evaluator_id=evaluator_id, - result_id=result_id, - conversation_turns=conversation_turns, - transcript_text=transcript_text, - duration=duration_result, - ) - logger.info( - "Queued finalize_telephony_recording for call_short_id={} " - "user_audio={} bot_audio={}", - call_short_id, - user_audio_path, - bot_audio_path, - ) - except Exception as celery_err: - logger.warning( - "Celery unavailable for post-call recording finalize ({}); running inline", - celery_err, - ) - from app.services.voice_agent.utils.audio_merge import merge_and_upload_audio - from app.database import SessionLocal - from app.services.telephony.call_recording_lifecycle import ( - persist_telephony_call_artifacts, - ) + if telephony_mode and call_short_id: + try: + from app.workers.celery_app import finalize_telephony_recording_task - s3_key_result, duration_result = merge_and_upload_audio( - user_audio_path=user_audio_path, - bot_audio_path=bot_audio_path, - call_start_time=call_start_time, - organization_id=organization_id, - evaluator_id=evaluator_id, - result_id=result_id, - ) - db = SessionLocal() - try: - persist_telephony_call_artifacts( - db, - call_short_id=call_short_id, - conversation_turns=conversation_turns, - transcript_text=transcript_text, - s3_key=s3_key_result, - duration=duration_result, - ) - finally: - db.close() - elif user_audio_path and bot_audio_path: + finalize_telephony_recording_task.delay( + call_short_id=call_short_id, + user_audio_path=user_audio_path, + bot_audio_path=bot_audio_path, + call_start_time=call_start_time, + organization_id=organization_id, + evaluator_id=evaluator_id, + result_id=result_id, + conversation_turns=conversation_turns, + transcript_text=transcript_text, + duration=duration_result, + call_direction=call_direction, + ) + logger.info( + "Queued finalize_telephony_recording for call_short_id={} " + "user_audio={} bot_audio={}", + call_short_id, + user_audio_path, + bot_audio_path, + ) + except Exception as celery_err: + logger.warning( + "Celery unavailable for post-call recording finalize ({}); running inline", + celery_err, + ) from app.services.voice_agent.utils.audio_merge import merge_and_upload_audio + from app.database import SessionLocal + from app.services.telephony.call_recording_lifecycle import ( + persist_telephony_call_artifacts, + ) s3_key_result, duration_result = merge_and_upload_audio( user_audio_path=user_audio_path, @@ -1046,58 +1008,10 @@ async def on_client_disconnected(transport, client): organization_id=organization_id, evaluator_id=evaluator_id, result_id=result_id, + call_direction=call_direction, + user_audio_frames=user_recorder.audio_frames_received, + bot_audio_frames=bot_recorder.audio_frames_received, ) - else: - await audio_buffer_input.stop_recording() - await audio_buffer_output.stop_recording() - logger.info("AudioBufferProcessors stopped recording") - - total_input_audio = b"".join(input_audio_chunks) if input_audio_chunks else b"" - total_output_audio = b"".join(output_audio_chunks) if output_audio_chunks else b"" - logger.info( - f"Input audio: {len(total_input_audio)} bytes, " - f"Output audio: {len(total_output_audio)} bytes" - ) - - if len(total_input_audio) > 100 or len(total_output_audio) > 100: - try: - import io - from efficientai.audio.utils import mix_audio - - sample_rate = recorded_audio_data.get("sample_rate") or tts_sample_rate - num_channels = 1 - mixed_audio = mix_audio(total_input_audio, total_output_audio) - logger.info(f"Mixed audio: {len(mixed_audio)} bytes") - - wav_buffer = io.BytesIO() - with wave.open(wav_buffer, "wb") as wf: - wf.setnchannels(num_channels) - wf.setsampwidth(2) - wf.setframerate(sample_rate) - wf.writeframes(mixed_audio) - - wav_buffer.seek(0) - file_content = wav_buffer.read() - file_id = uuid.uuid4() - meaningful_id = result_id if result_id else f"{int(time.time())}-{file_id.hex[:8]}" - s3_key_result = s3_service.upload_file( - file_content=file_content, - file_id=file_id, - file_format="wav", - organization_id=organization_id, - evaluator_id=evaluator_id, - meaningful_id=meaningful_id, - ) - logger.info(f"✅ Conversation audio uploaded to S3: {s3_key_result}") - except Exception as e: - logger.error(f"Failed to upload audio to S3: {e}", exc_info=True) - else: - logger.warning("No audio data captured or audio too small to upload") - - if telephony_mode and call_short_id: - from app.database import SessionLocal - from app.services.telephony.call_recording_lifecycle import persist_telephony_call_artifacts - db = SessionLocal() try: persist_telephony_call_artifacts( @@ -1110,6 +1024,20 @@ async def on_client_disconnected(transport, client): ) finally: db.close() + elif user_audio_path and bot_audio_path: + from app.services.voice_agent.utils.audio_merge import merge_and_upload_audio + + s3_key_result, duration_result = merge_and_upload_audio( + user_audio_path=user_audio_path, + bot_audio_path=bot_audio_path, + call_start_time=call_start_time, + organization_id=organization_id, + evaluator_id=evaluator_id, + result_id=result_id, + call_direction=call_direction if telephony_mode else None, + user_audio_frames=user_recorder.audio_frames_received, + bot_audio_frames=bot_recorder.audio_frames_received, + ) if telephony_mode and call_short_id: # region agent log @@ -1122,7 +1050,7 @@ async def on_client_disconnected(transport, client): "call_short_id": call_short_id, "conversation_turns_count": len(conversation_turns), "s3_key_set": bool(s3_key_result), - "aligned_recorders": use_aligned_recorders, + "aligned_recorders": True, }, "H4", ) diff --git a/app/services/webrtc_bridge/elevenlabs_ws_bridge.py b/app/services/webrtc_bridge/elevenlabs_ws_bridge.py index c5331e21..4a6fd3c4 100644 --- a/app/services/webrtc_bridge/elevenlabs_ws_bridge.py +++ b/app/services/webrtc_bridge/elevenlabs_ws_bridge.py @@ -10,14 +10,15 @@ Audio format: 16-bit PCM mono at 16kHz. Turn-taking strategy (ElevenLabs has NO explicit start/stop talking events): - - ``agent_response`` events deliver the agent's full text; we accumulate it. - - ``audio`` events indicate the agent is actively speaking; we track - the timestamp of the last audio event. - - A background silence-detector fires ``on_agent_stop_talking`` and then - ``on_transcript_received`` once no audio has arrived for SILENCE_THRESHOLD_S. - - This mirrors the Retell pattern (accumulate transcript, deliver on stop) - and the Vapi pattern (accumulate model-output, deliver on speech-update - stopped). + - ``agent_response`` events deliver the agent's full text; we accumulate them + locally and only forward to ``on_transcript_received`` once inbound audio + has been silent for ``SILENCE_THRESHOLD_S`` (text often arrives before TTS + finishes — never flush on text alone). + - ``audio`` events indicate the agent is actively speaking; inbound PCM is fed + to the turn gate's Silero VAD for logging/backup only (ElevenLabs uses the + bridge silence detector as the authoritative end-of-turn signal). + - The silence detector delivers the held transcript, then fires + ``on_agent_stop_talking`` so ProductionTurnGate flushes after ``stop_secs``. All callbacks that may take a long time (LLM + TTS) are dispatched via ``asyncio.create_task`` so they never block the WebSocket receive loop. @@ -44,7 +45,8 @@ BYTES_PER_SAMPLE = 2 # 16-bit audio # How long (seconds) without an audio event before we consider the agent done. -SILENCE_THRESHOLD_S = 0.7 +# ElevenLabs streams TTS in bursts; keep this above typical inter-chunk gaps. +SILENCE_THRESHOLD_S = 1.5 class ElevenLabsWSBridge: @@ -102,6 +104,8 @@ def __init__( self._silence_task: Optional[asyncio.Task] = None # Background task that simulates a live microphone by sending silence self._bg_silence_task: Optional[asyncio.Task] = None + # When True, an external ambient mic pump feeds continuous PCM instead. + self._external_mic_feed = False # Suppresses background silence while the test agent is actively sending audio self._user_is_sending = False @@ -157,10 +161,9 @@ async def connect_to_elevenlabs(self) -> bool: # Start background message receiver asyncio.create_task(self._receive_loop()) - # Start continuous silence stream to simulate a live microphone. - # Without this, ElevenLabs' VAD stalls between turns because it - # expects a constant audio input (just like a browser mic provides). - self._bg_silence_task = asyncio.create_task(self._background_silence_loop()) + # Start continuous silence stream unless an external mic feed is active. + if not self._external_mic_feed: + self._bg_silence_task = asyncio.create_task(self._background_silence_loop()) return True @@ -201,6 +204,14 @@ def mark_user_audio_done(self): """ self._user_is_sending = False + def suppress_background_silence(self): + """Stop the zero-silence mic loop when an ambient mic pump is active.""" + self._external_mic_feed = True + if self._bg_silence_task and not self._bg_silence_task.done(): + self._bg_silence_task.cancel() + self._bg_silence_task = None + logger.info("[ElevenLabsWS] Background silence stream suppressed (external mic feed active)") + async def send_silence(self, duration_ms: int = 500): """Send an explicit silence buffer (e.g. trailing silence after an utterance).""" if not self.is_connected or not self._ws: @@ -340,7 +351,6 @@ async def _handle_message(self, message: dict): text = event.get("agent_response", "").strip() if text: logger.info(f"[ElevenLabsWS] Agent response text: {text[:120]}...") - # Accumulate; delivery happens when silence is detected self._pending_agent_text = text # ---------------------------------------------------------- @@ -362,9 +372,7 @@ async def _handle_message(self, message: dict): # Agent was interrupted — consider the turn over immediately if self._agent_is_talking: self._agent_is_talking = False - if self.on_agent_stop_talking: - asyncio.create_task(self.on_agent_stop_talking()) - self._deliver_pending_transcript() + asyncio.create_task(self._deliver_turn_end()) # ---------------------------------------------------------- # ping — must pong to keep connection alive @@ -422,11 +430,11 @@ def _ensure_silence_detector(self): self._silence_task = asyncio.create_task(self._silence_detector()) async def _silence_detector(self): - """Background task that fires 'agent stopped talking' after silence. + """Background task that fires provider stop after inbound audio silence. - Polls ``_last_audio_ts``. When no new audio has arrived for - ``SILENCE_THRESHOLD_S`` seconds, the agent is considered done - speaking and we deliver the accumulated transcript. + Transcript delivery is owned by ProductionTurnGate (VAD on inbound PCM). + This task only notifies ``on_agent_stop_talking`` for the provider-stop + fallback path. """ try: while self.is_connected and self._agent_is_talking: @@ -439,22 +447,33 @@ async def _silence_detector(self): f"[ElevenLabsWS] Silence detected ({elapsed:.2f}s) — agent stopped speaking" ) self._agent_is_talking = False - if self.on_agent_stop_talking: - asyncio.create_task(self.on_agent_stop_talking()) - self._deliver_pending_transcript() + await self._deliver_turn_end() return except asyncio.CancelledError: pass except Exception as e: logger.error(f"[ElevenLabsWS] Silence detector error: {e}", exc_info=True) + async def _deliver_turn_end(self): + """Deliver held agent text, then signal provider stop for the turn gate.""" + text = self._pending_agent_text.strip() + self._pending_agent_text = "" + if text and self.on_transcript_received: + logger.info( + f"[ElevenLabsWS] Delivering agent transcript after silence " + f"({len(text)} chars): {text[:100]}..." + ) + await self.on_transcript_received(text) + if self.on_agent_stop_talking: + await self.on_agent_stop_talking() + def _deliver_pending_transcript(self): - """Fire-and-forget delivery of the accumulated agent text.""" + """Legacy helper — transcripts are held via on_transcript_received.""" text = self._pending_agent_text.strip() self._pending_agent_text = "" if text and self.on_transcript_received: logger.info( - f"[ElevenLabsWS] Delivering transcript ({len(text)} chars): {text[:100]}..." + f"[ElevenLabsWS] Delivering held transcript ({len(text)} chars): {text[:100]}..." ) asyncio.create_task(self.on_transcript_received(text)) elif not text: diff --git a/app/services/webrtc_bridge/production_turn_gate.py b/app/services/webrtc_bridge/production_turn_gate.py new file mode 100644 index 00000000..725ff0ca --- /dev/null +++ b/app/services/webrtc_bridge/production_turn_gate.py @@ -0,0 +1,266 @@ +"""VAD-gated turn-taking for synthetic evaluator WebRTC bridges. + +Holds production-agent transcripts until Silero VAD confirms inbound audio +has gone quiet, preventing the test agent from speaking over production TTS. +""" + +from __future__ import annotations + +import asyncio +import time +from typing import Awaitable, Callable, Optional, Protocol + +import numpy as np +from loguru import logger + +from app.services.audio.ambient_mixer import resample_mono_int16 + +TARGET_SAMPLE_RATE = 16_000 +# Silero requires 512 samples (16 kHz) or 256 samples (8 kHz) per frame. +SILERO_FRAME_BYTES = 512 * 2 +SILENCE_PUMP_INTERVAL_S = 0.032 + + +class VADAnalyzerProtocol(Protocol): + """Minimal VAD interface used by ProductionTurnGate.""" + + sample_rate: int + + async def analyze_audio(self, buffer: bytes): + ... + + +def _default_vad_analyzer(*, stop_secs: float) -> VADAnalyzerProtocol: + from efficientai.audio.vad.silero import SileroVADAnalyzer + from efficientai.audio.vad.vad_analyzer import VADParams, VADState + + _ = VADState # re-export guard for type checkers + return SileroVADAnalyzer( + sample_rate=TARGET_SAMPLE_RATE, + params=VADParams(start_secs=0.2, stop_secs=stop_secs), + ) + + +def _resample_pcm_bytes(pcm: bytes, source_rate: int) -> bytes: + if source_rate == TARGET_SAMPLE_RATE or not pcm: + return pcm + audio = np.frombuffer(pcm, dtype=np.int16) + resampled = resample_mono_int16(audio, source_rate, TARGET_SAMPLE_RATE) + return resampled.tobytes() + + +class ProductionTurnGate: + """Gate production-agent transcripts on inbound audio silence.""" + + def __init__( + self, + *, + on_flush: Callable[[str], Awaitable[None]], + stop_secs: float = 1.0, + late_text_wait_secs: float = 1.5, + flush_on_vad_quiet: bool = True, + vad_analyzer: Optional[VADAnalyzerProtocol] = None, + ) -> None: + self._on_flush = on_flush + self._stop_secs = stop_secs + self._late_text_wait_secs = late_text_wait_secs + self._flush_on_vad_quiet = flush_on_vad_quiet + self._vad = vad_analyzer or _default_vad_analyzer(stop_secs=stop_secs) + + self._held_transcript = "" + self._previous_vad_state = None + self._saw_speech_this_turn = False + self._outbound_active = False + self._flush_in_progress = False + + self._last_real_audio_ts = 0.0 + self._silence_pump_task: Optional[asyncio.Task] = None + self._provider_stop_task: Optional[asyncio.Task] = None + self._late_text_task: Optional[asyncio.Task] = None + self._started = False + + async def start(self) -> None: + if self._started: + return + self._started = True + if hasattr(self._vad, "set_sample_rate"): + self._vad.set_sample_rate(TARGET_SAMPLE_RATE) + self._silence_pump_task = asyncio.create_task( + self._silence_pump_loop(), + name="production-turn-gate-silence-pump", + ) + + async def stop(self) -> None: + self._started = False + for task in (self._silence_pump_task, self._provider_stop_task, self._late_text_task): + if task and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + self._silence_pump_task = None + self._provider_stop_task = None + self._late_text_task = None + + def set_outbound_active(self, active: bool) -> None: + self._outbound_active = active + + async def hold_transcript(self, text: str) -> None: + """Accumulate provider text; never invoke the test agent directly.""" + cleaned = (text or "").strip() + if not cleaned: + return + + if self._held_transcript: + if cleaned not in self._held_transcript: + self._held_transcript = f"{self._held_transcript} {cleaned}".strip() + else: + self._held_transcript = cleaned + + logger.debug( + f"[TurnGate] Held transcript ({len(self._held_transcript)} chars): " + f"{self._held_transcript[:80]}..." + ) + + if self._late_text_task and not self._late_text_task.done(): + self._late_text_task.cancel() + self._late_text_task = None + if self._flush_on_vad_quiet: + await self._try_flush("late-text") + + async def ingest_audio(self, pcm: bytes, *, source_rate: int = TARGET_SAMPLE_RATE) -> None: + if not pcm: + return + self._last_real_audio_ts = time.monotonic() + prepared = _resample_pcm_bytes(pcm, source_rate) + await self._analyze_and_update(prepared) + + async def on_production_start_talking(self) -> None: + """Reset per-turn counters when the provider signals speech start.""" + self._cancel_provider_stop_fallback() + self._saw_speech_this_turn = False + + async def on_provider_stop_talking(self) -> None: + """Provider thinks the agent stopped; use as fallback if VAD never saw speech.""" + if self._provider_stop_task and not self._provider_stop_task.done(): + return + self._provider_stop_task = asyncio.create_task( + self._provider_stop_fallback(), + name="production-turn-gate-provider-stop", + ) + + async def _silence_pump_loop(self) -> None: + """Feed zero frames when providers stop sending PCM mid-utterance (ElevenLabs).""" + silence_chunk = b"\x00" * SILERO_FRAME_BYTES + try: + while self._started: + await asyncio.sleep(SILENCE_PUMP_INTERVAL_S) + if self._last_real_audio_ts == 0: + continue + elapsed = time.monotonic() - self._last_real_audio_ts + if elapsed >= SILENCE_PUMP_INTERVAL_S: + await self._analyze_and_update(silence_chunk) + except asyncio.CancelledError: + pass + + async def _analyze_and_update(self, pcm: bytes) -> None: + from efficientai.audio.vad.vad_analyzer import VADState + + if self._previous_vad_state is None: + self._previous_vad_state = VADState.QUIET + + # Feed the analyzer in Silero frame increments. + offset = 0 + current_state = self._previous_vad_state + while offset < len(pcm): + chunk = pcm[offset : offset + SILERO_FRAME_BYTES] + offset += SILERO_FRAME_BYTES + if len(chunk) < SILERO_FRAME_BYTES: + chunk = chunk + b"\x00" * (SILERO_FRAME_BYTES - len(chunk)) + current_state = await self._vad.analyze_audio(chunk) + + previous = self._previous_vad_state + self._previous_vad_state = current_state + + if current_state == VADState.SPEAKING: + self._saw_speech_this_turn = True + self._cancel_late_text_wait() + + if previous in (VADState.SPEAKING, VADState.STOPPING) and current_state == VADState.QUIET: + await self._on_vad_quiet() + + async def _on_vad_quiet(self) -> None: + if self._flush_on_vad_quiet and self._held_transcript.strip(): + await self._try_flush("vad-quiet") + elif self._flush_on_vad_quiet: + self._schedule_late_text_wait() + else: + logger.debug("[TurnGate] VAD quiet ignored — waiting for provider stop signal") + + def _schedule_late_text_wait(self) -> None: + if self._late_text_task and not self._late_text_task.done(): + return + + async def _wait() -> None: + try: + await asyncio.sleep(self._late_text_wait_secs) + if self._held_transcript.strip(): + await self._try_flush("late-text-timeout") + else: + logger.debug("[TurnGate] VAD quiet with no held text — skipping empty turn") + self._reset_turn_state() + except asyncio.CancelledError: + pass + + self._late_text_task = asyncio.create_task(_wait(), name="production-turn-gate-late-text") + + def _cancel_late_text_wait(self) -> None: + if self._late_text_task and not self._late_text_task.done(): + self._late_text_task.cancel() + self._late_text_task = None + + def _cancel_provider_stop_fallback(self) -> None: + if self._provider_stop_task and not self._provider_stop_task.done(): + self._provider_stop_task.cancel() + self._provider_stop_task = None + + async def _provider_stop_fallback(self) -> None: + """Flush after provider stop once trailing TTS has had time to finish. + + Vapi/Daily may keep delivering PCM frames after the agent stops, so VAD + never reaches QUIET. Provider stop is the reliable end-of-turn signal + for those platforms when we already hold a transcript. + """ + try: + await asyncio.sleep(self._stop_secs) + if not self._held_transcript.strip(): + return + await self._try_flush("provider-stop-fallback") + except asyncio.CancelledError: + pass + + async def _try_flush(self, reason: str) -> None: + if self._flush_in_progress or self._outbound_active: + logger.debug(f"[TurnGate] Deferring flush ({reason}) — outbound active or flush in progress") + return + + text = self._held_transcript.strip() + if not text: + return + + self._flush_in_progress = True + self._held_transcript = "" + self._cancel_late_text_wait() + self._cancel_provider_stop_fallback() + + logger.info(f"[TurnGate] Flushing held transcript ({reason}, {len(text)} chars)") + try: + await self._on_flush(text) + finally: + self._flush_in_progress = False + self._reset_turn_state() + + def _reset_turn_state(self) -> None: + self._saw_speech_this_turn = False + self._last_real_audio_ts = 0.0 diff --git a/app/services/webrtc_bridge/test_agent_processor.py b/app/services/webrtc_bridge/test_agent_processor.py index 990fa6bd..6272c867 100644 --- a/app/services/webrtc_bridge/test_agent_processor.py +++ b/app/services/webrtc_bridge/test_agent_processor.py @@ -63,6 +63,7 @@ class TestAgentConfig: scenario_description: str = "General inquiry call" scenario_goal: str = "Have a conversation and evaluate the agent" first_message: str = "Hello, I'm calling because I need some help." + caller_speaks_first: bool = True # Context about the voice AI agent being tested agent_name: str = "Voice AI Agent" @@ -211,11 +212,16 @@ async def generate_first_message(self) -> Optional[bytes]: Generate the first message to start the conversation. Returns: - Audio bytes of the first message, or None if failed + Audio bytes of the first message, or None if failed or caller waits """ + if not self.config.caller_speaks_first: + logger.info("[TestAgent] Caller configured to wait for production agent greeting") + return None + try: - # Use configured first message or generate one - first_text = self.config.first_message + first_text = (self.config.first_message or "").strip() + if not first_text: + return None # Add to conversation history self.conversation_history.append({ diff --git a/app/services/webrtc_bridge/vapi_webrtc_bridge.py b/app/services/webrtc_bridge/vapi_webrtc_bridge.py index 872a0bdb..f7e19997 100644 --- a/app/services/webrtc_bridge/vapi_webrtc_bridge.py +++ b/app/services/webrtc_bridge/vapi_webrtc_bridge.py @@ -82,9 +82,9 @@ def __init__( self._mic_device = None self._speaker_device = None - # Audio queues for async bridging + # Audio queue for async bridging (test agent -> Vapi mic) self._outgoing_audio_queue = queue.Queue() - self._incoming_audio_queue = asyncio.Queue() + self._event_loop: Optional[asyncio.AbstractEventLoop] = None # Recording self.recording_enabled = False @@ -112,6 +112,8 @@ def __init__( self._agent_is_talking = False self._current_model_output = "" # accumulate model-output tokens self._current_assistant_transcript = "" # accumulate role=assistant transcripts + self._buffered_transcript = "" + self._buffered_provider_stop = False # Background threads self._send_audio_thread: Optional[threading.Thread] = None @@ -163,6 +165,7 @@ async def connect_to_vapi(self) -> bool: # Create event handler with reference to the current event loop # This allows async callbacks to be executed thread-safely loop = asyncio.get_event_loop() + self._event_loop = loop event_handler = VapiDailyEventHandler(self, loop) # Create call client @@ -260,6 +263,30 @@ def _start_audio_threads(self): self._receive_audio_thread.start() logger.info("[VapiWebRTC] Audio processing threads started") + + def _forward_incoming_audio(self, buffer: bytes) -> None: + """Forward Daily speaker PCM to the asyncio turn-gate callback.""" + if not buffer or not self.on_audio_received: + return + loop = self._event_loop + if loop is None or not loop.is_running(): + return + asyncio.run_coroutine_threadsafe(self.on_audio_received(buffer), loop) + + async def replay_buffered_turn_events(self) -> None: + """Deliver turn events that arrived before callbacks were wired.""" + if self._buffered_transcript.strip() and self.on_transcript_received: + text = self._buffered_transcript.strip() + self._buffered_transcript = "" + logger.info( + f"[VapiWebRTC] Replaying buffered transcript ({len(text)} chars): {text[:100]}..." + ) + await self.on_transcript_received(text) + + if self._buffered_provider_stop and self.on_agent_stop_talking: + self._buffered_provider_stop = False + logger.info("[VapiWebRTC] Replaying buffered provider stop signal") + await self.on_agent_stop_talking() def _send_audio_loop(self): """Background thread to send audio to Vapi.""" @@ -311,11 +338,7 @@ def _receive_audio_loop(self): if self.recording_enabled: self._recording_buffer.append(buffer) - # Queue for async processing - try: - self._incoming_audio_queue.put_nowait(buffer) - except asyncio.QueueFull: - pass + self._forward_incoming_audio(buffer) except Exception as e: if not self._quit_event.is_set(): logger.error(f"[VapiWebRTC] Error receiving audio: {e}") @@ -543,11 +566,27 @@ def _deliver_accumulated_transcript(self): self.bridge._current_assistant_transcript = "" self.bridge._current_model_output = "" - if transcript and self.bridge.on_transcript_received: - logger.info(f"[VapiWebRTC] Delivering accumulated assistant transcript ({len(transcript)} chars): {transcript[:100]}...") - self._run_async(self.bridge.on_transcript_received(transcript)) - elif not transcript: + if not transcript: logger.debug("[VapiWebRTC] No accumulated transcript to deliver on agent stop") + return + + if self.bridge.on_transcript_received: + logger.info( + f"[VapiWebRTC] Delivering accumulated assistant transcript " + f"({len(transcript)} chars): {transcript[:100]}..." + ) + self._run_async(self.bridge.on_transcript_received(transcript)) + else: + if self.bridge._buffered_transcript: + self.bridge._buffered_transcript = ( + f"{self.bridge._buffered_transcript} {transcript}".strip() + ) + else: + self.bridge._buffered_transcript = transcript + logger.info( + f"[VapiWebRTC] Buffered transcript until callback wired " + f"({len(transcript)} chars)" + ) def on_app_message(self, message, sender): """ @@ -616,11 +655,11 @@ def on_app_message(self, message, sender): elif status == "stopped": logger.info(f"[VapiWebRTC] Vapi agent stopped speaking (turn {turn})") self.bridge._agent_is_talking = False - # Signal stop BEFORE delivering transcript, so - # test_agent.agent_is_talking is False when transcript arrives if self.bridge.on_agent_stop_talking: self._run_async(self.bridge.on_agent_stop_talking()) - # Deliver accumulated transcript now that the agent finished speaking + else: + self.bridge._buffered_provider_stop = True + logger.debug("[VapiWebRTC] Buffered provider stop until callback wired") self._deliver_accumulated_transcript() else: logger.debug(f"[VapiWebRTC] speech-update role=assistant status={status}") @@ -650,6 +689,8 @@ def on_app_message(self, message, sender): self.bridge._agent_is_talking = False if self.bridge.on_agent_stop_talking: self._run_async(self.bridge.on_agent_stop_talking()) + else: + self.bridge._buffered_provider_stop = True self._deliver_accumulated_transcript() elif event_type in ["call_ended", "call-ended", "ended"]: diff --git a/app/workers/config.py b/app/workers/config.py index c6862c92..8a7e837b 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -87,6 +87,14 @@ def _usage_flush_beat_seconds() -> float: return 120.0 +def _cron_dispatch_beat_seconds() -> float: + raw = os.environ.get("CRON_DISPATCH_INTERVAL_SECONDS", "30") + try: + return max(10.0, float(raw)) + except (TypeError, ValueError): + return 30.0 + + def _platform_beat_schedule() -> dict: """Periodic platform tasks — run from dedicated ``celery beat`` (single replica).""" return { @@ -94,6 +102,10 @@ def _platform_beat_schedule() -> dict: "task": "flush_usage_counters", "schedule": _usage_flush_beat_seconds(), }, + "dispatch-cron-jobs": { + "task": "dispatch_cron_jobs", + "schedule": _cron_dispatch_beat_seconds(), + }, "evaluate-alerts": { "task": "evaluate_alerts", "schedule": crontab(minute="*/5"), @@ -169,6 +181,7 @@ def _platform_beat_schedule() -> dict: "generate_evaluation_tldr_insights": {"queue": "evaluations"}, "generate_evaluation_user_insights": {"queue": "evaluations"}, "generate_evaluation_metric_clusters": {"queue": "evaluations"}, + "generate_evaluator_result_metric_clusters": {"queue": "evaluations"}, "generate_evaluation_prompt_improvements": {"queue": "evaluations"}, "evaluate_studio_run_item": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, @@ -178,8 +191,6 @@ def _platform_beat_schedule() -> dict: "evaluate_alerts": {"queue": PLATFORM_WORKER_QUEUE}, "refresh_fx_rates": {"queue": PLATFORM_WORKER_QUEUE}, "prune_oss_usage_history": {"queue": PLATFORM_WORKER_QUEUE}, - "dispatch_cron_jobs": {"queue": "celery"}, + "dispatch_cron_jobs": {"queue": USAGE_WORKER_QUEUE}, "run_cron_evaluator_job": {"queue": "celery"}, } - -from app.workers import cron_bootstrap # noqa: F401,E402 diff --git a/app/workers/cron_bootstrap.py b/app/workers/cron_bootstrap.py deleted file mode 100644 index b0c1d62a..00000000 --- a/app/workers/cron_bootstrap.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Bootstrap cron dispatcher on worker startup.""" - -from __future__ import annotations - -from celery.signals import worker_ready -from loguru import logger - - -@worker_ready.connect -def _bootstrap_cron_dispatcher(sender, **kwargs): - try: - from app.services.cron.dispatcher_lock import try_acquire_dispatcher_leader - - if not try_acquire_dispatcher_leader(): - return - from app.workers.tasks.dispatch_cron_jobs import dispatch_cron_jobs_task - - dispatch_cron_jobs_task.apply_async(countdown=5) - logger.info("Cron dispatcher bootstrap enqueued") - except Exception as exc: - logger.warning("Cron dispatcher bootstrap skipped: {}", exc) diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index a8542175..24505237 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -17,6 +17,7 @@ from . import generate_evaluation_user_insights from . import generate_evaluation_tldr_insights from . import generate_evaluation_metric_clusters +from . import generate_evaluator_result_metric_clusters from . import generate_evaluation_prompt_improvements from . import agent_flowchart_jobs from . import initiate_vobiz_outbound @@ -94,6 +95,9 @@ generate_evaluation_metric_clusters_task = ( generate_evaluation_metric_clusters.generate_evaluation_metric_clusters_task ) +generate_evaluator_result_metric_clusters_task = ( + generate_evaluator_result_metric_clusters.generate_evaluator_result_metric_clusters_task +) generate_evaluation_prompt_improvements_task = ( generate_evaluation_prompt_improvements.generate_evaluation_prompt_improvements_task ) diff --git a/app/workers/tasks/dispatch_cron_jobs.py b/app/workers/tasks/dispatch_cron_jobs.py index 3c658c66..a7af9c42 100644 --- a/app/workers/tasks/dispatch_cron_jobs.py +++ b/app/workers/tasks/dispatch_cron_jobs.py @@ -1,28 +1,17 @@ -"""Self-scheduling cron job dispatcher (replaces Celery Beat for platform jobs).""" +"""Dispatch due org cron jobs to Celery workers (Beat-driven, no self-scheduling).""" from __future__ import annotations -import os - from loguru import logger from app.database import SessionLocal from app.workers.config import celery_app -def _dispatch_interval_seconds() -> int: - raw = os.environ.get("CRON_DISPATCH_INTERVAL_SECONDS", "30") - try: - return max(10, int(raw)) - except (TypeError, ValueError): - return 30 - - @celery_app.task(name="dispatch_cron_jobs") def dispatch_cron_jobs_task() -> dict: from app.services.cron.dispatcher_lock import ( acquire_dispatcher_run_lock, - refresh_dispatcher_leader, release_dispatcher_run_lock, ) from app.services.cron.job_dispatch import ( @@ -31,11 +20,7 @@ def dispatch_cron_jobs_task() -> dict: list_due_cron_jobs, ) - interval = _dispatch_interval_seconds() - refresh_dispatcher_leader() - if not acquire_dispatcher_run_lock(): - dispatch_cron_jobs_task.apply_async(countdown=interval) return {"skipped": "locked"} dispatched: list[dict] = [] @@ -54,5 +39,4 @@ def dispatch_cron_jobs_task() -> dict: db.close() release_dispatcher_run_lock() - dispatch_cron_jobs_task.apply_async(countdown=interval) return {"dispatched": len(dispatched), "jobs": dispatched} diff --git a/app/workers/tasks/finalize_telephony_recording.py b/app/workers/tasks/finalize_telephony_recording.py index d90d7f3b..f695ffbf 100644 --- a/app/workers/tasks/finalize_telephony_recording.py +++ b/app/workers/tasks/finalize_telephony_recording.py @@ -26,6 +26,7 @@ def finalize_telephony_recording_task( conversation_turns: Optional[List[Dict[str, Any]]] = None, transcript_text: Optional[str] = None, duration: Optional[float] = None, + call_direction: Optional[str] = None, ) -> dict: """Merge dual-track WAVs, upload to S3, persist CallRecording, queue evaluator.""" try: @@ -36,6 +37,7 @@ def finalize_telephony_recording_task( organization_id=organization_id, evaluator_id=evaluator_id, result_id=result_id, + call_direction=call_direction, ) effective_duration = merged_duration if merged_duration is not None else duration diff --git a/app/workers/tasks/generate_evaluator_result_metric_clusters.py b/app/workers/tasks/generate_evaluator_result_metric_clusters.py new file mode 100644 index 00000000..16cb6931 --- /dev/null +++ b/app/workers/tasks/generate_evaluator_result_metric_clusters.py @@ -0,0 +1,144 @@ +"""Celery task: failure clustering for filtered evaluator-result scopes.""" + +from __future__ import annotations + +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm.attributes import flag_modified + +from app.database import SessionLocal +from app.models.database import EvaluatorResultClusterJob +from app.services.call_import_metric_clusters import ( + generate_metric_clusters_for_source_rows, + metric_clusters_raw_is_cancelled, + metric_clusters_state_to_db, +) +from app.services.evaluators.evaluator_result_metric_clusters import ( + clustering_context_for_job, + load_completed_evaluator_results, +) +from app.services.metric_cluster_rows import ( + evaluator_result_to_cluster_row, + filter_cluster_rows_by_ids, +) +from app.workers.config import celery_app + + +@celery_app.task(name="generate_evaluator_result_metric_clusters", bind=True, max_retries=0) +def generate_evaluator_result_metric_clusters_task( + self, + cluster_job_id: str, + *, + provider: str | None = None, + model: str | None = None, + credential_id: str | None = None, + max_llm_calls: int | None = None, + evaluation_row_ids: list[str] | None = None, +): + from app.services.ai.llm_resolver import get_llm_provider_and_model + + db = SessionLocal() + job: EvaluatorResultClusterJob | None = None + try: + job = ( + db.query(EvaluatorResultClusterJob) + .filter(EvaluatorResultClusterJob.id == UUID(cluster_job_id)) + .first() + ) + if job is None: + logger.error("Evaluator metric clusters: job {} not found", cluster_job_id) + return + + if metric_clusters_raw_is_cancelled(job.metric_clusters): + logger.info( + "Evaluator metric clusters: job {} already cancelled, skipping", + cluster_job_id, + ) + return + + provider_enum, model_str = get_llm_provider_and_model( + job.organization_id, + db, + provider, + model, + UUID(credential_id) if credential_id else None, + ) + + metrics, aggregates, policies, _source, child_names_by_parent, source_rows, completed_count = ( + clustering_context_for_job(db, job) + ) + if evaluation_row_ids: + source_rows = filter_cluster_rows_by_ids( + source_rows, + [UUID(rid) for rid in evaluation_row_ids], + ) + else: + results = load_completed_evaluator_results( + db, + organization_id=job.organization_id, + workspace_id=job.workspace_id, + agent_id=job.agent_id, + suite_id=job.suite_id, + scenario_id=job.scenario_id, + ) + source_rows = [evaluator_result_to_cluster_row(r) for r in results] + + def _reload_cancelled() -> bool: + db.expire(job, ["metric_clusters"]) + db.refresh(job) + return metric_clusters_raw_is_cancelled(job.metric_clusters) + + def on_progress(completed: int, total: int) -> None: + if _reload_cancelled(): + return + job.metric_clusters = { + **(job.metric_clusters or {}), + "status": "running", + "progress": { + "completed_llm_calls": completed, + "total_llm_calls": total, + }, + } + flag_modified(job, "metric_clusters") + db.commit() + + state = generate_metric_clusters_for_source_rows( + db, + job_key=job.id, + metric_clusters_raw=job.metric_clusters, + completed_row_count=completed_count, + organization_id=job.organization_id, + provider=provider_enum, + model=model_str, + source_rows=source_rows, + metrics=metrics, + policies=policies, + on_progress=on_progress, + max_llm_calls=max_llm_calls, + is_cancelled=_reload_cancelled, + ) + + job.metric_clusters = metric_clusters_state_to_db(state) + job.celery_task_id = None + flag_modified(job, "metric_clusters") + db.commit() + logger.info( + "Evaluator metric clusters completed for job {} status={}", + cluster_job_id, + state.status, + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Evaluator metric clusters failed for job {}: {}", cluster_job_id, exc) + if job is not None: + prior = job.metric_clusters if isinstance(job.metric_clusters, dict) else {} + job.metric_clusters = { + **prior, + "status": "failed", + "error_message": str(exc)[:500], + "celery_task_id": None, + } + flag_modified(job, "metric_clusters") + db.commit() + finally: + db.close() diff --git a/docker-compose.yml b/docker-compose.yml index 1d6f0d0b..5f1a9239 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -190,8 +190,9 @@ services: # ``imports`` (recording fetch) before ``diarization`` (manual diarise), # then ``eval-control`` (cancel/retry/materialize), then ``evaluations`` # (fair dispatch + LLM scoring). - # The default ``worker`` service handles ``celery`` (evaluator cron dispatch only) + # The default ``worker`` service handles ``celery`` (evaluator cron runs) # and ``audio-metrics`` (Praat/UTMOS audio metric tasks). + # Evaluator cron dispatch runs on ``worker-usage`` (Beat schedule). # Platform schedules run via ``beat`` (scheduler + ``platform`` queue worker). worker-imports: image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest} diff --git a/docs-fumadocs/content/docs/products/evaluators.mdx b/docs-fumadocs/content/docs/products/evaluators.mdx index 06aa8faf..6e968cd6 100644 --- a/docs-fumadocs/content/docs/products/evaluators.mdx +++ b/docs-fumadocs/content/docs/products/evaluators.mdx @@ -8,9 +8,9 @@ sidebar_position: 4 ## What is an Evaluator Suite? -An **Evaluator Suite** groups one **Agent**, one **Persona**, and **multiple Scenarios** into test combinations. +An **Evaluator Suite** groups one **Agent**, one or more **Personas**, and **multiple Scenarios** into test combinations. -Each scenario becomes one combination (1:1:N). You can: +Each persona × scenario pair becomes one combination (1:M:N). You can: - Choose **metrics** to score at the suite level (optional — defaults to all enabled agent metrics) - **Outbound / web**: run every combination **X times** (total runs = N × X) @@ -20,8 +20,8 @@ Each scenario becomes one combination (1:1:N). You can: ## Creating a Suite (4-step wizard) -1. **Agent & Persona** — pick one agent and one TTS-compatible persona -2. **Scenarios** — multi-select scenarios (each becomes a combination) +1. **Agent & Personas** — pick one agent and one or more TTS-compatible personas +2. **Scenarios** — multi-select scenarios (each persona × scenario becomes a combination) 3. **Metrics** — optional metric picker 4. **Review** — name, tags, default runs per combination @@ -42,6 +42,9 @@ Each scenario becomes one combination (1:1:N). You can: | Method | Path | Purpose | |---|---|---| | POST | `/api/v1/evaluator-suites` | Create suite + combinations | +| PUT | `/api/v1/evaluator-suites/{id}/personas` | Replace persona set on a suite | +| POST | `/api/v1/evaluator-suites/{id}/personas` | Add personas to a suite | +| DELETE | `/api/v1/evaluator-suites/{id}/personas/{persona_id}` | Remove a persona from a suite | | GET | `/api/v1/evaluator-suites` | List suites | | POST | `/api/v1/evaluator-suites/{id}/run` | Batch run (outbound/web) | | POST | `/api/v1/evaluator-suites/{id}/activate` | Set active inbound suite for the agent | diff --git a/docs/telephony-media.md b/docs/telephony-media.md index b8bc2ce9..1af5d4cf 100644 --- a/docs/telephony-media.md +++ b/docs/telephony-media.md @@ -69,7 +69,9 @@ One listen port accepts **many concurrent WebSocket connections** (one per live ## Recording artifacts -1. **Pipeline capture (default)** — Dual-track WAV from the STT/TTS (or Gemini) pipeline on the Vobiz media WebSocket: recorders use **stream timeline** (sequential frames, no wall-clock padding). Celery merges with lag detection + optional bot playback delay (`TELEPHONY_BOT_PLAYBACK_DELAY_MS`, default 400ms). If inbound audio already contains agent energy, merge uploads **inbound-only** to avoid double-counting. Otherwise tracks are delay-aligned and summed in Python (NumPy). Stored on `CallRecording.call_data.recording_s3_key` via `finalize_telephony_recording`. +1. **Pipeline capture (default)** — Dual-track WAV from the STT/TTS (or Gemini) pipeline on the Vobiz media WebSocket. Both recorders share one `start_time`; the **user** recorder uses `wall_clock` alignment (telephony delivers frames in real time, so the wall clock is accurate). The **bot** recorder uses `playout` alignment and sits **downstream of `ws_transport.output()`**: TTS reaches the transport far faster than real time and is drained at ~2x by `_write_audio_sleep()`, so capturing upstream stamped agent speech at *generation* time — earlier than it was heard — and per-frame wall-clock padding then compressed the track, making agent and user speech overlap in the merged file. In `playout` mode each utterance is anchored to the `BotStartedSpeakingFrame` the transport emits as it drains, then written contiguously, so true playout duration and real silences are preserved. + + Because both tracks now sit on a common playout timeline, Celery sums them with **no alignment search**. `TELEPHONY_BOT_PLAYBACK_DELAY_MS` (default `0`) is a residual carrier-latency trim only — set it from measurement, not by guess. Envelope cross-correlation (`estimate_bot_lag_samples`) is retained for **diagnostics only**: user and bot envelopes are anti-correlated when speakers take turns, so its argmax maximises overlap and must never drive alignment. If the inbound leg already contains agent energy (`leak_peak` ≥ `TELEPHONY_MERGE_CORRELATION_DOUBLE_COUNT`), the merge logs a warning — the carrier is mixing the agent leg back in and summing would duplicate it. Stored on `CallRecording.call_data.recording_s3_key` via `finalize_telephony_recording`. 2. **Carrier session recording (optional)** — Set `vobiz.carrier_session_recording: true` to add Vobiz `` on answer XML and ingest MP3 from `recording-ready` (allowlisted `vobiz.ai` hosts). When disabled, answer XML is stream-only and audio comes from Celery `finalize_telephony_recording` only. diff --git a/env.example b/env.example index 40b2f1a5..f5fcb27f 100644 --- a/env.example +++ b/env.example @@ -71,8 +71,8 @@ AUTH_LOCAL_ALLOW_SIGNUP=true # ----------------------------------------------------------------------------- # Usage cost flush (Celery Beat + worker-usage queue) -# Platform schedules: Beat enqueues flush/alerts/FX/prune; worker-usage runs flush tasks. -# Evaluator crons: default worker dispatch_cron_jobs (CRON_DISPATCH_INTERVAL_SECONDS). +# Beat enqueues flush + evaluator cron dispatch; worker-usage runs those tasks. +# Platform schedules (alerts, FX, prune) run on the co-located platform worker. # See README "Usage Pricing Ops". Copy to .env for Docker Compose / local dev. # ----------------------------------------------------------------------------- USAGE_FLUSH_BUCKET_BATCH_SIZE=500 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b62b6e72..6ded0bbb 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -48,11 +48,11 @@ import EvaluateTestAgents from './pages/evaluators/evaluators/EvaluateTestAgents import EvaluatorDetail from './pages/evaluators/evaluators/EvaluatorDetail' // Evaluator Results -import ResultsOverview from './pages/evaluators/results/ResultsOverview' -import ResultsAgentWorkspace from './pages/evaluators/results/ResultsAgentWorkspace' +import ResultsHub from './pages/evaluators/results/ResultsHub' import { RedirectAgentScenarioToWorkspace, RedirectAgentSuiteToWorkspace, + RedirectAgentWorkspaceToHub, } from './pages/evaluators/results/ResultsAgentWorkspaceRedirects' import ResultsUnassigned from './pages/evaluators/results/ResultsUnassigned' import EvaluatorResultDetail from './pages/evaluators/results/EvaluatorResultDetail' @@ -181,9 +181,9 @@ function App() { } /> } /> - } /> + } /> } /> - } /> + } /> } diff --git a/frontend/src/components/VoiceAgent.tsx b/frontend/src/components/VoiceAgent.tsx index 07253840..935271f0 100644 --- a/frontend/src/components/VoiceAgent.tsx +++ b/frontend/src/components/VoiceAgent.tsx @@ -26,11 +26,31 @@ interface VoiceAgentProps { compact?: boolean sidebarLayout?: boolean agentDisplayName?: string + connectDisabled?: boolean + connectDisabledReason?: string + runEvaluation?: boolean + userTranscriptLabel?: string + botTranscriptLabel?: string } type TranscriptEntry = { role: 'user' | 'agent'; content: string; timestamp: string } -export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpoint, customEndpointLabel, onSessionSaved, compact = false, sidebarLayout = false, agentDisplayName }: VoiceAgentProps) { +export default function VoiceAgent({ + personaId, + scenarioId, + agentId, + customEndpoint, + customEndpointLabel, + onSessionSaved, + compact = false, + sidebarLayout = false, + agentDisplayName, + connectDisabled = false, + connectDisabledReason, + runEvaluation = false, + userTranscriptLabel = 'You', + botTranscriptLabel = 'Agent', +}: VoiceAgentProps) { const { selectedAgent } = useAgentStore() // Use agentId prop if provided, otherwise fall back to selectedAgent from store @@ -419,6 +439,7 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo } if (!customEndpoint && personaId) params.append('persona_id', personaId) if (!customEndpoint && scenarioId) params.append('scenario_id', scenarioId) + if (!customEndpoint && runEvaluation) params.append('run_evaluation', 'true') if (!customEndpoint && params.toString()) { endpointUrl += `?${params.toString()}` @@ -534,7 +555,7 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo entry.role === 'user' ? 'bg-blue-50 text-blue-900 ml-2' : 'bg-gray-100 text-gray-800 mr-2' }`} > - {entry.role === 'user' ? 'You' : 'Agent'}: + {entry.role === 'user' ? userTranscriptLabel : botTranscriptLabel}: {entry.content} )) @@ -567,7 +588,8 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo onClick={connect} isLoading={isConnecting} leftIcon={isConnecting ? : } - disabled={isConnecting} + disabled={isConnecting || connectDisabled} + title={connectDisabled ? connectDisabledReason : undefined} > Start call @@ -658,7 +680,8 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo onClick={connect} isLoading={isConnecting} leftIcon={isConnecting ? : } - disabled={isConnecting} + disabled={isConnecting || connectDisabled} + title={connectDisabled ? connectDisabledReason : undefined} > Connect diff --git a/frontend/src/components/audio/RecordingAudioPlayer.tsx b/frontend/src/components/audio/RecordingAudioPlayer.tsx new file mode 100644 index 00000000..06ed474b --- /dev/null +++ b/frontend/src/components/audio/RecordingAudioPlayer.tsx @@ -0,0 +1,142 @@ +import { Download, Loader2, Pause, Play, Volume2 } from 'lucide-react' +import type { Ref } from 'react' + +import { + formatRecordingTime, + formatVolumePercent, + PLAYBACK_RATES, + type PlaybackRate, + useRecordingAudioPlayer, +} from '../../hooks/useRecordingAudioPlayer' + +const SLIDER_CLASS = + 'flex-1 min-w-0 h-2 rounded-full appearance-none bg-gray-200 accent-primary-600 cursor-pointer disabled:opacity-40 disabled:cursor-not-allowed [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-primary-600 [&::-webkit-slider-thumb]:shadow [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:border-0 [&::-moz-range-thumb]:bg-primary-600' + +interface RecordingAudioPlayerProps { + src: string + downloadUrl?: string + audioRef?: Ref + onTimeUpdate?: (currentTime: number) => void + onLoadedMetadata?: (duration: number) => void + onEnded?: () => void + className?: string +} + +export default function RecordingAudioPlayer({ + src, + downloadUrl, + audioRef, + onTimeUpdate, + onLoadedMetadata, + onEnded, + className = '', +}: RecordingAudioPlayerProps) { + const { + setAudioElementRef, + isPlaying, + isLoading, + currentTime, + duration, + volume, + playbackRate, + togglePlay, + seek, + setVolume, + setPlaybackRate, + canSeek, + } = useRecordingAudioPlayer({ + src, + audioRef, + onTimeUpdate, + onLoadedMetadata, + onEnded, + }) + + return ( +
+
+ ) +} diff --git a/frontend/src/components/call-recordings/CustomWebSocketCallDetails.tsx b/frontend/src/components/call-recordings/CustomWebSocketCallDetails.tsx index cf1e88eb..c3d9e83d 100644 --- a/frontend/src/components/call-recordings/CustomWebSocketCallDetails.tsx +++ b/frontend/src/components/call-recordings/CustomWebSocketCallDetails.tsx @@ -3,7 +3,6 @@ import { MessageSquare, Clock, Globe, - Download, Loader, TrendingUp, CheckCircle, @@ -13,6 +12,7 @@ import { Sparkles, } from 'lucide-react' import { apiClient } from '../../lib/api' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' interface SpeakerSegment { speaker: string @@ -260,23 +260,19 @@ export default function CustomWebSocketCallDetails({ callData, callShortId }: Pr
{audioLoading && } - {audioBlobUrl && ( -
-
- )} {audioError && hasAudio && ( Audio unavailable )}
+ {audioBlobUrl && ( +
+ +
+ )}
{segments.length > 0 ? ( diff --git a/frontend/src/components/call-recordings/ElevenLabsCallDetails.tsx b/frontend/src/components/call-recordings/ElevenLabsCallDetails.tsx index 111c0d3c..14670b8f 100644 --- a/frontend/src/components/call-recordings/ElevenLabsCallDetails.tsx +++ b/frontend/src/components/call-recordings/ElevenLabsCallDetails.tsx @@ -1,8 +1,9 @@ import { useState, useMemo, useEffect, useRef } from 'react' import { - DollarSign, MessageSquare, TrendingUp, Activity, Server, CheckCircle, XCircle, Clock, Zap, Download, Loader + DollarSign, MessageSquare, TrendingUp, Activity, Server, CheckCircle, XCircle, Clock, Zap, Loader } from 'lucide-react' import { apiClient } from '../../lib/api' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer, PieChart, Pie, Cell @@ -326,19 +327,19 @@ export default function ElevenLabsCallDetails({ callData, callShortId, hideTrans {audioLoading && ( )} - {audioBlobUrl && ( -
-
- )} {audioError && hasAudio && ( Audio unavailable )}
+ {audioBlobUrl && ( +
+ +
+ )}
{transcriptEntries.length > 0 ? ( diff --git a/frontend/src/components/call-recordings/RetellCallDetails.tsx b/frontend/src/components/call-recordings/RetellCallDetails.tsx index 16bb4dcc..8921b6ce 100644 --- a/frontend/src/components/call-recordings/RetellCallDetails.tsx +++ b/frontend/src/components/call-recordings/RetellCallDetails.tsx @@ -1,7 +1,8 @@ import { useState } from 'react' import { - DollarSign, MessageSquare, TrendingUp, Download, Activity, Server + DollarSign, MessageSquare, TrendingUp, Activity, Server } from 'lucide-react' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer, PieChart, Pie, Cell @@ -166,15 +167,12 @@ export default function RetellCallDetails({ callData, hideTranscript = false }: Transcript - {callData.recording_url && ( -
-
- )}
+ {callData.recording_url && ( +
+ +
+ )}
{callData.transcript_object?.map((msg, idx) => ( diff --git a/frontend/src/components/call-recordings/SmallestCallDetails.tsx b/frontend/src/components/call-recordings/SmallestCallDetails.tsx index 637d50fb..828a32ef 100644 --- a/frontend/src/components/call-recordings/SmallestCallDetails.tsx +++ b/frontend/src/components/call-recordings/SmallestCallDetails.tsx @@ -1,7 +1,8 @@ import { useMemo, useState } from 'react' import { - DollarSign, MessageSquare, TrendingUp, Download, Activity, Server, CheckCircle, XCircle + DollarSign, MessageSquare, TrendingUp, Activity, Server, CheckCircle, XCircle } from 'lucide-react' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, PieChart, Pie, Cell, Legend } from 'recharts' @@ -172,15 +173,12 @@ export default function SmallestCallDetails({ callData, hideTranscript = false } Transcript - {recordingUrl && ( -
-
- )}
+ {recordingUrl && ( +
+ +
+ )}
{transcriptEntries.length > 0 ? ( transcriptEntries.map((entry, idx) => { diff --git a/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx b/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx index 7107a677..81fe0039 100644 --- a/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx +++ b/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx @@ -1,7 +1,8 @@ import { useState, type ReactNode } from 'react' import { - Clock, MessageSquare, TrendingUp, Download, Server, BarChart3, HelpCircle, Brain, Sparkles, AudioWaveform + Clock, MessageSquare, TrendingUp, Server, BarChart3, HelpCircle, Brain, Sparkles, AudioWaveform } from 'lucide-react' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' const LEGACY_CATEGORY_LABEL_METRIC_NAMES = new Set([ 'yes', @@ -190,7 +191,7 @@ interface TestVoiceAgentResultData { name?: string timestamp?: string duration_seconds?: number | null - status?: 'queued' | 'transcribing' | 'evaluating' | 'completed' | 'failed' + status?: 'queued' | 'transcribing' | 'evaluating' | 'completed' | 'failed' | 'call_ended' transcription?: string | null speaker_segments?: Array<{ speaker: string @@ -620,15 +621,12 @@ export default function TestVoiceAgentResultDetails({ resultData }: TestVoiceAge Transcript - {resultData.audioUrl && ( -
-
- )}
+ {resultData.audioUrl && ( +
+ +
+ )}
{resultData.speaker_segments && resultData.speaker_segments.length > 0 ? ( diff --git a/frontend/src/components/call-recordings/VapiCallDetails.tsx b/frontend/src/components/call-recordings/VapiCallDetails.tsx index 09ae62a8..fd3d2c2e 100644 --- a/frontend/src/components/call-recordings/VapiCallDetails.tsx +++ b/frontend/src/components/call-recordings/VapiCallDetails.tsx @@ -2,6 +2,7 @@ import { useState } from 'react' import { DollarSign, MessageSquare, TrendingUp, Download, Activity, Server, CheckCircle, XCircle } from 'lucide-react' +import RecordingAudioPlayer from '../audio/RecordingAudioPlayer' import { BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer, PieChart, Pie, Cell @@ -308,15 +309,12 @@ export default function VapiCallDetails({ callData, hideTranscript = false }: Va Transcript - {recordingUrl && ( -
-
- )}
+ {recordingUrl && ( +
+ +
+ )}
{transcriptEntries.length > 0 ? ( diff --git a/frontend/src/components/metricClusters/MetricClustersPanel.tsx b/frontend/src/components/metricClusters/MetricClustersPanel.tsx new file mode 100644 index 00000000..16d52015 --- /dev/null +++ b/frontend/src/components/metricClusters/MetricClustersPanel.tsx @@ -0,0 +1,1309 @@ +import { type ReactNode, useEffect, useState } from 'react' +import { createPortal } from 'react-dom' +import { Link } from 'react-router-dom' +import { useQuery } from '@tanstack/react-query' +import { ExternalLink, Loader2, X } from 'lucide-react' +import AIProviderModelPicker from '../AIProviderModelPicker' +import Button from '../Button' +import type { MetricClustersClient } from './clients' +import type { + EvaluationMetricClustersState, + MetricClustersRcaSummary, + MetricFailurePolicy, + MetricFailurePolicyMetricPreview, + MetricClustersPanelProps, +} from './types' + +const METRIC_CLUSTER_ROW_PRESETS = [25, 50, 500] as const +type MetricClusterRowPreset = + (typeof METRIC_CLUSTER_ROW_PRESETS)[number] | 'all' + +const PROVIDER_DISPLAY: Record = { + openai: 'OpenAI', + anthropic: 'Anthropic', + google: 'Google', + deepseek: 'DeepSeek', + groq: 'Groq', +} + +function clampProseToSentences( + text: string, + maxSentences = 3, + maxChars = 300, +): string { + const trimmed = text.trim().replace(/\s*\n+\s*/g, ' ') + if (!trimmed) return trimmed + const sentences = trimmed.split(/(?<=[.!?])\s+/).filter(Boolean) + let result = (sentences.length ? sentences.slice(0, maxSentences) : [trimmed]) + .join(' ') + .trim() + if (result.length > maxChars) { + const cut = result.slice(0, maxChars - 3).replace(/\s+\S*$/, '') + result = `${cut || result.slice(0, maxChars)}...` + } + return result +} + +function metricClusterSelectedCount( + totalEligible: number, + preset: MetricClusterRowPreset, +): number { + if (totalEligible <= 0) return 0 + if (preset === 'all') return totalEligible + return Math.min(preset, totalEligible) +} + +function MetricClusterRowPicker({ + totalEligible, + preset, + onChangePreset, + disabled, +}: { + totalEligible: number + preset: MetricClusterRowPreset + onChangePreset: (next: MetricClusterRowPreset) => void + disabled?: boolean +}) { + const selectedCount = metricClusterSelectedCount(totalEligible, preset) + + const presetActive = (n: number) => + preset !== 'all' && preset === n && selectedCount === n + + const allActive = preset === 'all' && totalEligible > 0 + + const presetButtonClass = (active: boolean) => + 'rounded-full px-2 py-0.5 border text-[10px] font-medium transition-colors disabled:opacity-40 ' + + (active + ? 'border-primary-300 bg-primary-50 text-primary-800' + : 'border-gray-200 bg-white text-gray-600 hover:border-gray-300 hover:bg-gray-50') + + return ( +
+
+
+

+ Calls to include ({selectedCount} / {totalEligible} eligible) +

+
+ {totalEligible > 0 ? ( +
+ {METRIC_CLUSTER_ROW_PRESETS.map((n) => ( + + ))} + +
+ ) : ( +

+ No completed calls with a flagged quality metric yet. +

+ )} +
+
+ ) +} + +function normalizeFailureLabel(label: string): string { + return label.trim().toLowerCase() +} + +function failureRowCountForPreview( + preview: MetricFailurePolicyMetricPreview, + policy: MetricFailurePolicy, +): number { + if (preview.is_multi_label_parent) { + let total = 0 + for (const name of policy.failure_child_names || []) { + total += preview.row_count_by_value[name] ?? 0 + } + return total + } + let total = 0 + const targets = new Set( + (policy.failure_values || []).map((v) => normalizeFailureLabel(v)), + ) + for (const [label, count] of Object.entries(preview.row_count_by_value)) { + if (targets.has(normalizeFailureLabel(label))) { + total += count + } + } + return total +} + +function policyHasFailureCriteria( + preview: MetricFailurePolicyMetricPreview, + policy: MetricFailurePolicy, +): boolean { + if (preview.is_multi_label_parent) { + return (policy.failure_child_names?.length ?? 0) > 0 + } + if (policy.numeric_rule) return true + return (policy.failure_values?.length ?? 0) > 0 +} + +function MetricFailurePolicyEditor({ + previews, + policies, + policiesSource, + onChangePolicies, + disabled, +}: { + previews: MetricFailurePolicyMetricPreview[] + policies: Record + policiesSource: 'inferred' | 'user' + onChangePolicies: (next: Record) => void + disabled?: boolean +}) { + if (!previews.length) { + return ( +

+ No quality metrics available for failure policy configuration. +

+ ) + } + + return ( +
+
+

+ Failure values per metric +

+

+ Select which answers count as failures for metrics you want to cluster. + Metrics with none selected, or with no matching calls, are skipped.{' '} + {policiesSource === 'inferred' ? ( + + Suggested defaults only where matching rows exist. + + ) : ( + Saved for this evaluation. + )} +

+
+ {previews.map((preview) => { + const policy = + policies[preview.metric_id] ?? preview.effective_policy + const failureCount = failureRowCountForPreview(preview, policy) + const hasCriteria = policyHasFailureCriteria(preview, policy) + const isSkipped = !hasCriteria || failureCount === 0 + + const toggleValue = (label: string, checked: boolean) => { + const norm = normalizeFailureLabel(label) + const current = new Set( + (policy.failure_values || []).map(normalizeFailureLabel), + ) + if (checked) current.add(norm) + else current.delete(norm) + const nextValues = preview.value_counts + .map((vc) => vc.label) + .filter((l) => current.has(normalizeFailureLabel(l))) + onChangePolicies({ + ...policies, + [preview.metric_id]: { + ...policy, + metric_id: preview.metric_id, + failure_values: nextValues.map(normalizeFailureLabel), + }, + }) + } + + const toggleChild = (name: string, checked: boolean) => { + const current = new Set(policy.failure_child_names || []) + if (checked) current.add(name) + else current.delete(name) + onChangePolicies({ + ...policies, + [preview.metric_id]: { + ...policy, + metric_id: preview.metric_id, + failure_child_names: Array.from(current), + }, + }) + } + + return ( +
+
+

+ {preview.metric_name} +

+ + {isSkipped + ? 'Skipped — no matching calls' + : `${failureCount} call${failureCount === 1 ? '' : 's'} to cluster`} + +
+ {preview.is_multi_label_parent ? ( +
+ {preview.child_names.map((name) => { + const checked = (policy.failure_child_names || []).includes( + name, + ) + const count = preview.row_count_by_value[name] ?? 0 + return ( + + ) + })} +
+ ) : preview.value_counts.length ? ( +
+ {preview.value_counts.map((vc) => { + const checked = (policy.failure_values || []).some( + (v) => + normalizeFailureLabel(v) === + normalizeFailureLabel(vc.label), + ) + return ( + + ) + })} +
+ ) : policy.numeric_rule ? ( +

+ Numeric failures: score {policy.numeric_rule.op}{' '} + {policy.numeric_rule.threshold} + {preview.metric_type ? ` (${preview.metric_type})` : ''} +

+ ) : ( +

No observed values yet.

+ )} +
+ ) + })} +
+ ) +} + +function MetricClusterGenerationModal({ + open, + onClose, + client, + defaultProvider = '', + defaultModel = '', + state, + onGenerated, + onError, + overlayZIndexClass = 'z-50', +}: { + open: boolean + onClose: () => void + client: MetricClustersClient + defaultProvider?: string + defaultModel?: string + state: EvaluationMetricClustersState | null + onGenerated: () => void + onError?: (message: string | null) => void + overlayZIndexClass?: string +}) { + const [generating, setGenerating] = useState(false) + const [error, setError] = useState(null) + const [pickerProvider, setPickerProvider] = useState('') + const [pickerModel, setPickerModel] = useState('') + const [rowPreset, setRowPreset] = useState(25) + const [llmPickerTouched, setLlmPickerTouched] = useState(false) + const [policies, setPolicies] = useState>( + {}, + ) + const [policiesSource, setPoliciesSource] = useState<'inferred' | 'user'>( + 'inferred', + ) + const [policiesTouched, setPoliciesTouched] = useState(false) + + const failurePoliciesQuery = useQuery({ + queryKey: [...client.queryKeyPrefix, 'failure-policies'], + queryFn: () => client.getFailurePolicies(), + enabled: open, + staleTime: 30_000, + }) + + const eligibleRowsQuery = useQuery({ + queryKey: [...client.queryKeyPrefix, 'eligible-rows'], + queryFn: () => client.listEligibleRows({ count_only: true }), + enabled: open, + staleTime: 30_000, + }) + + const totalEligible = eligibleRowsQuery.data?.total ?? 0 + const selectedRowCount = metricClusterSelectedCount(totalEligible, rowPreset) + + const hasExistingClusters = !!state?.groups?.length + + useEffect(() => { + if (!open) return + setError(null) + onError?.(null) + setRowPreset(25) + }, [open, onError]) + + useEffect(() => { + const data = failurePoliciesQuery.data + if (!open || !data || policiesTouched) return + setPolicies(data.policies) + setPoliciesSource(data.source) + }, [open, failurePoliciesQuery.data, policiesTouched]) + + useEffect(() => { + if (!open || !policiesTouched || generating) return + const timer = window.setTimeout(() => { + client + .saveFailurePolicies(policies) + .then((saved) => { + setPoliciesSource(saved.source) + eligibleRowsQuery.refetch() + }) + .catch(() => { + /* keep local edits; generate will persist */ + }) + }, 600) + return () => window.clearTimeout(timer) + }, [open, policies, policiesTouched, generating, client, eligibleRowsQuery]) + + useEffect(() => { + if (state?.provider) { + setPickerProvider(state.provider) + if (state.model) setPickerModel(state.model) + return + } + if (llmPickerTouched) return + if (defaultProvider) setPickerProvider(defaultProvider) + if (defaultModel) setPickerModel(defaultModel) + }, [ + state?.provider, + state?.model, + defaultProvider, + defaultModel, + llmPickerTouched, + ]) + + const reportError = (message: string | null) => { + setError(message) + onError?.(message) + } + + const handleGenerate = async () => { + if (selectedRowCount === 0) { + reportError('Select at least one call to cluster.') + return + } + const previews = failurePoliciesQuery.data?.previews ?? [] + const hasClusterableMetric = previews.some((p) => { + const policy = policies[p.metric_id] ?? p.effective_policy + return ( + policyHasFailureCriteria(p, policy) && + failureRowCountForPreview(p, policy) > 0 + ) + }) + if (!hasClusterableMetric) { + reportError( + 'No calls match any failure policy. Select failure values on at least one metric that has matching rows.', + ) + return + } + setGenerating(true) + reportError(null) + try { + const force = hasExistingClusters + await client.generateClusters({ + force, + regenerate: force, + provider: pickerProvider || undefined, + model: pickerModel || undefined, + row_limit: rowPreset === 'all' ? undefined : rowPreset, + failure_policies: policies, + }) + onGenerated() + onClose() + } catch (e: any) { + reportError( + e?.response?.data?.detail || + 'Failed to start metric cluster generation.', + ) + } finally { + setGenerating(false) + } + } + + const handleClose = () => { + if (generating) return + reportError(null) + onClose() + } + + if (!open || typeof document === 'undefined') return null + + return createPortal( +
+
+
+
+

+ Generate clusters +

+

+ Configure failure values per metric, choose calls, then generate + clusters from LLM rationales. +

+
+ +
+
+
+ {failurePoliciesQuery.isLoading ? ( +

Loading failure policies…

+ ) : failurePoliciesQuery.data ? ( + { + setPoliciesTouched(true) + setPolicies(next) + }} + disabled={generating} + /> + ) : null} +
+
+ {eligibleRowsQuery.isLoading ? ( +

Loading eligible calls…

+ ) : ( + + )} +
+
+

+ LLM for clustering +

+

+ Provider and model used for failure signatures and cluster + synthesis. Defaults to this evaluation's scoring LLM when + unset. +

+ { + setLlmPickerTouched(true) + setPickerProvider(next) + }} + onModelChange={(next) => { + setLlmPickerTouched(true) + setPickerModel(next) + }} + disabled={generating} + size="sm" + /> +
+ {error ?

{error}

: null} +
+
+ + +
+
+
, + document.body, + ) +} + +function RcaExecutiveBar({ pct, scaleMax }: { pct: number; scaleMax: number }) { + const width = + scaleMax > 0 ? Math.min(100, Math.round((pct / scaleMax) * 100)) : 0 + return ( +
+
+
+ ) +} + +function RcaExecutiveInterpretation({ children }: { children: ReactNode }) { + return ( +
+

+ Executive interpretation +

+

{children}

+
+ ) +} + +function MetricClustersRcaSummaryPanel({ + summary, +}: { + summary: MetricClustersRcaSummary +}) { + const topPattern = summary.repeated_patterns[0] + const topHotspot = summary.metric_hotspots[0] + const maxPatternShare = Math.max( + ...summary.repeated_patterns.map((r) => r.evidence_share_pct), + 1, + ) + const maxHotspotRate = Math.max( + ...summary.metric_hotspots.map((r) => r.metric_rate_pct), + 1, + ) + const totalFlagged = + summary.total_flagged_instances ?? + summary.metric_hotspots.reduce((sum, r) => sum + r.flagged_calls, 0) + + return ( +
+
+

+ Executive summary — evaluation set +

+

+ Top metrics by clustered failure patterns and overall flagged rate across{' '} + {summary.analysed_calls.toLocaleString()} analysed calls. +

+
+ + {summary.repeated_patterns.length ? ( +
+
+
+ Repeated failure patterns +
+

+ Base: {summary.total_clusters} RCA clusters from{' '} + {summary.total_clustered_instances.toLocaleString()} clustered instances ·{' '} + {totalFlagged.toLocaleString()} flagged metric-call instances +

+
+
+ + + + + + + + + + + + + + + + + {summary.repeated_patterns.map((row) => ( + + + + + + + ))} + +
Finding + Evidence share + Distribution + Evidence calls +
+

+ {row.metric_name} +

+

+ Top RCA patterns: {row.top_rca_patterns} +

+ 		+ {row.evidence_share_pct.toFixed(1)}% + + + +

+ {row.evidence_calls.toLocaleString()} +

+

+ {row.evidence_share_pct.toFixed(1)}% +

+
+
+ {topPattern ? ( + + These rows group repeated RCA failure patterns by metric so the same + metric is not repeated across multiple rows. The largest group is{' '} + {topPattern.metric_name}; focus + remediation there first using the example calls in each cluster below. + + ) : null} +
+ ) : null} + + {summary.metric_hotspots.length ? ( +
+
+
Metric hotspots
+

+ Base: selected metric flags across{' '} + {summary.analysed_calls.toLocaleString()} analysed calls +

+
+
+ + + + + + + + + + + {summary.metric_hotspots.map((row) => ( + + + + + + + ))} + +
Finding + Metric rate + Distribution + Flagged calls +
+

+ {row.metric_name} +

+ 		+ {row.metric_rate_pct.toFixed(2)}% + + + + {row.flagged_calls.toLocaleString()} +
+
+ {topHotspot ? ( + + Across {summary.analysed_calls.toLocaleString()} analysed calls,{' '} + {topHotspot.metric_name} has the + highest metric rate at {topHotspot.metric_rate_pct.toFixed(2)}%. + + ) : null} +
+ ) : null} + + {summary.prompt_areas.length ? ( +
+
RCA data summary
+

+ Prompt areas to inspect +

+ + + + + + + + + {summary.prompt_areas.map((row) => ( + + + + + ))} + +
Area%
{row.label} + {row.share_pct.toFixed(1)}% +
+
+ ) : null} + +
+
Appendix: What is a cluster?
+

+ A cluster groups flagged calls that share the same underlying failure theme within + a quality metric. Each cluster is labeled with an RCA pattern name and an + engineering gap type (such as MISSING, LOGIC_GAP, UNDERSPEC, or + EXISTS_NO_TRIGGER). Evidence share is the percentage of all clustered failure + instances attributed to that metric's patterns; evidence calls is the raw + count of those instances. +

+
+
+ ) +} + +export function MetricClustersPanel({ + client, + defaultProvider = '', + defaultModel = '', + state, + isLoading, + onGenerated, +}: MetricClustersPanelProps) { + const [cancelling, setCancelling] = useState(false) + const [error, setError] = useState(null) + const [pickerProvider, setPickerProvider] = useState('') + const [pickerModel, setPickerModel] = useState('') + const [llmPickerTouched, setLlmPickerTouched] = useState(false) + const [clusterActionModalOpen, setClusterActionModalOpen] = useState(false) + + useEffect(() => { + if (state?.provider) { + setPickerProvider(state.provider) + if (state.model) setPickerModel(state.model) + return + } + if (llmPickerTouched) return + if (defaultProvider) setPickerProvider(defaultProvider) + if (defaultModel) setPickerModel(defaultModel) + }, [ + state?.provider, + state?.model, + defaultProvider, + defaultModel, + llmPickerTouched, + ]) + + const llmPickerDisabled = cancelling || state?.status === 'running' + + const llmPickerBlock = ( +
+

+ LLM for clustering +

+

+ Provider and model used for failure signatures and cluster synthesis. + Defaults to this evaluation's scoring LLM when unset. +

+ { + setLlmPickerTouched(true) + setPickerProvider(next) + }} + onModelChange={(next) => { + setLlmPickerTouched(true) + setPickerModel(next) + }} + disabled={llmPickerDisabled} + size="sm" + /> +
+ ) + + const selectedCountLabel = state?.selected_evaluation_row_ids?.length + + const clusterGenerationModal = ( + { + setClusterActionModalOpen(false) + setError(null) + }} + client={client} + defaultProvider={defaultProvider} + defaultModel={defaultModel} + state={state} + onGenerated={onGenerated} + onError={setError} + /> + ) + + const handleCancel = async () => { + setCancelling(true) + setError(null) + try { + await client.cancelClusters() + onGenerated() + } catch (e: any) { + setError( + e?.response?.data?.detail || 'Failed to stop cluster generation.', + ) + } finally { + setCancelling(false) + } + } + + if (isLoading && !state) { + return ( + <> +
+

+ Failure diagnostics (internal) +

+

Loading…

+ {llmPickerBlock} +
+ {clusterGenerationModal} + + ) + } + + if (state?.status === 'running') { + const progress = state.progress + const completed = progress?.completed_llm_calls ?? 0 + const total = progress?.total_llm_calls ?? 0 + const pct = total > 0 ? Math.min(100, Math.round((completed / total) * 100)) : 0 + const providerLabel = state.provider + ? PROVIDER_DISPLAY[state.provider] || state.provider + : null + const callsLabel = selectedCountLabel + ? `${selectedCountLabel} selected call${selectedCountLabel === 1 ? '' : 's'}` + : 'flagged calls' + return ( + <> +
+
+

+ + Failure diagnostics — generating clusters +

+ {total > 0 ? ( +

+ {completed} / {total} LLM calls ({pct}%) +

+ ) : null} +
+

+ Clustering {callsLabel} for each enabled quality metric. +

+ {total > 0 ? ( +
+
+
+
+
+ ) : ( +
+
+
+ )} + {providerLabel || state.model ? ( +

+ Using {providerLabel || 'LLM'} + {state.model ? ` · ${state.model}` : ''} +

+ ) : null} +
+ +
+ {error ?

{error}

: null} +
+ {clusterGenerationModal} + + ) + } + + if (state?.status === 'cancelled') { + return ( +
+

+ Failure diagnostics stopped +

+

+ {state.error_message || + 'Cluster generation was cancelled. Partial results were not saved.'} +

+ {state.progress ? ( +

+ Stopped at {state.progress.completed_llm_calls} /{' '} + {state.progress.total_llm_calls} LLM calls +

+ ) : null} +
+ +
+ {clusterGenerationModal} +
+ ) + } + + if (state?.status === 'failed') { + return ( +
+

+ Failure diagnostics failed +

+

+ {state.error_message || 'Cluster generation failed.'} +

+
+ +
+ {clusterGenerationModal} +
+ ) + } + + if (!state || state.status === 'idle' || !state.groups.length) { + return ( +
+
+

+ Failure diagnostics (internal) +

+

+ Choose which flagged calls to include, then cluster per enabled + quality metric (gap labels: LOGIC_GAP, UNDERSPEC, EXISTS_NO_TRIGGER, + MISSING). +

+
+
+ +
+ {clusterGenerationModal} +
+ ) + } + + return ( +
+
+
+

Generation

+ +
+
+

+ Select the calls and model in a modal, then generate clusters. + Run again after more rows complete or when you change the model. +

+ {state.overview ? ( +

+ {clampProseToSentences(state.overview)} +

+ ) : null} + {state.is_stale ? ( +

+ More rows completed since clusters were generated. Generate again + to refresh. +

+ ) : null} + {state.selected_evaluation_row_ids?.length ? ( +

+ Based on {state.selected_evaluation_row_ids.length} selected call + {state.selected_evaluation_row_ids.length === 1 ? '' : 's'}. +

+ ) : null} +
+ {error ?

{error}

: null} +
+ + {clusterGenerationModal} + +
+
+

Results

+

+ Per-metric clusters of flagged calls with gap labels and Level-2 + sub-categories. +

+
+ {state.rca_summary ? ( + + ) : null} + {state.groups.map((group) => { + const topClusters = [...group.clusters] + .sort((a, b) => b.count - a.count || a.label.localeCompare(b.label)) + .slice(0, 5) + return ( +
+
+

+ {group.metric_name} +

+

+ {group.flagged_count} flagged calls · {topClusters.length} + {group.clusters.length > 5 + ? ` of ${group.clusters.length}` + : ''}{' '} + cluster(s) shown + {state.failure_policies?.[group.metric_id] ? ( + <> + {' '} + · failure:{' '} + {[ + ...(state.failure_policies[group.metric_id] + .failure_values || []), + ...(state.failure_policies[group.metric_id] + .failure_child_names || []), + ].join(', ') || 'numeric rule'} + + ) : null} +

+ {group.failure_reason ? ( +

+ Why flagged:{' '} + {group.failure_reason} +

+ ) : null} +
+
+ {(() => { + const categorizedCalls = topClusters.reduce( + (sum, cluster) => sum + Math.max(0, cluster.count || 0), + 0, + ) + const totalFlagged = Math.max(0, group.flagged_count || 0) + return ( +
+
+

+ Cluster breakdown +

+ + {categorizedCalls} / {totalFlagged} + +
+
+ ) + })()} + {topClusters.map((cluster) => { + const exampleHref = client.buildEvidenceHref(cluster.evidence) + return ( +
+
+

+ {cluster.label} +

+ + {cluster.gap_label.replace(/_/g, ' ')} + +
+

+ {cluster.count} calls · {cluster.share_pct.toFixed(1)}% share +

+ {cluster.failure_reason ? ( +

+ Why flagged:{' '} + {cluster.failure_reason} +

+ ) : null} + {group.flagged_count > 0 ? ( +
+
+
+
+
+ ) : null} + {cluster.observation ? ( +

+ {cluster.observation} +

+ ) : null} + {cluster.sub_clusters.length ? ( +
    + {cluster.sub_clusters.map((sub) => ( +
  • + {sub.label} — {sub.count} ({sub.share_pct.toFixed(1)}%) +
    • + ))} +
+ ) : null} + {(cluster.evidence.quote || + cluster.evidence.turns?.length || + cluster.evidence.conversation_id) ? ( +
+

+ Example call +

+ {cluster.evidence.turns?.length ? ( + cluster.evidence.turns.map((turn, i) => ( +

+ + {turn.speaker}: + {' '} + {turn.text} +

+ )) + ) : cluster.evidence.quote ? ( +

{cluster.evidence.quote}

+ ) : null} + {exampleHref && cluster.evidence.conversation_id ? ( + + + {cluster.evidence.conversation_id} + + ) : cluster.evidence.conversation_id ? ( +

+ {cluster.evidence.conversation_id} +

+ ) : null} +
+ ) : null} +
+ )})} +
+
+ )})} + {state.discovered_problems.length ? ( +
+

+ Proactive problem discovery +

+
+ {state.discovered_problems.map((item) => ( +
+ {item.label} + + {item.gap_label.replace(/_/g, ' ')} + + + {item.count} · {item.share_pct.toFixed(1)}% + + {item.observation ? ( +

{item.observation}

+ ) : null} +
+ ))} +
+
+ ) : null} +
+
+ ) +} + +export default MetricClustersPanel + +export type { MetricClustersPanelProps } diff --git a/frontend/src/components/metricClusters/clients.ts b/frontend/src/components/metricClusters/clients.ts new file mode 100644 index 00000000..31b281ca --- /dev/null +++ b/frontend/src/components/metricClusters/clients.ts @@ -0,0 +1,129 @@ +import { apiClient } from '../../lib/api' +import type { + EvaluationMetricClustersState, + MetricClusterEvidence, + MetricClusterEligibleRowsResponse, + MetricFailurePoliciesResponse, + MetricFailurePolicy, +} from '../../types/api' + +export interface MetricClustersClient { + queryKeyPrefix: readonly unknown[] + getFailurePolicies(): Promise + saveFailurePolicies( + policies: Record, + ): Promise + listEligibleRows(options?: { + limit?: number + count_only?: boolean + }): Promise + generateClusters(options: { + force?: boolean + regenerate?: boolean + provider?: string + model?: string + row_limit?: number + failure_policies?: Record + }): Promise + cancelClusters(): Promise + buildEvidenceHref(evidence: MetricClusterEvidence): string | null +} + +function buildCallImportEvidenceHref( + callImportId: string, + evaluationId: string, + evidence: MetricClusterEvidence, +): string | null { + const conv = evidence.conversation_id?.trim() + const rowId = evidence.evaluation_row_id?.trim() + if (!conv && !rowId) return null + const base = `/call-imports/${callImportId}/evaluations/${evaluationId}` + if (conv) { + return `${base}?conversation_id=${encodeURIComponent(conv)}` + } + return `${base}?row_id=${encodeURIComponent(rowId!)}` +} + +export function createCallImportMetricClustersClient( + callImportId: string, + evaluationId: string, + workspaceId: string | null | undefined, +): MetricClustersClient { + const queryKeyPrefix = [ + 'call-import-evaluation-metric-clusters', + workspaceId, + callImportId, + evaluationId, + ] as const + + return { + queryKeyPrefix, + getFailurePolicies: () => + apiClient.getCallImportEvaluationMetricClusterFailurePolicies( + callImportId, + evaluationId, + ), + saveFailurePolicies: (policies) => + apiClient.saveCallImportEvaluationMetricClusterFailurePolicies( + callImportId, + evaluationId, + policies, + ), + listEligibleRows: (options) => + apiClient.listCallImportEvaluationMetricClusterEligibleRows( + callImportId, + evaluationId, + options, + ), + generateClusters: (options) => + apiClient.generateCallImportEvaluationMetricClusters( + callImportId, + evaluationId, + options, + ), + cancelClusters: () => + apiClient.cancelCallImportEvaluationMetricClusters( + callImportId, + evaluationId, + ), + buildEvidenceHref: (evidence) => + buildCallImportEvidenceHref(callImportId, evaluationId, evidence), + } +} + +export type EvaluatorResultClusterScope = { + agentId?: string + suiteId?: string + scenarioId?: string +} + +export function createEvaluatorResultsMetricClustersClient( + scope: EvaluatorResultClusterScope, + workspaceId: string | null | undefined, +): MetricClustersClient { + const queryKeyPrefix = [ + 'evaluator-results-metric-clusters', + workspaceId, + scope.agentId ?? '', + scope.suiteId ?? '', + scope.scenarioId ?? '', + ] as const + + return { + queryKeyPrefix, + getFailurePolicies: () => + apiClient.getEvaluatorResultMetricClusterFailurePolicies(scope), + saveFailurePolicies: (policies) => + apiClient.saveEvaluatorResultMetricClusterFailurePolicies(scope, policies), + listEligibleRows: (options) => + apiClient.listEvaluatorResultMetricClusterEligibleRows(scope, options), + generateClusters: (options) => + apiClient.generateEvaluatorResultMetricClusters(scope, options), + cancelClusters: () => apiClient.cancelEvaluatorResultMetricClusters(scope), + buildEvidenceHref: (evidence) => { + const id = evidence.conversation_id?.trim() + if (!id) return null + return `/results/${id}` + }, + } +} diff --git a/frontend/src/components/metricClusters/types.ts b/frontend/src/components/metricClusters/types.ts new file mode 100644 index 00000000..2ab630c8 --- /dev/null +++ b/frontend/src/components/metricClusters/types.ts @@ -0,0 +1,19 @@ +import type { EvaluationMetricClustersState } from '../../types/api' +import type { MetricClustersClient } from './clients' + +export type { + EvaluationMetricClustersState, + MetricClusterEvidence, + MetricClustersRcaSummary, + MetricFailurePolicy, + MetricFailurePolicyMetricPreview, +} from '../../types/api' + +export interface MetricClustersPanelProps { + client: MetricClustersClient + defaultProvider?: string + defaultModel?: string + state: EvaluationMetricClustersState | null + isLoading: boolean + onGenerated: () => void +} diff --git a/frontend/src/components/walkthrough/walkthroughRegistry.ts b/frontend/src/components/walkthrough/walkthroughRegistry.ts index 10b33c0c..603b4dc5 100644 --- a/frontend/src/components/walkthrough/walkthroughRegistry.ts +++ b/frontend/src/components/walkthrough/walkthroughRegistry.ts @@ -273,17 +273,17 @@ function getEvaluatorsWalkthrough(state?: EvaluatorsWalkthroughState): Walkthrou return { id: 'evaluators', title: 'Evaluators Walkthrough', - subtitle: 'Create an evaluator suite: agent + persona + multiple scenarios.', + subtitle: 'Create an evaluator suite: agent + personas + multiple scenarios.', steps: [ { - title: 'Step 1: Agent & persona', - description: 'Pick one agent and one compatible persona.', - bullets: ['Personas are filtered by TTS provider match', 'One persona per suite'], + title: 'Step 1: Agent & personas', + description: 'Pick one agent and one or more compatible personas.', + bullets: ['Personas are filtered by TTS provider match', 'Multiple personas expand the combination grid'], }, { title: 'Step 2: Scenarios', - description: 'Select multiple scenarios — each becomes one combination.', - bullets: ['N scenarios = N combinations', 'All share the same agent and persona'], + description: 'Select multiple scenarios — each persona × scenario becomes one combination.', + bullets: ['M personas × N scenarios = M×N combinations', 'All share the same agent and suite metrics'], }, { title: 'Step 3: Metrics', diff --git a/frontend/src/hooks/useRecordingAudioPlayer.ts b/frontend/src/hooks/useRecordingAudioPlayer.ts new file mode 100644 index 00000000..3715a580 --- /dev/null +++ b/frontend/src/hooks/useRecordingAudioPlayer.ts @@ -0,0 +1,188 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type MutableRefObject, + type Ref, +} from 'react' + +export const PLAYBACK_RATES = [0.5, 0.75, 1, 1.25, 1.5, 2] as const +export type PlaybackRate = (typeof PLAYBACK_RATES)[number] + +const DEFAULT_VOLUME = 0.85 + +export function formatRecordingTime(seconds: number): string { + if (!Number.isFinite(seconds) || seconds < 0) return '0:00' + const mins = Math.floor(seconds / 60) + const secs = Math.floor(seconds % 60) + return `${mins}:${secs.toString().padStart(2, '0')}` +} + +export function formatVolumePercent(volume: number): string { + return `${Math.round(volume * 100)}%` +} + +interface UseRecordingAudioPlayerOptions { + src?: string | null + audioRef?: Ref + onTimeUpdate?: (currentTime: number) => void + onLoadedMetadata?: (duration: number) => void + onEnded?: () => void +} + +function assignAudioRef( + ref: Ref | undefined, + element: HTMLAudioElement | null, +) { + if (!ref) return + if (typeof ref === 'function') { + ref(element) + return + } + ;(ref as MutableRefObject).current = element +} + +export function useRecordingAudioPlayer({ + src, + audioRef, + onTimeUpdate, + onLoadedMetadata, + onEnded, +}: UseRecordingAudioPlayerOptions) { + const internalRef = useRef(null) + const [isPlaying, setIsPlaying] = useState(false) + const [isLoading, setIsLoading] = useState(false) + const [currentTime, setCurrentTime] = useState(0) + const [duration, setDuration] = useState(0) + const [volume, setVolumeState] = useState(DEFAULT_VOLUME) + const [playbackRate, setPlaybackRateState] = useState(1) + + const setAudioElementRef = useCallback( + (element: HTMLAudioElement | null) => { + internalRef.current = element + assignAudioRef(audioRef, element) + }, + [audioRef], + ) + + const getAudio = useCallback(() => internalRef.current, []) + + const togglePlay = useCallback(async () => { + const audio = internalRef.current + if (!audio || !src) return + + if (audio.paused) { + setIsLoading(true) + try { + await audio.play() + setIsPlaying(true) + } catch { + setIsPlaying(false) + } finally { + setIsLoading(false) + } + } else { + audio.pause() + setIsPlaying(false) + } + }, [src]) + + const seek = useCallback((next: number) => { + const audio = internalRef.current + if (!audio || !Number.isFinite(audio.duration)) return + const clamped = Math.max(0, Math.min(audio.duration, next)) + audio.currentTime = clamped + setCurrentTime(clamped) + }, []) + + const setVolume = useCallback((next: number) => { + const clamped = Math.max(0, Math.min(1, next)) + setVolumeState(clamped) + if (internalRef.current) { + internalRef.current.volume = clamped + } + }, []) + + const setPlaybackRate = useCallback((next: PlaybackRate) => { + setPlaybackRateState(next) + if (internalRef.current) { + internalRef.current.playbackRate = next + } + }, []) + + useEffect(() => { + const audio = internalRef.current + if (!audio) return + + const handleLoadedMetadata = () => { + const nextDuration = Number.isFinite(audio.duration) ? audio.duration : 0 + setDuration(nextDuration) + setCurrentTime(audio.currentTime || 0) + onLoadedMetadata?.(nextDuration) + } + + const handleTimeUpdate = () => { + setCurrentTime(audio.currentTime || 0) + onTimeUpdate?.(audio.currentTime || 0) + } + + const handleEnded = () => { + setIsPlaying(false) + onEnded?.() + } + + const handlePlay = () => setIsPlaying(true) + const handlePause = () => setIsPlaying(false) + const handleWaiting = () => setIsLoading(true) + const handleCanPlay = () => setIsLoading(false) + + audio.addEventListener('loadedmetadata', handleLoadedMetadata) + audio.addEventListener('timeupdate', handleTimeUpdate) + audio.addEventListener('ended', handleEnded) + audio.addEventListener('play', handlePlay) + audio.addEventListener('pause', handlePause) + audio.addEventListener('waiting', handleWaiting) + audio.addEventListener('canplay', handleCanPlay) + + return () => { + audio.removeEventListener('loadedmetadata', handleLoadedMetadata) + audio.removeEventListener('timeupdate', handleTimeUpdate) + audio.removeEventListener('ended', handleEnded) + audio.removeEventListener('play', handlePlay) + audio.removeEventListener('pause', handlePause) + audio.removeEventListener('waiting', handleWaiting) + audio.removeEventListener('canplay', handleCanPlay) + } + }, [onEnded, onLoadedMetadata, onTimeUpdate, src]) + + useEffect(() => { + setIsPlaying(false) + setIsLoading(false) + setCurrentTime(0) + setDuration(0) + }, [src]) + + useEffect(() => { + const audio = internalRef.current + if (!audio) return + audio.volume = volume + audio.playbackRate = playbackRate + }, [playbackRate, volume, src]) + + return { + setAudioElementRef, + getAudio, + isPlaying, + isLoading, + currentTime, + duration, + volume, + playbackRate, + togglePlay, + seek, + setVolume, + setPlaybackRate, + canSeek: Boolean(src) && duration > 0 && !isLoading, + } +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index c432705b..5df0804b 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -429,18 +429,27 @@ export interface VobizOutboundCallResponse { export interface EvaluatorSuiteCombination { id: string evaluator_id: string + persona_id?: string + persona_name?: string | null scenario_id: string scenario_name?: string | null scenario_description?: string | null scenario_required_info?: Record | null } +export interface EvaluatorSuitePersonaSummary { + id: string + name?: string | null +} + export interface EvaluatorSuite { id: string organization_id: string name?: string | null agent_id: string persona_id: string + persona_ids?: string[] + personas?: EvaluatorSuitePersonaSummary[] agent_name?: string | null persona_name?: string | null agent_call_type?: string | null @@ -468,6 +477,8 @@ export interface RunEvaluatorSuiteResponse { export interface RunNextCombinationResponse { evaluator_id: string + persona_id?: string | null + persona_name?: string | null scenario_id: string scenario_name: string combination_index: number @@ -481,6 +492,8 @@ export interface RunNextCombinationResponse { export interface ChooseNextCombinationResponse { evaluator_id: string + persona_id?: string | null + persona_name?: string | null scenario_id: string scenario_name: string combination_index: number @@ -1213,6 +1226,21 @@ class ApiClient { ): Promise<{ sections: Array<{ key: string; title: string; content: string }> test_agent_prompt: string + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + test_agent_template: { + sections: Array<{ key: string; title: string; content: string }> + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + } provider: string model: string }> { @@ -1383,6 +1411,82 @@ class ApiClient { return response.data } + async listAmbientPresets(): Promise<{ presets: { id: string; label: string }[] }> { + const response = await this.client.get('/api/v1/personas/ambient-presets') + return response.data + } + + async listAmbientLibrary(): Promise< + Array<{ + id: string + name: string + original_filename?: string | null + created_at?: string + }> + > { + const response = await this.client.get('/api/v1/personas/ambient-library') + return response.data + } + + async uploadAmbientLibraryAsset(file: File, name?: string): Promise { + const formData = new FormData() + formData.append('file', file) + if (name) { + formData.append('name', name) + } + const response = await this.client.post('/api/v1/personas/ambient-library', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async updateAmbientLibraryAsset(assetId: string, data: { name: string }): Promise { + const response = await this.client.patch(`/api/v1/personas/ambient-library/${assetId}`, data) + return response.data + } + + async deleteAmbientLibraryAsset(assetId: string): Promise { + await this.client.delete(`/api/v1/personas/ambient-library/${assetId}`) + } + + async previewAmbientLibraryAsset(assetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async getAmbientLibraryPreviewUrl( + assetId: string, + expiration: number = 3600, + ): Promise<{ url: string; expires_in: number }> { + const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview-url`, { + params: { expiration }, + }) + return response.data + } + + async previewAmbientPreset(presetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-presets/${presetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async uploadPersonaAmbientAudio(personaId: string, file: File): Promise { + const formData = new FormData() + formData.append('file', file) + const response = await this.client.post(`/api/v1/personas/${personaId}/ambient-audio`, formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async deletePersonaAmbientAudio(personaId: string): Promise { + const response = await this.client.delete(`/api/v1/personas/${personaId}/ambient-audio`) + return response.data + } + // Scenarios endpoints async listScenarios(skip = 0, limit = 100, agentId?: string): Promise { const response = await this.client.get('/api/v1/scenarios', { @@ -4105,7 +4209,8 @@ class ApiClient { async createEvaluatorSuite(data: { name?: string agent_id: string - persona_id: string + persona_id?: string + persona_ids?: string[] scenario_ids: string[] metric_ids?: string[] llm_provider?: string @@ -4156,6 +4261,27 @@ class ApiClient { return response.data } + async addEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { + const response = await this.client.post(`/api/v1/evaluator-suites/${suiteId}/personas`, { + persona_ids: personaIds, + }) + return response.data + } + + async replaceEvaluatorSuitePersonas(suiteId: string, personaIds: string[]): Promise { + const response = await this.client.put(`/api/v1/evaluator-suites/${suiteId}/personas`, { + persona_ids: personaIds, + }) + return response.data + } + + async removeEvaluatorSuitePersona(suiteId: string, personaId: string): Promise { + const response = await this.client.delete( + `/api/v1/evaluator-suites/${suiteId}/personas/${personaId}`, + ) + return response.data + } + async deleteEvaluatorSuite(suiteId: string): Promise { await this.client.delete(`/api/v1/evaluator-suites/${suiteId}`) } @@ -4442,6 +4568,8 @@ class ApiClient { if (p.suiteId) params.suite_id = p.suiteId if (p.scenarioId) params.scenario_id = p.scenarioId if (p.status) params.status = p.status + if (p.since) params.since = p.since + if (p.until) params.until = p.until if (p.unassignedOnly) params.unassigned_only = true if (p.playground !== undefined) params.playground = p.playground if (p.testAgentsOnly !== undefined) params.test_agents_only = p.testAgentsOnly @@ -4453,10 +4581,14 @@ class ApiClient { async getEvaluatorResultsOverview(params?: { agentId?: string suiteId?: string + since?: string + until?: string }): Promise { const query: Record = {} if (params?.agentId) query.agent_id = params.agentId if (params?.suiteId) query.suite_id = params.suiteId + if (params?.since) query.since = params.since + if (params?.until) query.until = params.until const response = await this.client.get('/api/v1/evaluator-results/overview', { params: query }) return response.data } @@ -4465,15 +4597,107 @@ class ApiClient { suiteId?: string agentId?: string scenarioId?: string + since?: string + until?: string }): Promise { const query: Record = {} if (params.suiteId) query.suite_id = params.suiteId if (params.agentId) query.agent_id = params.agentId if (params.scenarioId) query.scenario_id = params.scenarioId + if (params.since) query.since = params.since + if (params.until) query.until = params.until const response = await this.client.get('/api/v1/evaluator-results/aggregate', { params: query }) return response.data } + private _evaluatorResultClusterQuery(scope: { + agentId?: string + suiteId?: string + scenarioId?: string + }): Record { + const query: Record = {} + if (scope.agentId) query.agent_id = scope.agentId + if (scope.suiteId) query.suite_id = scope.suiteId + if (scope.scenarioId) query.scenario_id = scope.scenarioId + return query + } + + async getEvaluatorResultMetricClusterFailurePolicies(scope: { + agentId?: string + suiteId?: string + scenarioId?: string + }): Promise { + const response = await this.client.get( + '/api/v1/evaluator-results/metric-clusters/failure-policies', + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + + async saveEvaluatorResultMetricClusterFailurePolicies( + scope: { agentId?: string; suiteId?: string; scenarioId?: string }, + policies: Record, + ): Promise { + const response = await this.client.put( + '/api/v1/evaluator-results/metric-clusters/failure-policies', + { policies }, + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + + async listEvaluatorResultMetricClusterEligibleRows( + scope: { agentId?: string; suiteId?: string; scenarioId?: string }, + options?: { limit?: number; count_only?: boolean }, + ): Promise { + const response = await this.client.get( + '/api/v1/evaluator-results/metric-clusters/eligible-rows', + { params: { ...this._evaluatorResultClusterQuery(scope), ...options } }, + ) + return response.data + } + + async getEvaluatorResultMetricClusters(scope: { + agentId?: string + suiteId?: string + scenarioId?: string + }): Promise { + const response = await this.client.get('/api/v1/evaluator-results/metric-clusters', { + params: this._evaluatorResultClusterQuery(scope), + }) + return response.data + } + + async generateEvaluatorResultMetricClusters( + scope: { agentId?: string; suiteId?: string; scenarioId?: string }, + body: { + force?: boolean + regenerate?: boolean + provider?: string + model?: string + row_limit?: number + failure_policies?: Record + }, + ): Promise { + const response = await this.client.post('/api/v1/evaluator-results/metric-clusters', body, { + params: this._evaluatorResultClusterQuery(scope), + }) + return response.data + } + + async cancelEvaluatorResultMetricClusters(scope: { + agentId?: string + suiteId?: string + scenarioId?: string + }): Promise { + const response = await this.client.post( + '/api/v1/evaluator-results/metric-clusters/cancel', + {}, + { params: this._evaluatorResultClusterQuery(scope) }, + ) + return response.data + } + async getEvaluatorResult(id: string, includeRelations: boolean = true): Promise { const params = includeRelations ? { include_relations: 'true' } : {} const response = await this.client.get(`/api/v1/evaluator-results/${id}`, { params }) diff --git a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx index feba5879..d8247847 100644 --- a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx +++ b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx @@ -13,6 +13,12 @@ import AgentEditForm from './components/AgentEditForm' import AgentTalkSidebar, { type AgentTalkMode } from './components/AgentTalkSidebar' import { Save, X } from 'lucide-react' import { extractPhoneConflictDetail } from './components/agentPhoneValidation' +import { + TestAgentTemplateDraft, + assembleTestAgentPrompt, + defaultTestAgentTemplate, + templateFromApi, +} from './components/agentTestSetupConstants' const VALID_TABS: AgentDetailTab[] = ['overview', 'test_agent', 'voice_ai_agent'] @@ -33,6 +39,7 @@ interface FormData { phone_number: string language: string description: string + test_agent_template: TestAgentTemplateDraft prompt_variables: Record silence_hangup_secs: number call_type: string @@ -91,6 +98,7 @@ export default function AgentWorkspaceDetail({ phone_number: '', language: 'en', description: '', + test_agent_template: defaultTestAgentTemplate(), prompt_variables: {}, silence_hangup_secs: 15, call_type: 'outbound', @@ -140,6 +148,9 @@ export default function AgentWorkspaceDetail({ phone_number: agent.phone_number || '', language: agent.language, description: agent.description || '', + test_agent_template: agent.test_agent_template + ? templateFromApi(agent.test_agent_template) + : defaultTestAgentTemplate(), prompt_variables: agent.prompt_variables || {}, silence_hangup_secs: agent.silence_hangup_secs ?? 15, call_type: agent.call_type, @@ -160,7 +171,10 @@ export default function AgentWorkspaceDetail({ language: data.language, call_type: data.call_type, call_medium: data.call_medium, - description: data.description?.trim() || null, + description: + data.description?.trim() || + assembleTestAgentPrompt(data.test_agent_template.sections), + test_agent_template: data.test_agent_template, prompt_variables: data.prompt_variables || {}, silence_hangup_secs: data.silence_hangup_secs ?? 15, } @@ -279,6 +293,9 @@ export default function AgentWorkspaceDetail({ phone_number: agent.phone_number || '', language: agent.language, description: agent.description || '', + test_agent_template: agent.test_agent_template + ? templateFromApi(agent.test_agent_template) + : defaultTestAgentTemplate(), prompt_variables: agent.prompt_variables || {}, silence_hangup_secs: agent.silence_hangup_secs ?? 15, call_type: agent.call_type, @@ -370,10 +387,10 @@ export default function AgentWorkspaceDetail({ return (
-
-
+
+

@@ -421,7 +438,7 @@ export default function AgentWorkspaceDetail({

-
+
{!isEditMode ? ( silence_hangup_secs: number call_type: string @@ -71,13 +77,9 @@ export default function AgentEditForm({ agentId, }: AgentEditFormProps) { const navigate = useNavigate() - const [descriptionEditorMode, setDescriptionEditorMode] = useState<'write' | 'preview'>('write') const [providerPromptEditorMode, setProviderPromptEditorMode] = useState<'write' | 'preview'>('write') - const [showAIGeneratePanel, setShowAIGeneratePanel] = useState(false) - const [includeLinkedScenarios, setIncludeLinkedScenarios] = useState(true) - const [aiDescription, setAiDescription] = useState('') - const [aiTone, setAiTone] = useState('professional') - const [aiFormat, setAiFormat] = useState('structured') + const [showGenerateFromProductionPanel, setShowGenerateFromProductionPanel] = useState(false) + const [setupAdditionalContext, setSetupAdditionalContext] = useState('') const [aiCredentialId, setAiCredentialId] = useState('') const [aiModel, setAiModel] = useState('') const [phoneNumberInputMode, setPhoneNumberInputMode] = useState<'provider' | 'custom'>('provider') @@ -175,29 +177,43 @@ export default function AgentEditForm({ telephonyNumbers, ]) - const generateDescriptionMutation = useMutation({ - mutationFn: (data: { - description: string - tone?: string - format_style?: string - provider?: string - model?: string - agent_id?: string - include_linked_scenarios?: boolean - append_scenarios_to_output?: boolean - }) => apiClient.generateAgentDescription(data), + const generateFromProductionMutation = useMutation({ + mutationFn: () => { + if (!formData.provider_prompt?.trim()) { + throw new Error('Production prompt is required') + } + return apiClient.generateTestPromptFromProduction({ + production_prompt: formData.provider_prompt, + agent_name: formData.name, + language: formData.language, + call_type: formData.call_type, + additional_context: setupAdditionalContext.trim() || undefined, + ...(aiProvider ? { provider: aiProvider } : {}), + ...(aiCredentialId ? { credential_id: aiCredentialId } : {}), + ...(aiModel ? { model: aiModel } : {}), + }) + }, onSuccess: (data) => { - onChange({ ...formData, description: data.content }) - setShowAIGeneratePanel(false) - setAiDescription('') - setDescriptionEditorMode('preview') - showToast('Description generated successfully!', 'success') + const nextTemplate = applyGeneratedTemplate(formData.test_agent_template, data) + onChange({ + ...formData, + test_agent_template: nextTemplate, + description: data.test_agent_prompt || assembleTestAgentPrompt(nextTemplate.sections), + }) + setShowGenerateFromProductionPanel(false) + showToast('Test agent template generated from production prompt', 'success') }, onError: (err: any) => { - showToast(err?.response?.data?.detail || 'Failed to generate description with AI', 'error') + showToast( + err?.message || err?.response?.data?.detail || 'Failed to generate from production', + 'error', + ) }, }) + const hasStructuredTemplate = isTemplateFilled(formData.test_agent_template) + const showLegacyPrompt = Boolean(formData.description?.trim() && !hasStructuredTemplate) + const voiceAgentIntegrations = integrations.filter( (integration) => integration.is_active && @@ -212,6 +228,29 @@ export default function AgentEditForm({ ? voiceBundles.find((vb) => vb.id === formData.voice_bundle_id) : undefined + const testAgentConfigured = Boolean( + linkedVoiceBundle && linkedVoiceBundle.is_active !== false, + ) + const voiceAiConfigured = Boolean( + formData.voice_ai_integration_id?.trim() && formData.voice_ai_agent_id?.trim(), + ) + const voiceBundleLabel = linkedVoiceBundle + ? linkedVoiceBundle.name + : formData.voice_bundle_id + ? 'Unknown bundle' + : OVERVIEW_NOT_CONFIGURED + const voiceAiIntegrationLabel = selectedVoiceIntegration + ? (() => { + const platformLabel = getIntegrationPlatformLabel( + selectedVoiceIntegration.platform as IntegrationPlatform, + ) + const name = selectedVoiceIntegration.name?.trim() + return name ? `${name} (${platformLabel})` : platformLabel + })() + : formData.voice_ai_integration_id + ? 'Unknown integration' + : OVERVIEW_NOT_CONFIGURED + const hasPlatformLink = Boolean(formData.voice_ai_integration_id || formData.voice_ai_agent_id) const productionPromptProse = @@ -220,7 +259,16 @@ export default function AgentEditForm({ return (
{activeTab === 'overview' && ( -
+
+
+

Overview

+

+ Identity, call routing, voice stacks, and session behavior. +

+
+ +
+
@@ -460,11 +508,53 @@ export default function AgentEditForm({ Delete agent
+
+ +
+ +
+ } + /> + +
+ + + +
+ } + /> + + + {formData.voice_ai_agent_id.trim()} + + ) : ( + OVERVIEW_NOT_CONFIGURED + ) + } + /> +
+ +
+
)} {activeTab === 'test_agent' && ( -
+
{testAgentSubTab === 'configuration' && ( @@ -494,7 +584,7 @@ export default function AgentEditForm({ {linkedVoiceBundle && ( { const returnPath = agentId ? `/agents/${agentId}?tab=test_agent` : '/agents' navigate( @@ -508,137 +598,69 @@ export default function AgentEditForm({ )} {testAgentSubTab === 'prompt' && ( -
-
- +
+
+
-
- - -
- {/* AI Generate Panel */} - {showAIGeneratePanel && ( -
-
- - Generate Description with AI -
-

- Describe what this agent should do and AI will generate a rich markdown description. + {showGenerateFromProductionPanel && ( +

+

+ Uses the production agent prompt from the Voice AI Agent tab to generate complementary + caller sections and first-message settings.

+ {!formData.provider_prompt?.trim() ? ( +

+ Add a production prompt on the Voice AI Agent tab first. +

+ ) : null}