From d725d95e849b0fd7dd4ddeb961248ee14e3cf579 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 22 Aug 2026 20:30:33 +0000 Subject: [PATCH 1/6] feat: updating noises --- app/api/v1/routes/agents.py | 69 ++- app/api/v1/routes/evaluator_results.py | 35 +- app/api/v1/routes/personas.py | 449 +++++++++++++++++- app/api/v1/routes/vobiz_telephony.py | 1 + app/api/v1/routes/voice_agent.py | 169 ++++--- .../078_add_test_agent_template_to_agents.py | 28 ++ app/migrations/079_persona_ambient_noise.py | 32 ++ app/migrations/080_ambient_noise_library.py | 52 ++ app/models/database.py | 30 ++ app/models/enums.py | 11 +- app/models/schemas.py | 98 +++- app/services/audio/ambient_catalog.py | 250 ++++++++++ app/services/audio/ambient_mic_pump.py | 95 ++++ app/services/audio/ambient_mixer.py | 163 +++++++ .../evaluators/evaluator_results_query.py | 5 +- app/services/media_urls.py | 3 + app/services/personas/ambient_library.py | 49 ++ .../personas/persona_ambient_noise.py | 129 +++++ .../testing/agent_test_setup_generation.py | 226 +++++---- .../testing/test_agent_bridge_service.py | 79 ++- .../testing/test_agent_simulation_prompt.py | 96 +++- app/services/testing/test_agent_template.py | 249 ++++++++++ app/services/voice_agent/bot_fast_api.py | 8 +- .../voice_agent/llm_voice_providers.py | 339 +++++++++++++ app/services/voice_agent/voice_bundle.py | 109 +++-- .../webrtc_bridge/elevenlabs_ws_bridge.py | 17 +- .../webrtc_bridge/test_agent_processor.py | 12 +- frontend/src/components/VoiceAgent.tsx | 31 +- .../TestVoiceAgentResultDetails.tsx | 2 +- frontend/src/lib/api.ts | 81 ++++ .../src/pages/agents/AgentWorkspaceDetail.tsx | 27 +- .../pages/agents/components/AgentEditForm.tsx | 381 ++++++++------- .../pages/agents/components/AgentInfoView.tsx | 83 +++- .../agents/components/CreateAgentModal.tsx | 44 +- .../components/TestAgentTemplateEditor.tsx | 297 ++++++++++++ .../components/VoiceBundleDetailCard.tsx | 21 +- .../components/VoiceBundleParamsModal.tsx | 154 +++++- .../components/agentTestSetupConstants.ts | 155 +++++- .../create/ProductionPromptStep.tsx | 152 ++---- .../components/create/createAgentTypes.ts | 5 + .../personas/AmbientNoiseLibraryPanel.tsx | 215 +++++++++ .../pages/personas/AmbientPreviewControls.tsx | 88 ++++ .../pages/personas/PersonaAmbientPanel.tsx | 274 +++++++++++ .../src/pages/personas/PersonaTabContent.tsx | 24 + frontend/src/pages/personas/PersonaTile.tsx | 2 +- frontend/src/pages/personas/Personas.tsx | 51 +- .../personas/UploadAmbientNoiseModal.tsx | 255 ++++++++++ frontend/src/pages/personas/personaTypes.ts | 46 +- .../src/pages/personas/useAmbientPreview.ts | 146 ++++++ .../playground/agent/AgentPlayground.tsx | 221 ++++++++- .../agent/TestAgentResultDetail.tsx | 65 ++- frontend/src/types/api.ts | 13 + .../test_agents_generate_test_setup.py | 52 +- tests/test_services/test_audio/__init__.py | 0 .../test_audio/test_ambient_mixer.py | 72 +++ .../test_agent_simulation_prompt.py | 27 +- .../test_testing/test_agent_template.py | 81 ++++ .../test_agent_template_opening.py | 53 +++ .../test_agent_test_setup_generation.py | 41 +- .../test_llm_voice_providers.py | 47 ++ 60 files changed, 5289 insertions(+), 720 deletions(-) create mode 100644 app/migrations/078_add_test_agent_template_to_agents.py create mode 100644 app/migrations/079_persona_ambient_noise.py create mode 100644 app/migrations/080_ambient_noise_library.py create mode 100644 app/services/audio/ambient_catalog.py create mode 100644 app/services/audio/ambient_mic_pump.py create mode 100644 app/services/audio/ambient_mixer.py create mode 100644 app/services/personas/ambient_library.py create mode 100644 app/services/personas/persona_ambient_noise.py create mode 100644 app/services/testing/test_agent_template.py create mode 100644 app/services/voice_agent/llm_voice_providers.py create mode 100644 frontend/src/pages/agents/components/TestAgentTemplateEditor.tsx create mode 100644 frontend/src/pages/personas/AmbientNoiseLibraryPanel.tsx create mode 100644 frontend/src/pages/personas/AmbientPreviewControls.tsx create mode 100644 frontend/src/pages/personas/PersonaAmbientPanel.tsx create mode 100644 frontend/src/pages/personas/UploadAmbientNoiseModal.tsx create mode 100644 frontend/src/pages/personas/useAmbientPreview.ts create mode 100644 tests/test_services/test_audio/__init__.py create mode 100644 tests/test_services/test_audio/test_ambient_mixer.py create mode 100644 tests/test_services/test_testing/test_agent_template.py create mode 100644 tests/test_services/test_testing/test_agent_template_opening.py create mode 100644 tests/test_services/test_voice_agent/test_llm_voice_providers.py diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 17094877..d7c84d06 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -33,11 +33,57 @@ GenerateTestSetupResponse, GeneratedScenarioDraftResponse, TestPromptSectionResponse, + TestAgentFirstMessageResponse, + TestAgentTemplateResponse, + TestAgentTemplateInput, +) +from app.services.testing.test_agent_template import ( + TestAgentFirstMessage, + TestAgentTemplate, + assemble_test_agent_prompt, + normalize_first_message, + normalize_sections, + template_from_generation, ) router = APIRouter(prefix="/agents", tags=["agents"]) +def _first_message_response(first_message: TestAgentFirstMessage) -> TestAgentFirstMessageResponse: + return TestAgentFirstMessageResponse( + production_mode=first_message.production_mode, + production_message=first_message.production_message, + caller_mode=first_message.caller_mode, + caller_message=first_message.caller_message, + ) + + +def _template_response(template: TestAgentTemplate) -> TestAgentTemplateResponse: + return TestAgentTemplateResponse( + sections=_test_prompt_section_responses(template.sections), + first_message=_first_message_response(template.first_message), + ) + + +def _template_input_to_storage(template_input: TestAgentTemplateInput) -> dict: + sections = normalize_sections([s.model_dump() for s in template_input.sections]) + first_message = normalize_first_message(template_input.first_message.model_dump()) + return template_from_generation(sections, first_message).to_dict() + + +def _apply_test_agent_template_fields( + *, + description: Optional[str], + template_input: Optional[TestAgentTemplateInput], +) -> tuple[Optional[str], Optional[dict]]: + """Return (description, test_agent_template_json) for persistence.""" + if template_input is None: + return description, None + template_dict = _template_input_to_storage(template_input) + assembled = assemble_test_agent_prompt(normalize_sections(template_dict.get("sections"))) + return assembled or description, template_dict + + def _validate_agent_phone_assignment( db: Session, *, @@ -130,7 +176,7 @@ class GenerateAgentDescriptionRequest(BaseModel): "well-formatted agent description in markdown.\n\n" "Guidelines:\n" "- Use clear markdown structure: headings, bullet points, numbered lists\n" - "- Include sections for: Purpose, Behavior, Expected Interactions, Personality Traits, and Constraints\n" + "- Include sections for: Role and Goal, Talking Style, Questions to Ask, Information to Relay, and Constraints\n" "- Be specific about the agent's role, tone of voice, and how it should handle conversations\n" "- Include example scenarios or edge cases where helpful\n" "- Return ONLY the description in markdown, no preamble or explanation about what you did" @@ -300,6 +346,8 @@ async def generate_test_prompt( return GenerateTestPromptResponse( sections=_test_prompt_section_responses(result.sections), test_agent_prompt=result.test_agent_prompt, + first_message=_first_message_response(result.first_message), + test_agent_template=_template_response(result.test_agent_template), provider=result.provider, model=result.model, ) @@ -433,6 +481,8 @@ async def generate_test_setup( return GenerateTestSetupResponse( sections=_test_prompt_section_responses(prompt_result.sections), test_agent_prompt=prompt_result.test_agent_prompt, + first_message=_first_message_response(prompt_result.first_message), + test_agent_template=_template_response(prompt_result.test_agent_template), scenarios=_scenario_draft_responses(scenario_result.scenarios), provider=prompt_result.provider, model=prompt_result.model, @@ -575,6 +625,11 @@ async def create_agent( # Generate unique 6-digit agent_id agent_id = generate_unique_agent_id(db) + + description, template_dict = _apply_test_agent_template_fields( + description=agent.description, + template_input=agent.test_agent_template, + ) db_agent = Agent( agent_id=agent_id, @@ -583,7 +638,8 @@ async def create_agent( name=agent.name, phone_number=agent.phone_number, language=agent.language, - description=agent.description, + description=description, + test_agent_template=template_dict, call_type=agent.call_type, call_medium=agent.call_medium, telephony_phone_number_id=agent.telephony_phone_number_id, @@ -813,6 +869,15 @@ async def update_agent( update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False) + if "test_agent_template" in update_data: + template_input = agent_update.test_agent_template + assembled_description, template_dict = _apply_test_agent_template_fields( + description=update_data.get("description", db_agent.description), + template_input=template_input, + ) + update_data["description"] = assembled_description + update_data["test_agent_template"] = template_dict + effective_call_medium = ( agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium ) diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index 962207c6..66e5a4ed 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -865,10 +865,37 @@ def re_evaluate_result( ) if not result.evaluator_id: - raise HTTPException( - status_code=400, - detail="Cannot re-evaluate: this result is not linked to an evaluator." - ) + if result.agent_id and result.persona_id and result.scenario_id: + from app.api.v1.routes.evaluators import generate_unique_evaluator_id + + evaluator = db.query(Evaluator).filter( + Evaluator.agent_id == result.agent_id, + Evaluator.persona_id == result.persona_id, + Evaluator.scenario_id == result.scenario_id, + Evaluator.organization_id == organization_id, + Evaluator.workspace_id == workspace_id, + ).first() + if not evaluator: + new_evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=new_evaluator_id, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=result.agent_id, + persona_id=result.persona_id, + scenario_id=result.scenario_id, + tags=["auto-created", "test-voice-agent"], + ) + db.add(evaluator) + db.commit() + db.refresh(evaluator) + result.evaluator_id = evaluator.id + db.commit() + else: + raise HTTPException( + status_code=400, + detail="Cannot re-evaluate: this result is not linked to an evaluator." + ) evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() if not evaluator: diff --git a/app/api/v1/routes/personas.py b/app/api/v1/routes/personas.py index de07c715..82cb43ca 100644 --- a/app/api/v1/routes/personas.py +++ b/app/api/v1/routes/personas.py @@ -3,8 +3,8 @@ CRUD for TTS provider-tied voice personas, voice-options catalog, and custom voice management (ungated). """ -from fastapi import APIRouter, Depends, HTTPException, status, Body, Query -from fastapi.responses import JSONResponse +from fastapi import APIRouter, Depends, HTTPException, status, Body, Query, UploadFile, File, Form +from fastapi.responses import Response from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any @@ -12,15 +12,17 @@ from pydantic import BaseModel from loguru import logger -from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key +from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key, require_enterprise_entitlement from app.models.database import ( Persona, Evaluator, EvaluatorResult, TestAgentConversation, CustomTTSVoice, - PromptOptimizationRun, CallRecording, Agent, + PromptOptimizationRun, CallRecording, Agent, AmbientNoiseAsset, ) -from app.models.enums import LanguageEnum, AccentEnum, GenderEnum, BackgroundNoiseEnum +from app.models.enums import LanguageEnum, AccentEnum, GenderEnum, BackgroundNoiseEnum, BackgroundNoiseSourceEnum from app.models.schemas import ( PersonaCreate, PersonaUpdate, PersonaResponse, PersonaCloneRequest, AgentPromptSourcesResponse, GeneratePersonaPromptRequest, GeneratePersonaPromptResponse, + AmbientNoiseAssetResponse, + AmbientNoiseAssetUpdateRequest, ) from app.models.enums import ModelProvider from app.services.ai.model_config_service import model_config_service @@ -33,6 +35,22 @@ generate_persona_prompt_from_agent, resolve_agent_prompt_sources, ) +from app.services.personas.persona_ambient_noise import ( + ALLOWED_AMBIENT_EXTENSIONS, + MAX_AMBIENT_UPLOAD_BYTES, + persona_ambient_s3_key, + validate_persona_ambient_fields, +) +from app.services.personas.ambient_library import ( + ambient_library_s3_key, + new_ambient_asset_id, + sanitize_ambient_name, + validate_ambient_upload_bytes, +) +from app.services.audio.ambient_catalog import get_ambient_asset_provider, list_ambient_presets, normalize_ambient_preset +from app.services.audio.ambient_mixer import decode_audio_bytes_to_pcm_int16 +from app.services.storage.s3_service import s3_service, StorageError +from fastapi.responses import JSONResponse router = APIRouter(prefix="/personas", tags=["personas"]) @@ -41,6 +59,85 @@ def _normalized_persona_tts_config(provider: Optional[str], tts_config: Optional return normalize_persona_tts_config(provider, tts_config) +def _ambient_fields_from_create( + persona: PersonaCreate, + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, +) -> Dict[str, Any]: + return validate_persona_ambient_fields( + source=persona.background_noise_source.value, + preset=persona.background_noise_preset, + volume=persona.background_noise_volume, + s3_key=None, + asset_id=persona.background_noise_asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + require_custom_file=( + persona.background_noise_source == BackgroundNoiseSourceEnum.CUSTOM + and not persona.background_noise_asset_id + ), + ) + + +def _apply_ambient_update( + db_persona: Persona, + update_data: Dict[str, Any], + organization_id: UUID, + workspace_id: UUID, + db: Session, +) -> Dict[str, Any]: + if not any( + key in update_data + for key in ( + "background_noise_source", + "background_noise_preset", + "background_noise_volume", + "background_noise_asset_id", + ) + ): + return update_data + + source = update_data.get( + "background_noise_source", + db_persona.background_noise_source or BackgroundNoiseSourceEnum.NONE.value, + ) + if hasattr(source, "value"): + source = source.value + preset = update_data.get("background_noise_preset", db_persona.background_noise_preset) + volume = update_data.get("background_noise_volume", db_persona.background_noise_volume) + asset_id = update_data.get("background_noise_asset_id", db_persona.background_noise_asset_id) + s3_key = db_persona.background_noise_s3_key + + validated = validate_persona_ambient_fields( + source=source, + preset=preset, + volume=volume, + s3_key=s3_key, + asset_id=asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + require_custom_file=( + str(source).lower() == BackgroundNoiseSourceEnum.CUSTOM.value + and not asset_id + and not s3_key + ), + ) + update_data["background_noise_source"] = validated["background_noise_source"] + update_data["background_noise_preset"] = validated["background_noise_preset"] + update_data["background_noise_volume"] = validated["background_noise_volume"] + update_data["background_noise_asset_id"] = validated["background_noise_asset_id"] + if validated["background_noise_source"] != BackgroundNoiseSourceEnum.CUSTOM.value: + update_data["background_noise_s3_key"] = None + update_data["background_noise_asset_id"] = None + else: + update_data["background_noise_s3_key"] = validated["background_noise_s3_key"] + return update_data + + def _get_agent_for_workspace( db: Session, *, @@ -188,10 +285,12 @@ def _is_valid_persona_row(persona: Persona) -> bool: accent_value = str(getattr(persona, "accent", "neutral") or "neutral").lower() gender_value = str(getattr(persona, "gender", "neutral") or "neutral").lower() noise_value = str(getattr(persona, "background_noise", "none") or "none").lower() + source_value = str(getattr(persona, "background_noise_source", "none") or "none").lower() LanguageEnum(language_value) AccentEnum(accent_value) GenderEnum(gender_value) BackgroundNoiseEnum(noise_value) + BackgroundNoiseSourceEnum(source_value) return True except Exception: return False @@ -206,6 +305,12 @@ async def create_persona( ): """Create a new persona stamped with the active workspace.""" try: + ambient_fields = _ambient_fields_from_create( + persona, + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + ) db_persona = Persona( organization_id=organization_id, workspace_id=workspace_id, @@ -222,6 +327,10 @@ async def create_persona( response_delay_ms=persona.response_delay_ms, max_turns=persona.max_turns, allow_interruptions=persona.allow_interruptions, + background_noise_source=ambient_fields["background_noise_source"], + background_noise_preset=ambient_fields["background_noise_preset"], + background_noise_volume=ambient_fields["background_noise_volume"], + background_noise_asset_id=ambient_fields["background_noise_asset_id"], ) db.add(db_persona) db.commit() @@ -584,6 +693,218 @@ async def delete_custom_voice( return {"message": "Custom voice deleted"} +# ============================================ +# AMBIENT NOISE (static paths before /{persona_id}) +# ============================================ + +def _guess_audio_media_type(filename: Optional[str], fallback: str = "audio/wav") -> str: + if not filename: + return fallback + ext = filename.rsplit(".", 1)[-1].lower() + return { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "ogg": "audio/ogg", + "m4a": "audio/mp4", + "flac": "audio/flac", + }.get(ext, fallback) + + +@router.get("/ambient-presets", operation_id="listAmbientPresets") +async def list_platform_ambient_presets( + api_key: str = Depends(get_api_key), +): + """List platform ambient presets available from installed asset packs.""" + return {"presets": list_ambient_presets()} + + +@router.get( + "/ambient-presets/{preset_id}/preview", + operation_id="previewAmbientPreset", +) +async def preview_ambient_preset( + preset_id: str, + api_key: str = Depends(get_api_key), +): + """Stream a platform preset for in-browser preview.""" + normalized = normalize_ambient_preset(preset_id) or preset_id + provider = get_ambient_asset_provider() + try: + file_bytes = provider.load_wav(normalized) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=f"Preset '{preset_id}' is not available") from exc + return Response(content=file_bytes, media_type=_guess_audio_media_type(f"{normalized}.wav")) + + +@router.get( + "/ambient-library", + response_model=List[AmbientNoiseAssetResponse], + operation_id="listAmbientLibrary", +) +async def list_ambient_library( + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + rows = ( + db.query(AmbientNoiseAsset) + .filter( + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ) + .order_by(AmbientNoiseAsset.created_at.desc()) + .all() + ) + return rows + + +@router.post( + "/ambient-library", + response_model=AmbientNoiseAssetResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="uploadAmbientLibraryAsset", +) +async def upload_ambient_library_asset( + file: UploadFile = File(...), + name: Optional[str] = Form(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Upload a reusable ambient bed to the workspace library.""" + if not s3_service.is_enabled(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=s3_service.get_status_message(), + ) + + filename = file.filename or "" + file_bytes = await file.read() + try: + extension = validate_ambient_upload_bytes(file_bytes, filename=filename) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + asset_id = new_ambient_asset_id() + display_name = sanitize_ambient_name(name, filename.rsplit(".", 1)[0] if "." in filename else filename) + s3_key = ambient_library_s3_key(organization_id, asset_id, extension) + content_type = file.content_type or _guess_audio_media_type(filename) + try: + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + row = AmbientNoiseAsset( + id=asset_id, + organization_id=organization_id, + workspace_id=workspace_id, + name=display_name, + s3_key=s3_key, + original_filename=filename or None, + ) + db.add(row) + db.commit() + db.refresh(row) + return row + + +@router.patch( + "/ambient-library/{asset_id}", + response_model=AmbientNoiseAssetResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="updateAmbientLibraryAsset", +) +async def update_ambient_library_asset( + asset_id: UUID, + data: AmbientNoiseAssetUpdateRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Rename a library ambient bed.""" + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + + row.name = sanitize_ambient_name(data.name, row.name) + db.commit() + db.refresh(row) + return row + + +@router.delete( + "/ambient-library/{asset_id}", + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="deleteAmbientLibraryAsset", +) +async def delete_ambient_library_asset( + asset_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + + in_use = db.query(Persona).filter(Persona.background_noise_asset_id == asset_id).count() + if in_use: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Ambient bed is used by {in_use} persona(s). Reassign them before deleting.", + ) + + if s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(row.s3_key) + except Exception as exc: + logger.warning("Failed to delete ambient library object {}: {}", row.s3_key, exc) + + db.delete(row) + db.commit() + return JSONResponse(status_code=204, content=None) + + +@router.get( + "/ambient-library/{asset_id}/preview", + operation_id="previewAmbientLibraryAsset", +) +async def preview_ambient_library_asset( + asset_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Stream a library ambient bed for in-browser preview.""" + row = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + AmbientNoiseAsset.workspace_id == workspace_id, + ).first() + if not row: + raise HTTPException(status_code=404, detail="Ambient library asset not found") + if not s3_service.is_enabled(): + raise HTTPException(status_code=503, detail=s3_service.get_status_message()) + try: + file_bytes = s3_service.download_file_by_key(row.s3_key) + except StorageError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + return Response( + content=file_bytes, + media_type=_guess_audio_media_type(row.original_filename or row.s3_key), + ) + + # ============================================ # PERSONA BY ID (parameterized routes last) # ============================================ @@ -655,6 +976,12 @@ async def update_persona( except ValueError as exc: raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) update_data["tts_config"] = _normalized_persona_tts_config(provider, update_data["tts_config"]) + try: + update_data = _apply_ambient_update( + db_persona, update_data, organization_id, workspace_id, db + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) for field, value in update_data.items(): setattr(db_persona, field, value) @@ -843,6 +1170,11 @@ async def clone_persona( response_delay_ms=source_persona.response_delay_ms, max_turns=source_persona.max_turns, allow_interruptions=source_persona.allow_interruptions, + background_noise_source=source_persona.background_noise_source, + background_noise_preset=source_persona.background_noise_preset, + background_noise_volume=source_persona.background_noise_volume, + background_noise_s3_key=source_persona.background_noise_s3_key, + background_noise_asset_id=source_persona.background_noise_asset_id, ) db.add(new_persona) db.commit() @@ -880,6 +1212,113 @@ async def clone_persona( ) +@router.post( + "/{persona_id}/ambient-audio", + response_model=PersonaResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="uploadPersonaAmbientAudio", +) +async def upload_persona_ambient_audio( + persona_id: UUID, + file: UploadFile = File(...), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Upload or replace custom ambient audio for a persona (enterprise).""" + if not s3_service.is_enabled(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=s3_service.get_status_message(), + ) + + db_persona = db.query(Persona).filter( + Persona.id == persona_id, + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ).first() + if not db_persona: + raise HTTPException(status_code=404, detail=f"Persona {persona_id} not found") + + filename = file.filename or "" + extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if extension not in ALLOWED_AMBIENT_EXTENSIONS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported ambient audio format. Allowed: {', '.join(sorted(ALLOWED_AMBIENT_EXTENSIONS))}", + ) + + file_bytes = await file.read() + if not file_bytes: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Uploaded file is empty") + if len(file_bytes) > MAX_AMBIENT_UPLOAD_BYTES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Ambient audio must be at most {MAX_AMBIENT_UPLOAD_BYTES // (1024 * 1024)} MB", + ) + + try: + decode_audio_bytes_to_pcm_int16(file_bytes, 16000) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Could not decode ambient audio file: {exc}", + ) from exc + + s3_key = persona_ambient_s3_key(organization_id, persona_id, extension) + content_type = file.content_type or f"audio/{extension}" + try: + if db_persona.background_noise_s3_key and db_persona.background_noise_s3_key != s3_key: + try: + s3_service.delete_file_by_key(db_persona.background_noise_s3_key) + except Exception: + logger.warning("Could not delete previous ambient audio key {}", db_persona.background_noise_s3_key) + s3_service.upload_file_by_key(file_bytes, s3_key, content_type=content_type) + except StorageError as exc: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc + + db_persona.background_noise_s3_key = s3_key + db_persona.background_noise_source = BackgroundNoiseSourceEnum.CUSTOM.value + db.commit() + db.refresh(db_persona) + return db_persona + + +@router.delete( + "/{persona_id}/ambient-audio", + response_model=PersonaResponse, + dependencies=[Depends(require_enterprise_entitlement())], + operation_id="deletePersonaAmbientAudio", +) +async def delete_persona_ambient_audio( + persona_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), +): + """Delete custom ambient audio for a persona (enterprise).""" + db_persona = db.query(Persona).filter( + Persona.id == persona_id, + Persona.organization_id == organization_id, + Persona.workspace_id == workspace_id, + ).first() + if not db_persona: + raise HTTPException(status_code=404, detail=f"Persona {persona_id} not found") + + if db_persona.background_noise_s3_key and s3_service.is_enabled(): + try: + s3_service.delete_file_by_key(db_persona.background_noise_s3_key) + except Exception as exc: + logger.warning("Failed to delete ambient audio {}: {}", db_persona.background_noise_s3_key, exc) + + db_persona.background_noise_s3_key = None + if db_persona.background_noise_source == BackgroundNoiseSourceEnum.CUSTOM.value: + db_persona.background_noise_source = BackgroundNoiseSourceEnum.NONE.value + db.commit() + db.refresh(db_persona) + return db_persona + + # ============================================ # SEED DATA (Helper for demo) # ============================================ diff --git a/app/api/v1/routes/vobiz_telephony.py b/app/api/v1/routes/vobiz_telephony.py index 052d03b9..1081623e 100644 --- a/app/api/v1/routes/vobiz_telephony.py +++ b/app/api/v1/routes/vobiz_telephony.py @@ -732,6 +732,7 @@ async def vobiz_media_websocket(websocket: WebSocket): telephony_mode=True, call_short_id=call_short_id, silence_hangup_secs=hangup_secs, + persona=context.persona, ) except ValueError as e: logger.error("Vobiz media websocket setup failed: {}", e) diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index e6d553b5..1b400fad 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -13,6 +13,12 @@ from app.dependencies import get_organization_id, get_api_key from app.models.database import AIProvider, ModelProvider, Integration, IntegrationPlatform, Workspace from app.core.encryption import decrypt_api_key + + +def _parse_bool_query(value: Optional[str], *, default: bool = False) -> bool: + if value is None: + return default + return value.strip().lower() in ("1", "true", "yes", "on") from app.services.voice_agent.bot_fast_api import run_bot from app.services.ai.llm_service import _resolve_azure_endpoint_from_provider from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi @@ -103,6 +109,7 @@ async def websocket_endpoint( agent_id = websocket.query_params.get("agent_id") persona_id = websocket.query_params.get("persona_id") scenario_id = websocket.query_params.get("scenario_id") + run_evaluation = _parse_bool_query(websocket.query_params.get("run_evaluation"), default=False) # Fetch agent and voice bundle once for routing and instructions agent = None @@ -292,21 +299,22 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: return system_instruction = None - instruction_parts = [] + caller_speaks_first = True + caller_opening_text = None - # Build system instruction as a bundle: Agent + Persona + Scenario + # Build system instruction from agent + persona + scenario from app.models.database import Agent, Persona, Scenario + persona = None + scenario = None + # 1. Add Agent description (base instruction) and get voice bundle for model model_name = None if agent: - if agent.description: - instruction_parts.append(agent.description) if voice_bundle and voice_bundle.bundle_type == "s2s" and voice_bundle.s2s_model: model_name = voice_bundle.s2s_model - # 2. Add Persona information (characteristics) - persona = None + # 2. Load Persona if persona_id: try: persona_uuid = UUID(persona_id) @@ -317,23 +325,10 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: if workspace_id is not None: persona_query = persona_query.filter(Persona.workspace_id == workspace_id) persona = persona_query.first() - if persona: - persona_parts = [] - persona_parts.append(f"\n\nPersona: {persona.name}") - if persona.gender: - gender_val = persona.gender.value if hasattr(persona.gender, "value") else persona.gender - persona_parts.append(f"Gender: {gender_val}") - if getattr(persona, "tts_provider", None): - persona_parts.append(f"Voice provider: {persona.tts_provider}") - if getattr(persona, "tts_voice_name", None): - persona_parts.append(f"Voice: {persona.tts_voice_name}") - - if persona_parts: - instruction_parts.append("\n".join(persona_parts)) except ValueError: pass - # 3. Add Scenario information (context and goals) + # 3. Load Scenario if scenario_id: try: scenario_uuid = UUID(scenario_id) @@ -344,24 +339,32 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: if workspace_id is not None: scenario_query = scenario_query.filter(Scenario.workspace_id == workspace_id) scenario = scenario_query.first() - if scenario: - scenario_parts = [] - scenario_parts.append(f"\n\nScenario: {scenario.name}") - if scenario.description: - scenario_parts.append(f"Description: {scenario.description}") - if scenario.required_info: - required_info_str = ", ".join([f"{k}: {v}" for k, v in scenario.required_info.items()]) if isinstance(scenario.required_info, dict) else str(scenario.required_info) - if required_info_str: - scenario_parts.append(f"Required information to collect: {required_info_str}") - - if scenario_parts: - instruction_parts.append("\n".join(scenario_parts)) except ValueError: pass - - # Combine all parts into final system instruction - if instruction_parts: - system_instruction = "\n".join(instruction_parts) + + if agent and persona and scenario: + from app.services.testing.test_agent_simulation_prompt import ( + build_live_test_agent_system_prompt, + ) + from app.services.testing.test_agent_template import ( + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, + ) + + system_instruction = build_live_test_agent_system_prompt(agent, persona, scenario) + first_message_config = resolve_first_message_from_agent(agent) + scenario_first_message = None + if scenario.required_info and isinstance(scenario.required_info, dict): + scenario_first_message = scenario.required_info.get("first_message") + caller_opening_text = resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona.name or "Test Caller", + scenario_first_message=scenario_first_message, + ) + caller_speaks_first = should_caller_speak_first(first_message_config) + elif agent and agent.description: + system_instruction = agent.description.strip() # Generate result_id BEFORE running bot (for meaningful S3 path) # Evaluator is only created if persona_id and scenario_id are provided @@ -498,6 +501,13 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: ).lower() == "azure" else None ) + from app.services.voice_agent.llm_voice_providers import resolve_voice_llm_base_url + + llm_base_url = ( + resolve_voice_llm_base_url(db, organization_id, voice_bundle, llm_provider) + if llm_provider + else None + ) # If in bridge mode, we need to bridge test agent to Retell call # For now, we'll run the voice bundle normally and note that bridging @@ -529,7 +539,10 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: tts_api_key=tts_api_key, llm_api_key=llm_api_key, llm_endpoint_url=llm_endpoint_url, + llm_base_url=llm_base_url, silence_hangup_secs=agent_silence_hangup_secs, + caller_speaks_first=caller_speaks_first, + caller_opening_text=caller_opening_text, ) else: call_metadata = await run_bot( @@ -544,6 +557,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: result_id=result_id, model_name=model_name, # Pass model name from voice bundle silence_hangup_secs=agent_silence_hangup_secs, + persona=persona, ) except Exception as bot_error: logger.error(f"Error in run_bot: {bot_error}", exc_info=True) @@ -619,10 +633,13 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: else: result_name = "Test Call" - logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}") + logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}, run_evaluation={run_evaluation}") - # Create evaluator result with QUEUED status - # persona_id and scenario_id can be None for test calls without persona/scenario + initial_status = ( + EvaluatorResultStatus.QUEUED.value + if run_evaluation + else EvaluatorResultStatus.CALL_ENDED.value + ) evaluator_result = EvaluatorResult( result_id=result_id, organization_id=organization_id, @@ -633,7 +650,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: scenario_id=UUID(scenario_id) if scenario_id else None, # Optional name=result_name, duration_seconds=call_metadata.get("duration"), - status=EvaluatorResultStatus.QUEUED.value, # Use .value to get the string + status=initial_status, audio_s3_key=call_metadata.get("s3_key"), transcription=call_metadata.get("transcription"), speaker_segments=call_metadata.get("speaker_segments"), @@ -642,44 +659,42 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: db.commit() db.refresh(evaluator_result) - logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}") + logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}, status={initial_status}") - # Trigger Celery task - try: - logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") - - # Check if Celery app is properly configured - from app.workers.celery_app import celery_app - logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") - logger.info(f"Celery result backend: {celery_app.conf.result_backend}") - - # Verify task is registered - if 'process_evaluator_result' not in celery_app.tasks: - logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") - logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") - else: - logger.info("✅ Task 'process_evaluator_result' is registered") - - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") - - # Try to get task info to verify it was queued + if run_evaluation: try: - task_info = task.info - logger.info(f"Task info: {task_info}") - except Exception as info_error: - logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") - - evaluator_result.celery_task_id = task.id - db.commit() - logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") - except Exception as task_error: - logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) - # Still log that we created the result even if task trigger failed - logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") - logger.warning(f"Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") - - logger.info(f"✅ Created evaluator result {result_id} and triggered processing task") + logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") + + from app.workers.celery_app import celery_app + logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") + logger.info(f"Celery result backend: {celery_app.conf.result_backend}") + + if 'process_evaluator_result' not in celery_app.tasks: + logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") + logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") + else: + logger.info("✅ Task 'process_evaluator_result' is registered") + + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") + + try: + task_info = task.info + logger.info(f"Task info: {task_info}") + except Exception as info_error: + logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") + + evaluator_result.celery_task_id = task.id + db.commit() + logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") + except Exception as task_error: + logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) + logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") + logger.warning("Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") + else: + logger.info(f"Skipping post-call evaluation for result {result_id} (run_evaluation=false)") + + logger.info(f"✅ Created evaluator result {result_id}" + (" and triggered processing task" if run_evaluation else "")) except Exception as e: logger.error(f"❌ Error creating evaluator result: {e}", exc_info=True) @@ -814,6 +829,7 @@ def _extract_bearer(value: Optional[str]) -> Optional[str]: agent_id = request.query_params.get("agent_id") persona_id = request.query_params.get("persona_id") scenario_id = request.query_params.get("scenario_id") + run_evaluation = _parse_bool_query(request.query_params.get("run_evaluation"), default=False) # Determine which AI Provider to use based on agent configuration ai_provider = None @@ -959,6 +975,7 @@ def check_provider(provider_enum): agent_id=agent_id, persona_id=persona_id, scenario_id=scenario_id, + run_evaluation=run_evaluation, fallback_host=request.headers.get("host", f"localhost:{settings.PORT}"), fallback_scheme=( request.headers.get("x-forwarded-proto") diff --git a/app/migrations/078_add_test_agent_template_to_agents.py b/app/migrations/078_add_test_agent_template_to_agents.py new file mode 100644 index 00000000..54855a02 --- /dev/null +++ b/app/migrations/078_add_test_agent_template_to_agents.py @@ -0,0 +1,28 @@ +""" +Migration: Add test_agent_template JSON column to agents. + +Stores structured test agent prompt sections and first-message configuration. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add test_agent_template to agents" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE agents + ADD COLUMN IF NOT EXISTS test_agent_template JSON + """)) + db.commit() + print("Added test_agent_template column to agents") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE agents + DROP COLUMN IF EXISTS test_agent_template + """)) + db.commit() + print("Dropped test_agent_template column from agents") diff --git a/app/migrations/079_persona_ambient_noise.py b/app/migrations/079_persona_ambient_noise.py new file mode 100644 index 00000000..38c91840 --- /dev/null +++ b/app/migrations/079_persona_ambient_noise.py @@ -0,0 +1,32 @@ +""" +Migration: Add persona ambient noise configuration columns. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add persona background noise source, preset, volume, and s3 key" + + +def upgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + ADD COLUMN IF NOT EXISTS background_noise_source VARCHAR(20) NOT NULL DEFAULT 'none', + ADD COLUMN IF NOT EXISTS background_noise_preset VARCHAR(50), + ADD COLUMN IF NOT EXISTS background_noise_volume DOUBLE PRECISION DEFAULT 0.22, + ADD COLUMN IF NOT EXISTS background_noise_s3_key VARCHAR + """)) + db.commit() + print("Added persona ambient noise columns") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + DROP COLUMN IF EXISTS background_noise_s3_key, + DROP COLUMN IF EXISTS background_noise_volume, + DROP COLUMN IF EXISTS background_noise_preset, + DROP COLUMN IF EXISTS background_noise_source + """)) + db.commit() + print("Dropped persona ambient noise columns") diff --git a/app/migrations/080_ambient_noise_library.py b/app/migrations/080_ambient_noise_library.py new file mode 100644 index 00000000..05bc069f --- /dev/null +++ b/app/migrations/080_ambient_noise_library.py @@ -0,0 +1,52 @@ +""" +Migration: Add reusable ambient noise library and persona asset reference. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add ambient_noise_assets table and persona background_noise_asset_id" + + +def upgrade(db: Session): + db.execute(text(""" + CREATE TABLE IF NOT EXISTS ambient_noise_assets ( + id UUID PRIMARY KEY, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + name VARCHAR(255) NOT NULL, + s3_key VARCHAR NOT NULL, + original_filename VARCHAR(255), + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + ) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_ambient_noise_assets_organization_id + ON ambient_noise_assets (organization_id) + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_ambient_noise_assets_workspace_id + ON ambient_noise_assets (workspace_id) + """)) + db.execute(text(""" + ALTER TABLE personas + ADD COLUMN IF NOT EXISTS background_noise_asset_id UUID + REFERENCES ambient_noise_assets(id) ON DELETE SET NULL + """)) + db.execute(text(""" + CREATE INDEX IF NOT EXISTS ix_personas_background_noise_asset_id + ON personas (background_noise_asset_id) + """)) + db.commit() + print("Added ambient_noise_assets table and persona background_noise_asset_id") + + +def downgrade(db: Session): + db.execute(text(""" + ALTER TABLE personas + DROP COLUMN IF EXISTS background_noise_asset_id + """)) + db.execute(text("DROP TABLE IF EXISTS ambient_noise_assets")) + db.commit() + print("Dropped ambient_noise_assets table and persona background_noise_asset_id") diff --git a/app/models/database.py b/app/models/database.py index 92350139..4e35ecfa 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -522,6 +522,7 @@ class Agent(Base): voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) prompt_variables = Column(JSON, nullable=True) + test_agent_template = Column(JSON, nullable=True) silence_hangup_secs = Column(Integer, nullable=False, server_default="15") created_at = Column(DateTime, server_default=func.now()) @@ -557,12 +558,41 @@ class Persona(Base): response_delay_ms = Column(Integer, nullable=True) max_turns = Column(Integer, nullable=True) allow_interruptions = Column(Boolean, nullable=True) + background_noise_source = Column(String(20), nullable=False, default="none") + background_noise_preset = Column(String(50), nullable=True) + background_noise_volume = Column(Float, nullable=True, default=0.22) + background_noise_s3_key = Column(String, nullable=True) + background_noise_asset_id = Column( + UUID(as_uuid=True), + ForeignKey("ambient_noise_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) created_at = Column(DateTime, server_default=func.now()) updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) created_by = Column(String) +class AmbientNoiseAsset(Base): + """Reusable ambient noise bed uploaded for test-agent personas.""" + __tablename__ = "ambient_noise_assets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + s3_key = Column(String, nullable=False) + original_filename = Column(String(255), nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + class Scenario(Base): """Scenario - The conversation scenario/test case""" __tablename__ = "scenarios" diff --git a/app/models/enums.py b/app/models/enums.py index 140ffe48..020acecf 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -81,12 +81,21 @@ class AccentEnum(str, enum.Enum): GERMAN = "german" NEUTRAL = "neutral" +class BackgroundNoiseSourceEnum(str, enum.Enum): + """Where persona ambient audio is loaded from.""" + NONE = "none" + PLATFORM = "platform" + CUSTOM = "custom" + + class BackgroundNoiseEnum(str, enum.Enum): - """Background noise options""" + """Platform ambient preset names.""" NONE = "none" OFFICE = "office" STREET = "street" + TRAFFIC = "traffic" CAFE = "cafe" + CONCERT = "concert" HOME = "home" CALL_CENTER = "call_center" diff --git a/app/models/schemas.py b/app/models/schemas.py index 40e988e9..dc268618 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -8,6 +8,7 @@ from app.models.enums import ( EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + BackgroundNoiseSourceEnum, IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, @@ -173,6 +174,37 @@ class ErrorResponse(BaseModel): # Enums moved to enums.py + + + +# Test agent template schemas (defined before AgentCreate/AgentUpdate) +class TestPromptSectionResponse(BaseModel): + """One canonical section of a generated test agent prompt.""" + key: str + title: str + content: str + + +class TestAgentFirstMessageResponse(BaseModel): + """Who speaks first on production vs test caller sides.""" + production_mode: str + production_message: Optional[str] = None + caller_mode: str + caller_message: Optional[str] = None + + +class TestAgentTemplateResponse(BaseModel): + """Structured test agent template stored on agents.""" + sections: List[TestPromptSectionResponse] + first_message: TestAgentFirstMessageResponse + + +class TestAgentTemplateInput(BaseModel): + """Structured test agent template for create/update.""" + sections: List[TestPromptSectionResponse] + first_message: TestAgentFirstMessageResponse + + # Agent Schemas class AgentCreate(BaseModel): """Schema for creating a new agent""" @@ -188,6 +220,7 @@ class AgentCreate(BaseModel): voice_ai_integration_id: Optional[UUID] = None voice_ai_agent_id: Optional[str] = None provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateInput] = None silence_hangup_secs: int = Field( default=15, ge=0, @@ -245,6 +278,7 @@ class AgentUpdate(BaseModel): voice_ai_integration_id: Optional[UUID] = None voice_ai_agent_id: Optional[str] = None provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateInput] = None prompt_variables: Optional[Dict[str, str]] = None silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) @@ -303,15 +337,6 @@ class AgentPhoneAssignmentCheckResponse(BaseModel): -class TestPromptSectionResponse(BaseModel): - """One canonical section of a generated test agent prompt.""" - key: str - title: str - content: str - - - - class GeneratedScenarioDraftResponse(BaseModel): """LLM-generated scenario draft before persistence.""" name: str @@ -339,6 +364,8 @@ class GenerateTestPromptRequest(BaseModel): class GenerateTestPromptResponse(BaseModel): sections: List[TestPromptSectionResponse] test_agent_prompt: str + first_message: TestAgentFirstMessageResponse + test_agent_template: TestAgentTemplateResponse provider: str model: str @@ -388,6 +415,8 @@ class GenerateTestSetupRequest(BaseModel): class GenerateTestSetupResponse(BaseModel): sections: List[TestPromptSectionResponse] test_agent_prompt: str + first_message: TestAgentFirstMessageResponse + test_agent_template: TestAgentTemplateResponse scenarios: List[GeneratedScenarioDraftResponse] provider: str model: str @@ -410,6 +439,7 @@ class AgentResponse(BaseModel): voice_ai_integration_id: Optional[UUID] voice_ai_agent_id: Optional[str] provider_prompt: Optional[str] = None + test_agent_template: Optional[TestAgentTemplateResponse] = None prompt_variables: Optional[Dict[str, str]] = None silence_hangup_secs: int = Field( default=15, @@ -496,6 +526,10 @@ class PersonaCreate(BaseModel): response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) max_turns: Optional[int] = Field(None, ge=1, le=100) allow_interruptions: Optional[bool] = None + background_noise_source: BackgroundNoiseSourceEnum = BackgroundNoiseSourceEnum.NONE + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = Field(None, ge=0.05, le=0.60) + background_noise_asset_id: Optional[UUID] = None @model_validator(mode="after") def validate_tts_config(self): @@ -504,6 +538,28 @@ def validate_tts_config(self): validate_persona_tts_config(self.tts_provider, self.tts_config) return self + @model_validator(mode="after") + def validate_ambient_fields(self): + from app.services.personas.persona_ambient_noise import validate_persona_ambient_fields + + validated = validate_persona_ambient_fields( + source=self.background_noise_source.value, + preset=self.background_noise_preset, + volume=self.background_noise_volume, + s3_key=None, + asset_id=self.background_noise_asset_id, + ) + if validated["background_noise_source"] == BackgroundNoiseSourceEnum.CUSTOM.value: + if not self.background_noise_asset_id: + raise ValueError( + "Select an uploaded ambient bed from the Environment tab or upload one under Background Noise" + ) + self.background_noise_source = BackgroundNoiseSourceEnum(validated["background_noise_source"]) + self.background_noise_preset = validated["background_noise_preset"] + self.background_noise_volume = validated["background_noise_volume"] + self.background_noise_asset_id = validated["background_noise_asset_id"] + return self + class PersonaUpdate(BaseModel): """Schema for updating a persona""" @@ -520,6 +576,10 @@ class PersonaUpdate(BaseModel): response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) max_turns: Optional[int] = Field(None, ge=1, le=100) allow_interruptions: Optional[bool] = None + background_noise_source: Optional[BackgroundNoiseSourceEnum] = None + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = Field(None, ge=0.05, le=0.60) + background_noise_asset_id: Optional[UUID] = None @model_validator(mode="after") def validate_tts_config(self): @@ -546,6 +606,11 @@ class PersonaResponse(BaseModel): response_delay_ms: Optional[int] = None max_turns: Optional[int] = None allow_interruptions: Optional[bool] = None + background_noise_source: str = BackgroundNoiseSourceEnum.NONE.value + background_noise_preset: Optional[str] = None + background_noise_volume: Optional[float] = None + background_noise_s3_key: Optional[str] = None + background_noise_asset_id: Optional[UUID] = None created_at: datetime updated_at: datetime @@ -563,6 +628,21 @@ def convert_gender(cls, v): model_config = ConfigDict(from_attributes=True) +class AmbientNoiseAssetResponse(BaseModel): + id: UUID + name: str + s3_key: str + original_filename: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class AmbientNoiseAssetUpdateRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + + class PersonaCloneRequest(BaseModel): """Schema for cloning a persona""" name: Optional[str] = None diff --git a/app/services/audio/ambient_catalog.py b/app/services/audio/ambient_catalog.py new file mode 100644 index 00000000..b4478c01 --- /dev/null +++ b/app/services/audio/ambient_catalog.py @@ -0,0 +1,250 @@ +"""Pluggable ambient asset catalog and persona resolution.""" + +from __future__ import annotations + +import os +from importlib import metadata +from pathlib import Path +from typing import Any, Optional, Protocol, runtime_checkable + +from loguru import logger + +from app.models.enums import BackgroundNoiseEnum, BackgroundNoiseSourceEnum +from app.services.audio.ambient_mixer import AmbientMixer, clamp_ambient_volume + +PRESET_DISPLAY_NAMES = { + BackgroundNoiseEnum.CAFE.value: "Cafe", + BackgroundNoiseEnum.TRAFFIC.value: "Traffic", + BackgroundNoiseEnum.STREET.value: "Street traffic", + BackgroundNoiseEnum.CONCERT.value: "Concert crowd", + BackgroundNoiseEnum.OFFICE.value: "Office", + BackgroundNoiseEnum.HOME.value: "Home", + BackgroundNoiseEnum.CALL_CENTER.value: "Call center", +} + + +def normalize_ambient_preset(name: Optional[str]) -> Optional[str]: + if not name: + return None + value = str(name).strip().lower() + if value == BackgroundNoiseEnum.NONE.value: + return None + if value == BackgroundNoiseEnum.STREET.value: + return BackgroundNoiseEnum.TRAFFIC.value + return value + + +@runtime_checkable +class AmbientAssetProvider(Protocol): + def list_presets(self) -> list[str]: + ... + + def load_wav(self, name: str) -> bytes: + ... + + +class EmptyAmbientAssetProvider: + def list_presets(self) -> list[str]: + return [] + + def load_wav(self, name: str) -> bytes: + raise FileNotFoundError(name) + + +class DirectoryAmbientAssetProvider: + """Load preset WAV/MP3/OGG files from EFFICIENTAI_AMBIENT_DIR.""" + + def __init__(self, directory: Optional[str] = None): + self._directory = Path(directory or os.getenv("EFFICIENTAI_AMBIENT_DIR", "")).expanduser() + + def list_presets(self) -> list[str]: + if not self._directory.is_dir(): + return [] + names: list[str] = [] + for path in sorted(self._directory.iterdir()): + if path.is_file() and path.suffix.lower() in {".wav", ".mp3", ".ogg", ".m4a", ".flac"}: + names.append(path.stem.lower()) + return names + + def load_wav(self, name: str) -> bytes: + normalized = normalize_ambient_preset(name) or name + if not self._directory.is_dir(): + raise FileNotFoundError(normalized) + for ext in (".wav", ".mp3", ".ogg", ".m4a", ".flac"): + candidate = self._directory / f"{normalized}{ext}" + if candidate.is_file(): + return candidate.read_bytes() + raise FileNotFoundError(normalized) + + +class EntryPointAmbientAssetProvider: + """Load presets from efficientai.ambient_assets entry points.""" + + GROUP = "efficientai.ambient_assets" + + def __init__(self): + self._provider: Optional[AmbientAssetProvider] = None + self._loaded = False + + def _ensure_provider(self) -> Optional[AmbientAssetProvider]: + if self._loaded: + return self._provider + self._loaded = True + try: + eps = metadata.entry_points(group=self.GROUP) + except TypeError: + eps = metadata.entry_points().get(self.GROUP, []) + for ep in eps: + try: + provider = ep.load() + if hasattr(provider, "list_presets") and hasattr(provider, "load_wav"): + self._provider = provider + logger.info("Loaded ambient asset provider from entry point {}", ep.name) + break + except Exception as exc: + logger.warning("Failed to load ambient asset entry point {}: {}", ep.name, exc) + return self._provider + + def list_presets(self) -> list[str]: + provider = self._ensure_provider() + return provider.list_presets() if provider else [] + + def load_wav(self, name: str) -> bytes: + provider = self._ensure_provider() + if not provider: + raise FileNotFoundError(name) + return provider.load_wav(name) + + +class ChainedAmbientAssetProvider: + def __init__(self, providers: list[AmbientAssetProvider]): + self._providers = providers + + def list_presets(self) -> list[str]: + seen: set[str] = set() + ordered: list[str] = [] + for provider in self._providers: + for preset in provider.list_presets(): + normalized = normalize_ambient_preset(preset) or preset + if normalized not in seen: + seen.add(normalized) + ordered.append(normalized) + return ordered + + def load_wav(self, name: str) -> bytes: + normalized = normalize_ambient_preset(name) or name + last_error: Optional[Exception] = None + for provider in self._providers: + try: + return provider.load_wav(normalized) + except FileNotFoundError as exc: + last_error = exc + raise FileNotFoundError(normalized) from last_error + + +_default_provider: Optional[ChainedAmbientAssetProvider] = None + + +def get_ambient_asset_provider() -> ChainedAmbientAssetProvider: + global _default_provider + if _default_provider is None: + _default_provider = ChainedAmbientAssetProvider( + [ + EntryPointAmbientAssetProvider(), + DirectoryAmbientAssetProvider(), + EmptyAmbientAssetProvider(), + ] + ) + return _default_provider + + +def list_ambient_presets() -> list[dict[str, str]]: + provider = get_ambient_asset_provider() + return [ + { + "id": preset, + "label": PRESET_DISPLAY_NAMES.get(preset, preset.replace("_", " ").title()), + } + for preset in provider.list_presets() + ] + + +def persona_ambient_source(persona: Any) -> str: + source = getattr(persona, "background_noise_source", None) or BackgroundNoiseSourceEnum.NONE.value + return str(source).strip().lower() or BackgroundNoiseSourceEnum.NONE.value + + +def persona_ambient_volume(persona: Any) -> float: + return clamp_ambient_volume(getattr(persona, "background_noise_volume", None)) + + +def persona_has_active_ambient(persona: Any) -> bool: + return persona_ambient_source(persona) != BackgroundNoiseSourceEnum.NONE.value + + +def _resolve_custom_ambient_s3_key(persona: Any) -> Optional[str]: + asset_id = getattr(persona, "background_noise_asset_id", None) + if asset_id: + try: + from app.database import SessionLocal + from app.models.database import AmbientNoiseAsset + + db = SessionLocal() + try: + row = db.query(AmbientNoiseAsset).filter(AmbientNoiseAsset.id == asset_id).first() + if row and row.s3_key: + return str(row.s3_key) + finally: + db.close() + except Exception as exc: + logger.warning("Failed to resolve ambient asset {}: {}", asset_id, exc) + legacy_key = getattr(persona, "background_noise_s3_key", None) + return str(legacy_key) if legacy_key else None + + +async def resolve_ambient_mixer(persona: Any, sample_rate: int) -> Optional[AmbientMixer]: + """Build an AmbientMixer for a persona at the given sample rate, or None.""" + source = persona_ambient_source(persona) + volume = persona_ambient_volume(persona) + + if source == BackgroundNoiseSourceEnum.NONE.value: + return None + + if source == BackgroundNoiseSourceEnum.PLATFORM.value: + preset = normalize_ambient_preset(getattr(persona, "background_noise_preset", None)) + if not preset: + logger.warning("Persona {} has platform ambient source but no preset", getattr(persona, "id", "?")) + return None + provider = get_ambient_asset_provider() + try: + file_bytes = provider.load_wav(preset) + except FileNotFoundError: + logger.warning( + "Ambient preset {} is not available (install ambient asset pack or set EFFICIENTAI_AMBIENT_DIR)", + preset, + ) + return None + return AmbientMixer.from_pcm_bytes(file_bytes, sample_rate=sample_rate, volume=volume) + + if source == BackgroundNoiseSourceEnum.CUSTOM.value: + s3_key = _resolve_custom_ambient_s3_key(persona) + if not s3_key: + logger.warning( + "Persona {} has custom ambient source but no resolvable audio", + getattr(persona, "id", "?"), + ) + return None + try: + from app.services.storage.s3_service import s3_service + + if not s3_service.is_enabled(): + logger.warning("Blob storage disabled; cannot load custom ambient audio for persona {}", getattr(persona, "id", "?")) + return None + file_bytes = s3_service.download_file_by_key(str(s3_key)) + except Exception as exc: + logger.warning("Failed to load custom ambient audio for persona {}: {}", getattr(persona, "id", "?"), exc) + return None + return AmbientMixer.from_pcm_bytes(file_bytes, sample_rate=sample_rate, volume=volume) + + logger.warning("Unknown ambient source {} on persona {}", source, getattr(persona, "id", "?")) + return None diff --git a/app/services/audio/ambient_mic_pump.py b/app/services/audio/ambient_mic_pump.py new file mode 100644 index 00000000..801b5e7d --- /dev/null +++ b/app/services/audio/ambient_mic_pump.py @@ -0,0 +1,95 @@ +"""Continuous ambient mic feed for evaluator WebRTC bridges.""" + +from __future__ import annotations + +import asyncio +from typing import Awaitable, Callable, Optional + +from loguru import logger + +from app.services.audio.ambient_mixer import AmbientBed + + +class AmbientMicPump: + """ + Streams continuous ambient-only PCM while idle and mixed speech while active. + + Replaces ElevenLabs' zero-silence loop when a persona has background noise. + """ + + def __init__( + self, + bed: AmbientBed, + *, + sample_rate: int, + chunk_duration_ms: int = 20, + send_callback: Callable[[bytes], Awaitable[None]], + mark_speech_done: Optional[Callable[[], None]] = None, + ): + self._bed = bed + self._sample_rate = sample_rate + self._chunk_duration_ms = chunk_duration_ms + self._send_callback = send_callback + self._mark_speech_done = mark_speech_done + self._speaking = False + self._stop = asyncio.Event() + self._task: Optional[asyncio.Task] = None + + @property + def chunk_duration_ms(self) -> int: + return self._chunk_duration_ms + + def _chunk_samples(self) -> int: + return max(1, (self._sample_rate * self._chunk_duration_ms) // 1000) + + async def start(self): + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._idle_loop()) + logger.info( + "Ambient mic pump started (sample_rate={}, chunk_ms={})", + self._sample_rate, + self._chunk_duration_ms, + ) + + async def stop(self): + self._stop.set() + if self._task: + try: + await asyncio.wait_for(self._task, timeout=2.0) + except asyncio.TimeoutError: + self._task.cancel() + self._task = None + + async def _idle_loop(self): + chunk_samples = self._chunk_samples() + interval = self._chunk_duration_ms / 1000.0 + try: + while not self._stop.is_set(): + if self._speaking: + await asyncio.sleep(0.01) + continue + await self._send_callback(self._bed.chunk_bytes(chunk_samples)) + await asyncio.sleep(interval) + except asyncio.CancelledError: + pass + except Exception as exc: + logger.error("Ambient mic pump idle loop error: {}", exc) + + async def send_speech( + self, + audio_bytes: bytes, + stream_chunks: Callable[[bytes, Callable[[bytes], Awaitable[None]], int], Awaitable[None]], + ): + self._speaking = True + try: + async def mixed_callback(chunk: bytes): + mixed = self._bed.mix_speech(chunk) + await self._send_callback(mixed) + + await stream_chunks(audio_bytes, mixed_callback, self._chunk_duration_ms) + finally: + self._speaking = False + if self._mark_speech_done: + self._mark_speech_done() diff --git a/app/services/audio/ambient_mixer.py b/app/services/audio/ambient_mixer.py new file mode 100644 index 00000000..7db238c9 --- /dev/null +++ b/app/services/audio/ambient_mixer.py @@ -0,0 +1,163 @@ +"""Ambient background audio mixing for test-agent caller simulation.""" + +from __future__ import annotations + +import io +from typing import Optional + +import numpy as np +from loguru import logger + +from efficientai.audio.mixers.base_audio_mixer import BaseAudioMixer +from efficientai.frames.frames import MixerControlFrame + +DEFAULT_AMBIENT_VOLUME = 0.22 +MIN_AMBIENT_VOLUME = 0.05 +MAX_AMBIENT_VOLUME = 0.60 + + +def clamp_ambient_volume(volume: Optional[float]) -> float: + if volume is None: + return DEFAULT_AMBIENT_VOLUME + return float(max(MIN_AMBIENT_VOLUME, min(MAX_AMBIENT_VOLUME, volume))) + + +def resample_mono_int16(audio: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray: + if source_rate == target_rate or len(audio) == 0: + return audio.astype(np.int16, copy=False) + target_len = max(1, int(round(len(audio) * target_rate / source_rate))) + try: + from scipy.signal import resample + + resampled = resample(audio.astype(np.float64), target_len) + except Exception: + source_positions = np.linspace(0, len(audio) - 1, num=len(audio)) + target_positions = np.linspace(0, len(audio) - 1, num=target_len) + resampled = np.interp(target_positions, source_positions, audio.astype(np.float64)) + return np.clip(np.round(resampled), -32768, 32767).astype(np.int16) + + +def decode_audio_bytes_to_pcm_int16(file_bytes: bytes, target_sample_rate: int) -> np.ndarray: + """Decode arbitrary audio bytes to mono int16 PCM at target sample rate.""" + try: + import soundfile as sf + except ModuleNotFoundError as exc: + raise RuntimeError("soundfile is required for ambient audio decoding") from exc + + data, sample_rate = sf.read(io.BytesIO(file_bytes), dtype="float32", always_2d=False) + if data.ndim > 1: + data = np.mean(data, axis=1) + pcm = np.clip(data * 32767.0, -32768, 32767).astype(np.int16) + return resample_mono_int16(pcm, int(sample_rate), target_sample_rate) + + +class AmbientBed: + """Looping mono int16 bed used by bridge pumps and transport mixers.""" + + def __init__( + self, + bed_pcm: np.ndarray, + *, + volume: float = DEFAULT_AMBIENT_VOLUME, + loop: bool = True, + ): + self._bed = np.asarray(bed_pcm, dtype=np.int16) + if self._bed.size == 0: + raise ValueError("ambient bed PCM must not be empty") + self._volume = clamp_ambient_volume(volume) + self._loop = loop + self._pos = 0 + + @property + def volume(self) -> float: + return self._volume + + def chunk(self, num_samples: int) -> np.ndarray: + if num_samples <= 0: + return np.zeros(0, dtype=np.int16) + if not self._loop and self._pos >= len(self._bed): + return np.zeros(num_samples, dtype=np.int16) + + out = np.empty(num_samples, dtype=np.int16) + offset = 0 + while offset < num_samples: + if self._pos >= len(self._bed): + if not self._loop: + out[offset:] = 0 + break + self._pos = 0 + take = min(num_samples - offset, len(self._bed) - self._pos) + segment = self._bed[self._pos : self._pos + take] + out[offset : offset + take] = np.clip( + np.round(segment.astype(np.float64) * self._volume), + -32768, + 32767, + ).astype(np.int16) + self._pos += take + offset += take + return out + + def chunk_bytes(self, num_samples: int) -> bytes: + return self.chunk(num_samples).tobytes() + + def mix_speech(self, speech_bytes: bytes) -> bytes: + if not speech_bytes: + return speech_bytes + speech = np.frombuffer(speech_bytes, dtype=np.int16) + bed = self.chunk(len(speech)) + mixed = np.clip( + speech.astype(np.int32) + bed.astype(np.int32), + -32768, + 32767, + ).astype(np.int16) + return mixed.tobytes() + + +class AmbientMixer(BaseAudioMixer): + """Output-transport mixer that overlays a looping ambient bed on bot TTS audio.""" + + def __init__(self, bed: AmbientBed): + self._bed = bed + self._sample_rate = 0 + self._enabled = True + + @classmethod + def from_pcm_bytes( + cls, + file_bytes: bytes, + *, + sample_rate: int, + volume: Optional[float] = None, + ) -> "AmbientMixer": + pcm = decode_audio_bytes_to_pcm_int16(file_bytes, sample_rate) + return cls(AmbientBed(pcm, volume=clamp_ambient_volume(volume))) + + @classmethod + def from_pcm_array( + cls, + pcm: np.ndarray, + *, + volume: Optional[float] = None, + ) -> "AmbientMixer": + return cls(AmbientBed(np.asarray(pcm, dtype=np.int16), volume=clamp_ambient_volume(volume))) + + @property + def bed(self) -> AmbientBed: + return self._bed + + async def start(self, sample_rate: int): + self._sample_rate = sample_rate + + async def stop(self): + pass + + async def process_frame(self, frame: MixerControlFrame): + del frame + + async def mix(self, audio: bytes) -> bytes: + if not self._enabled: + return audio + if not audio: + samples = max(1, self._sample_rate // 50) if self._sample_rate else 320 + return self._bed.chunk_bytes(samples) + return self._bed.mix_speech(audio) diff --git a/app/services/evaluators/evaluator_results_query.py b/app/services/evaluators/evaluator_results_query.py index f4e6b79c..4cd37eb4 100644 --- a/app/services/evaluators/evaluator_results_query.py +++ b/app/services/evaluators/evaluator_results_query.py @@ -67,9 +67,12 @@ def build_evaluator_results_query( ) if playground is True: - query = query.filter(EvaluatorResult.evaluator_id.is_(None)) if test_agents_only is True: + # Voice-bundle playground test agents (not Retell/Vapi). Includes runs + # that auto-link an evaluator when persona + scenario are selected. query = query.filter(EvaluatorResult.provider_platform.is_(None)) + else: + query = query.filter(EvaluatorResult.evaluator_id.is_(None)) elif playground is False: query = query.filter(EvaluatorResult.evaluator_id.isnot(None)) else: diff --git a/app/services/media_urls.py b/app/services/media_urls.py index 6ef55adf..5ab87c68 100644 --- a/app/services/media_urls.py +++ b/app/services/media_urls.py @@ -67,6 +67,7 @@ def build_voice_agent_ws_url( agent_id: Optional[str] = None, persona_id: Optional[str] = None, scenario_id: Optional[str] = None, + run_evaluation: bool = False, fallback_host: Optional[str] = None, fallback_scheme: str = "http", ) -> str: @@ -86,5 +87,7 @@ def build_voice_agent_ws_url( query += f"&persona_id={quote(persona_id)}" if scenario_id: query += f"&scenario_id={quote(scenario_id)}" + if run_evaluation: + query += "&run_evaluation=true" return f"{base}{settings.API_V1_PREFIX}/voice-agent/ws?{query}" diff --git a/app/services/personas/ambient_library.py b/app/services/personas/ambient_library.py new file mode 100644 index 00000000..ce3d17aa --- /dev/null +++ b/app/services/personas/ambient_library.py @@ -0,0 +1,49 @@ +"""Workspace ambient noise library helpers.""" + +from __future__ import annotations + +import re +from typing import Any, Optional +from uuid import UUID, uuid4 + +from app.services.audio.ambient_mixer import decode_audio_bytes_to_pcm_int16 +from app.services.personas.persona_ambient_noise import ( + ALLOWED_AMBIENT_EXTENSIONS, + MAX_AMBIENT_UPLOAD_BYTES, +) + + +def sanitize_ambient_name(name: Optional[str], fallback: str) -> str: + raw = (name or fallback or "Ambient bed").strip() + cleaned = re.sub(r"\s+", " ", raw) + return cleaned[:255] or "Ambient bed" + + +def ambient_library_s3_key(organization_id: Any, asset_id: Any, extension: str) -> str: + from app.services.storage.s3_service import s3_service + + ext = extension.lower().lstrip(".") + return ( + f"{s3_service.prefix}organizations/{organization_id}/ambient-library/" + f"{asset_id}.{ext}" + ) + + +def validate_ambient_upload_bytes(file_bytes: bytes, *, filename: str) -> str: + extension = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if extension not in ALLOWED_AMBIENT_EXTENSIONS: + raise ValueError( + f"Unsupported ambient audio format. Allowed: {', '.join(sorted(ALLOWED_AMBIENT_EXTENSIONS))}" + ) + if not file_bytes: + raise ValueError("Uploaded file is empty") + if len(file_bytes) > MAX_AMBIENT_UPLOAD_BYTES: + raise ValueError( + f"Ambient audio must be at most {MAX_AMBIENT_UPLOAD_BYTES // (1024 * 1024)} MB" + ) + decode_audio_bytes_to_pcm_int16(file_bytes, 16000) + return extension + + +def new_ambient_asset_id() -> UUID: + return uuid4() diff --git a/app/services/personas/persona_ambient_noise.py b/app/services/personas/persona_ambient_noise.py new file mode 100644 index 00000000..3bf88cdc --- /dev/null +++ b/app/services/personas/persona_ambient_noise.py @@ -0,0 +1,129 @@ +"""Validation and normalization for persona ambient noise settings.""" + +from __future__ import annotations + +from typing import Any, Optional + +from app.core.usage_entitlement import has_enterprise_entitlement +from app.models.enums import BackgroundNoiseEnum, BackgroundNoiseSourceEnum +from app.services.audio.ambient_catalog import ( + get_ambient_asset_provider, + normalize_ambient_preset, +) +from app.services.audio.ambient_mixer import ( + MAX_AMBIENT_VOLUME, + MIN_AMBIENT_VOLUME, + clamp_ambient_volume, +) + +ALLOWED_AMBIENT_EXTENSIONS = {"wav", "mp3", "ogg", "m4a", "flac"} +MAX_AMBIENT_UPLOAD_BYTES = 10 * 1024 * 1024 + + +def normalize_ambient_source(value: Optional[str]) -> str: + if not value: + return BackgroundNoiseSourceEnum.NONE.value + normalized = str(value).strip().lower() + try: + return BackgroundNoiseSourceEnum(normalized).value + except ValueError as exc: + raise ValueError( + f"background_noise_source must be one of: " + f"{', '.join(s.value for s in BackgroundNoiseSourceEnum)}" + ) from exc + + +def validate_ambient_preset(preset: Optional[str]) -> Optional[str]: + normalized = normalize_ambient_preset(preset) + if not normalized: + raise ValueError("background_noise_preset is required for platform ambient source") + known = {item.value for item in BackgroundNoiseEnum if item != BackgroundNoiseEnum.NONE} + known.add(BackgroundNoiseEnum.TRAFFIC.value) + provider_presets = set(get_ambient_asset_provider().list_presets()) + allowed = known | provider_presets + if normalized not in allowed: + raise ValueError(f"Unknown ambient preset: {normalized}") + return normalized + + +def validate_persona_ambient_fields( + *, + source: Optional[str], + preset: Optional[str], + volume: Optional[float], + s3_key: Optional[str], + asset_id: Optional[Any] = None, + organization_id: Any = None, + workspace_id: Any = None, + require_custom_file: bool = False, + db=None, +) -> dict[str, Any]: + normalized_source = normalize_ambient_source(source) + normalized_volume = clamp_ambient_volume(volume) + if normalized_volume < MIN_AMBIENT_VOLUME or normalized_volume > MAX_AMBIENT_VOLUME: + raise ValueError( + f"background_noise_volume must be between {MIN_AMBIENT_VOLUME} and {MAX_AMBIENT_VOLUME}" + ) + + normalized_preset = None + resolved_asset_id = None + if normalized_source == BackgroundNoiseSourceEnum.PLATFORM.value: + normalized_preset = validate_ambient_preset(preset) + elif preset: + normalized_preset = normalize_ambient_preset(preset) + + if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value: + if organization_id is not None and not has_enterprise_entitlement(organization_id): + raise ValueError( + "Custom ambient audio requires a valid EfficientAI Enterprise license" + ) + resolved_asset_id = None + if asset_id: + if db is None: + resolved_asset_id = asset_id + else: + if db is None: + raise ValueError("Internal error resolving ambient asset") + from app.models.database import AmbientNoiseAsset + + query = db.query(AmbientNoiseAsset).filter( + AmbientNoiseAsset.id == asset_id, + AmbientNoiseAsset.organization_id == organization_id, + ) + if workspace_id is not None: + query = query.filter(AmbientNoiseAsset.workspace_id == workspace_id) + row = query.first() + if not row: + raise ValueError("Selected ambient library asset was not found") + resolved_asset_id = row.id + s3_key = None + elif require_custom_file and not s3_key: + raise ValueError("Select an uploaded ambient bed or upload one in the Background Noise tab") + if s3_key and not str(s3_key).strip(): + raise ValueError("background_noise_s3_key must not be empty") + + if normalized_source == BackgroundNoiseSourceEnum.NONE.value: + return { + "background_noise_source": BackgroundNoiseSourceEnum.NONE.value, + "background_noise_preset": None, + "background_noise_volume": normalized_volume, + "background_noise_s3_key": None, + "background_noise_asset_id": None, + } + + return { + "background_noise_source": normalized_source, + "background_noise_preset": normalized_preset, + "background_noise_volume": normalized_volume, + "background_noise_s3_key": s3_key if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value and not asset_id else None, + "background_noise_asset_id": resolved_asset_id if normalized_source == BackgroundNoiseSourceEnum.CUSTOM.value else None, + } + + +def persona_ambient_s3_key(organization_id: Any, persona_id: Any, extension: str) -> str: + from app.services.storage.s3_service import s3_service + + ext = extension.lower().lstrip(".") + return ( + f"{s3_service.prefix}organizations/{organization_id}/personas/{persona_id}/ambient.{ext}" + ) diff --git a/app/services/testing/agent_test_setup_generation.py b/app/services/testing/agent_test_setup_generation.py index ed4f0704..a1940120 100644 --- a/app/services/testing/agent_test_setup_generation.py +++ b/app/services/testing/agent_test_setup_generation.py @@ -13,23 +13,19 @@ from app.models.enums import ModelProvider from app.services.ai.llm_service import llm_service - -CANONICAL_SECTION_KEYS: tuple[str, ...] = ( - "purpose", - "behavior", - "expected_interactions", - "personality_traits", - "constraints", +from app.services.testing.test_agent_template import ( + CANONICAL_SECTION_KEYS, + CANONICAL_SECTION_TITLES, + TestAgentFirstMessage, + TestAgentPromptSection, + TestAgentTemplate, + assemble_test_agent_prompt, + derive_caller_first_message, + normalize_first_message, + normalize_sections, + template_from_generation, ) -CANONICAL_SECTION_TITLES: dict[str, str] = { - "purpose": "Purpose", - "behavior": "Behavior", - "expected_interactions": "Expected Interactions", - "personality_traits": "Personality Traits", - "constraints": "Constraints", -} - SCENARIO_DESCRIPTION_SECTIONS: tuple[str, ...] = ( "### Background (2-3 sentences)", "### Caller Intent (1-2 sentences)", @@ -40,23 +36,41 @@ GENERATE_TEST_PROMPT_SYSTEM = ( "You are an expert at generating synthetic voice AI test agents.\n\n" - "You will receive the system prompt of a production voice AI agent.\n\n" - "Your task is to generate the foundational system prompt for the complementary test agent.\n\n" + "You will receive the system prompt of a production voice AI agent (the assistant that answers calls).\n\n" + "Your task is to generate the foundational configuration for the complementary test caller " + "that will call this production agent during evaluation.\n\n" "Do NOT generate a specific persona or scenario. Those are supplied separately.\n\n" - "Instead, generate instructions that define how the synthetic caller should generally behave " - "when interacting with this production agent.\n\n" - "The generated prompt should:\n\n" - "- Infer the complementary role from the production agent.\n" - "- Describe the test agent's general responsibilities.\n" - "- Define conversational behaviour.\n" - "- Define how information should be disclosed.\n" - "- Define how questions should be answered.\n" - "- Encourage realistic, human conversation.\n" - "- Avoid mentioning testing, QA, automation, prompts or evaluation.\n" - "- Assume persona-specific details and scenario context will be injected later.\n" - "- Leave placeholders where appropriate for persona and scenario.\n\n" - "The prompt should be reusable across many personas and scenarios.\n" - "Return only the generated system prompt." + "Return ONLY valid JSON with this exact shape:\n" + "{\n" + ' "sections": [\n' + ' {"key": "complementary_goal", "title": "Role and Goal", "content": "..."},\n' + ' {"key": "talking_style", "title": "Talking Style", "content": "..."},\n' + ' {"key": "questions_to_ask", "title": "Questions to Ask", "content": "..."},\n' + ' {"key": "information_to_relay", "title": "Information to Relay", "content": "..."},\n' + ' {"key": "constraints", "title": "Constraints", "content": "..."}\n' + " ],\n" + ' "first_message": {\n' + ' "production_mode": "assistant_speaks_first" | "assistant_waits_for_user" | ' + '"assistant_speaks_first_model_generated",\n' + ' "production_message": "static greeting text or null"\n' + " }\n" + "}\n\n" + "Section guidance:\n" + "- complementary_goal: what the caller should ultimately achieve, inverted from the production agent's goals/criteria\n" + "- talking_style: how callers of this production agent typically speak (pacing, tone, phone realism)\n" + "- questions_to_ask: typical questions this caller type would ask the production agent\n" + "- information_to_relay: facts/details the caller should be ready to provide when asked\n" + "- constraints: boundaries (don't dump everything at once, don't mention testing/QA, stay realistic)\n\n" + "First message guidance:\n" + "- Infer who speaks first on the production side from greeting/opening instructions in the production prompt\n" + "- production_message: include the static greeting when production_mode is assistant_speaks_first, else null\n" + "- Do not include caller opening lines in sections; caller behavior is derived from production_mode\n\n" + "Rules:\n" + "- Avoid mentioning testing, QA, automation, prompts, or evaluation in section content\n" + "- Assume persona-specific details and scenario context will be injected later\n" + "- Leave placeholders where appropriate for persona and scenario\n" + "- The template must be reusable across many personas and scenarios\n" + "- Return only JSON, no markdown wrapper, no explanation" ) GENERATE_SCENARIOS_SYSTEM = ( @@ -66,17 +80,16 @@ ) -@dataclass -class AgentTestPromptSection: - key: str - title: str - content: str +# Re-export for backward compatibility in tests/imports +AgentTestPromptSection = TestAgentPromptSection @dataclass class TestPromptGenerationResult: - sections: List[AgentTestPromptSection] + sections: List[TestAgentPromptSection] test_agent_prompt: str + first_message: TestAgentFirstMessage + test_agent_template: TestAgentTemplate provider: str model: str @@ -99,11 +112,20 @@ def _strip_llm_text_wrapper(text: str) -> str: """Strip optional markdown fences and trim LLM preamble from plain-text responses.""" cleaned = (text or "").strip() if cleaned.startswith("```"): - cleaned = re.sub(r"^```(?:markdown|md|text)?\s*\n?", "", cleaned) + cleaned = re.sub(r"^```(?:markdown|md|text|json)?\s*\n?", "", cleaned) cleaned = re.sub(r"\n?```\s*$", "", cleaned) return cleaned.strip() +def _extract_json_object(text: str) -> Dict[str, Any]: + cleaned = _strip_llm_text_wrapper(text) + start = cleaned.find("{") + end = cleaned.rfind("}") + if start == -1 or end == -1 or end <= start: + raise ValueError("LLM response did not contain a JSON object") + return json.loads(cleaned[start : end + 1]) + + def _extract_json_array(text: str) -> List[Any]: cleaned = text.strip() if cleaned.startswith("```"): @@ -116,81 +138,6 @@ def _extract_json_array(text: str) -> List[Any]: return json.loads(cleaned[start : end + 1]) -def assemble_test_agent_prompt(sections: Sequence[AgentTestPromptSection]) -> str: - """Deterministically assemble canonical sections into markdown.""" - by_key = {section.key: section for section in sections} - ordered: list[AgentTestPromptSection] = [] - for key in CANONICAL_SECTION_KEYS: - section = by_key.get(key) - if section is None: - ordered.append( - AgentTestPromptSection( - key=key, - title=CANONICAL_SECTION_TITLES[key], - content="Not specified in source prompt.", - ) - ) - else: - ordered.append(section) - - parts: list[str] = [] - for section in ordered: - title = section.title.strip() or CANONICAL_SECTION_TITLES.get(section.key, section.key) - content = (section.content or "").strip() or "Not specified in source prompt." - parts.append(f"## {title}\n\n{content}") - return "\n\n".join(parts) - - -def _normalize_sections(raw_sections: Any) -> List[AgentTestPromptSection]: - if not isinstance(raw_sections, list): - raise ValueError("LLM response sections must be a list") - - by_key: dict[str, AgentTestPromptSection] = {} - for item in raw_sections: - if not isinstance(item, dict): - continue - key = str(item.get("key") or "").strip() - if key not in CANONICAL_SECTION_KEYS: - continue - title = str(item.get("title") or CANONICAL_SECTION_TITLES[key]).strip() - content = str(item.get("content") or "").strip() - by_key[key] = AgentTestPromptSection(key=key, title=title, content=content) - - sections: list[AgentTestPromptSection] = [] - for key in CANONICAL_SECTION_KEYS: - if key in by_key: - sections.append(by_key[key]) - else: - sections.append( - AgentTestPromptSection( - key=key, - title=CANONICAL_SECTION_TITLES[key], - content="Not specified in source prompt.", - ) - ) - return sections - - -def _normalize_scenario_drafts(raw: Any) -> List[ScenarioDraft]: - if not isinstance(raw, list): - raise ValueError("LLM response scenarios must be a list") - - drafts: list[ScenarioDraft] = [] - for item in raw: - if not isinstance(item, dict): - continue - name = str(item.get("name") or "").strip() - description = str(item.get("description") or "").strip() - if not name or not description: - continue - goal = item.get("goal") - goal_str = str(goal).strip() if goal else None - drafts.append(ScenarioDraft(name=name, description=description, goal=goal_str or None)) - if not drafts: - raise ValueError("LLM response did not contain valid scenario drafts") - return drafts - - def build_test_prompt_user_message( *, production_prompt: str, @@ -255,6 +202,26 @@ def build_scenario_generation_user_message( return "\n".join(parts) +def _parse_generation_payload(text: str) -> tuple[List[TestAgentPromptSection], TestAgentFirstMessage]: + try: + payload = _extract_json_object(text) + except (ValueError, json.JSONDecodeError) as exc: + logger.warning(f"[AgentTestSetup] Failed to parse test prompt JSON: {exc}") + raise ValueError("Could not parse generated test agent template from LLM response") from exc + + sections = normalize_sections(payload.get("sections")) + first_message_raw = payload.get("first_message") + if isinstance(first_message_raw, dict): + production_mode = str(first_message_raw.get("production_mode") or "").strip() + production_message = first_message_raw.get("production_message") + production_message_str = str(production_message).strip() if production_message else None + first_message = derive_caller_first_message(production_mode, production_message_str) + else: + first_message = normalize_first_message(first_message_raw) + + return sections, first_message + + def generate_test_prompt_from_production( production_prompt: str, *, @@ -269,7 +236,7 @@ def generate_test_prompt_from_production( llm_config: Optional[Dict[str, Any]] = None, credential_id: Optional[UUID] = None, ) -> TestPromptGenerationResult: - """Stage 1: generate foundational test agent prompt from production prompt.""" + """Stage 1: generate foundational test agent template from production prompt.""" if not production_prompt.strip(): raise ValueError("Production prompt is required") @@ -298,13 +265,18 @@ def generate_test_prompt_from_production( credential_id=credential_id, ) - test_agent_prompt = _strip_llm_text_wrapper(result["text"]) - if not test_agent_prompt: + sections, first_message = _parse_generation_payload(result["text"]) + test_agent_prompt = assemble_test_agent_prompt(sections) + if not test_agent_prompt.strip(): raise ValueError("LLM response did not contain a test agent prompt") + template = template_from_generation(sections, first_message) + return TestPromptGenerationResult( - sections=[], + sections=sections, test_agent_prompt=test_agent_prompt, + first_message=first_message, + test_agent_template=template, provider=llm_provider.value, model=llm_model, ) @@ -369,3 +341,23 @@ def generate_scenarios_from_test_prompt( provider=llm_provider.value, model=llm_model, ) + + +def _normalize_scenario_drafts(raw: Any) -> List[ScenarioDraft]: + if not isinstance(raw, list): + raise ValueError("LLM response scenarios must be a list") + + drafts: list[ScenarioDraft] = [] + for item in raw: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + description = str(item.get("description") or "").strip() + if not name or not description: + continue + goal = item.get("goal") + goal_str = str(goal).strip() if goal else None + drafts.append(ScenarioDraft(name=name, description=description, goal=goal_str or None)) + if not drafts: + raise ValueError("LLM response did not contain valid scenario drafts") + return drafts diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index b13b78b1..0fe59ed5 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -30,6 +30,11 @@ compose_test_agent_simulation_prompt, scenario_goal_from_required_info, ) +from app.services.testing.test_agent_template import ( + resolve_caller_opening_text, + resolve_first_message_from_agent, + should_caller_speak_first, +) from app.workers.celery_app import process_evaluator_result_task @@ -310,6 +315,7 @@ async def _connect_and_bridge_with_webrtc( webrtc_bridge = None test_agent = None + ambient_mic_pump = None # Helper function to update status async def update_status(new_status: str, event: str = None, error: str = None): @@ -596,10 +602,17 @@ def resolve_api_key_for_provider( test_agent = None else: scenario_goal = scenario_goal_from_required_info(scenario) - first_message = f"Hello, this is {persona.name} calling." - + first_message_config = resolve_first_message_from_agent(agent) + scenario_first_message = None if scenario.required_info and isinstance(scenario.required_info, dict): - first_message = scenario.required_info.get("first_message", first_message) + scenario_first_message = scenario.required_info.get("first_message") + + first_message = resolve_caller_opening_text( + first_message=first_message_config, + persona_name=persona.name or "Test Caller", + scenario_first_message=scenario_first_message, + ) + caller_speaks_first = should_caller_speak_first(first_message_config) persona_description = build_persona_description_for_bridge(persona) effective_max_turns = resolve_persona_max_turns(persona) @@ -623,7 +636,8 @@ def resolve_api_key_for_provider( persona_description=persona_description, scenario_description=getattr(scenario, "description", None) or scenario.name or "Test call scenario", scenario_goal=scenario_goal, - first_message=first_message, + first_message=first_message or "", + caller_speaks_first=caller_speaks_first, llm_api_key=llm_api_key, llm_temperature=getattr(persona, "llm_temperature", None), llm_max_tokens=getattr(persona, "llm_max_tokens", None), @@ -669,14 +683,43 @@ def touch_voice_activity() -> None: # Set up callbacks to connect test agent with voice provider chunk_ms = 40 if provider_platform == "vapi" else 20 + from app.services.audio.ambient_catalog import resolve_ambient_mixer + from app.services.audio.ambient_mic_pump import AmbientMicPump + + ambient_mixer = await resolve_ambient_mixer(persona, sample_rate) + if ambient_mixer: + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "suppress_background_silence"): + webrtc_bridge.suppress_background_silence() + mark_done = ( + webrtc_bridge.mark_user_audio_done + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done") + else None + ) + ambient_mic_pump = AmbientMicPump( + ambient_mixer.bed, + sample_rate=sample_rate, + chunk_duration_ms=chunk_ms, + send_callback=webrtc_bridge.receive_audio_from_test_agent, + mark_speech_done=mark_done, + ) + await ambient_mic_pump.start() + async def send_audio_chunks(audio: bytes): """Stream audio to voice provider in real-time chunks.""" touch_voice_activity() - await test_agent.stream_audio_chunks(audio, webrtc_bridge.receive_audio_from_test_agent, chunk_duration_ms=chunk_ms) - # Tell ElevenLabs bridge that we're done sending real audio - # so the background silence stream can resume immediately. - if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done"): - webrtc_bridge.mark_user_audio_done() + if ambient_mic_pump: + await ambient_mic_pump.send_speech( + audio, + test_agent.stream_audio_chunks, + ) + else: + await test_agent.stream_audio_chunks( + audio, + webrtc_bridge.receive_audio_from_test_agent, + chunk_duration_ms=chunk_ms, + ) + if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "mark_user_audio_done"): + webrtc_bridge.mark_user_audio_done() async def on_transcript_received(transcript: str): """When voice agent finishes speaking, process with test agent.""" @@ -726,14 +769,12 @@ async def on_call_should_end(): webrtc_bridge.on_agent_stop_talking = on_agent_stop_talking test_agent.on_call_should_end = on_call_should_end - # ElevenLabs agents have a built-in greeting; we skip the test - # agent's first message and let the ElevenLabs agent speak first. - # The background silence loop (started at connection time) simulates - # a live microphone so ElevenLabs' VAD activates normally. - if provider_platform == "elevenlabs": + # Honor template first-message config; ElevenLabs agents often greet first. + if provider_platform == "elevenlabs" and not caller_speaks_first: logger.info("[Bridge WebRTC] ElevenLabs: waiting for agent greeting (background silence stream active)") + elif not caller_speaks_first: + logger.info("[Bridge WebRTC] Test caller waiting for production agent to speak first") else: - # Retell / Vapi: test agent initiates the conversation logger.info("[Bridge WebRTC] Sending test agent's first message...") first_audio = await test_agent.generate_first_message() if first_audio: @@ -741,8 +782,10 @@ async def on_call_should_end(): await send_audio_chunks(first_audio) logger.info(f"[Bridge WebRTC] ✅ First message sent to {provider_platform}") else: - logger.error(f"[Bridge WebRTC] ⚠️ First message TTS returned no audio — test agent will be silent! " - f"Check TTS provider ({test_agent.config.tts_provider}) config and ffmpeg availability.") + logger.error( + f"[Bridge WebRTC] ⚠️ First message TTS returned no audio — test agent will be silent! " + f"Check TTS provider ({test_agent.config.tts_provider}) config and ffmpeg availability." + ) logger.info("[Bridge WebRTC] ✅ Test agent connected, conversation starting...") else: @@ -800,6 +843,8 @@ async def on_call_should_end(): await update_status(EvaluatorResultStatus.FAILED.value, "call_error", str(e)) finally: # Cleanup + if ambient_mic_pump: + await ambient_mic_pump.stop() if webrtc_bridge: await webrtc_bridge.disconnect() if test_agent: diff --git a/app/services/testing/test_agent_simulation_prompt.py b/app/services/testing/test_agent_simulation_prompt.py index ac1315ab..ddc63393 100644 --- a/app/services/testing/test_agent_simulation_prompt.py +++ b/app/services/testing/test_agent_simulation_prompt.py @@ -6,6 +6,7 @@ from typing import Any, Optional, Sequence, TYPE_CHECKING from app.models.database import Agent, Persona, Scenario +from app.services.testing.test_agent_template import SPOKEN_IDENTITY_GUARDRAIL if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -79,6 +80,24 @@ def format_scenario_prompt(scenario: Scenario) -> str: return "\n".join(parts) if parts else "General test call scenario" +def append_persona_identity_to_caller_prompt(caller_template: str, persona_name: str) -> str: + """Append explicit caller identity so the LLM speaks as the selected persona.""" + name = (persona_name or "").strip() + base = (caller_template or "").strip() + if not name: + return base or "Simulate a natural caller for the scenario below." + identity = ( + f"## Caller identity\n\n" + f"You are {name}. Introduce yourself as {name} and stay in character as {name} " + f"for the entire call." + ) + if not base: + return identity + if name.lower() in base.lower(): + return f"{base}\n\n{identity}" + return f"{base}\n\n{identity}" + + def compose_test_agent_simulation_prompt(agent: Agent, scenario: Scenario) -> str: """Merge core agent prompt and active scenario into test_agent_simulation_prompt.""" agent_name = (agent.name or "Voice AI Agent").strip() @@ -151,23 +170,26 @@ def build_test_agent_system_prompt( ) -> str: """Full caller LLM system prompt: simulation core + persona + instructions.""" under_test_name = (agent_name or agent.name or "Voice AI Agent").strip() + persona_name = (persona.name or "Caller").strip() effective_max_turns = max_turns if max_turns is not None else resolve_persona_max_turns(persona) simulation = compose_test_agent_simulation_prompt(agent, scenario) persona_block = format_persona_block( persona, persona_description=persona_description or build_persona_description_for_bridge(persona), ) - return f"""You are simulating a caller in a voice conversation. Your role is to test a voice AI agent. + return f"""You are {persona_name}, a real person on a phone call. -TEST AGENT SIMULATION PROMPT +{SPOKEN_IDENTITY_GUARDRAIL} + +CONTEXT (for your eyes only — do not read aloud) {simulation} PERSONA {persona_block} INSTRUCTIONS: -1. You are CALLING the voice AI agent described in the test agent simulation prompt -2. Stay in character as the persona described +1. You are CALLING the voice AI agent described above +2. Stay in character as {persona_name} at all times 3. Follow the scenario and work toward the goal 4. Speak naturally as if on a phone call 5. Keep responses concise (1-3 sentences) for natural conversation flow @@ -180,6 +202,72 @@ def build_test_agent_system_prompt( After {effective_max_turns} exchanges, wrap up the conversation politely.""" +def build_live_test_agent_system_prompt( + agent: Agent, + persona: Persona, + scenario: Scenario, + *, + max_turns: Optional[int] = None, + persona_description: Optional[str] = None, +) -> str: + """System prompt for live playground calls: caller template + persona + scenario. + + The human plays the production agent; the voice bundle simulates the caller. + """ + agent_name = (agent.name or "Voice AI Agent").strip() + persona_name = (persona.name or "Caller").strip() + effective_max_turns = max_turns if max_turns is not None else resolve_persona_max_turns(persona) + caller_template = append_persona_identity_to_caller_prompt( + strip_scenario_reference_appendix(agent.description or "").strip(), + persona_name, + ) + + production_context = (getattr(agent, "provider_prompt", None) or "").strip() + persona_block = format_persona_block( + persona, + persona_description=persona_description or build_persona_description_for_bridge(persona), + ) + scenario_block = format_scenario_prompt(scenario) + if persona_name: + scenario_block = f"{scenario_block}\n\nPlay this scenario as {persona_name}." + + parts = [ + f"You are {persona_name}, a real person on a phone call.", + SPOKEN_IDENTITY_GUARDRAIL, + "", + "CALLER PROMPT", + caller_template, + "", + "PERSONA", + persona_block, + "", + "SCENARIO", + scenario_block, + ] + if production_context: + parts.extend( + [ + "", + "PRODUCTION AGENT CONTEXT (what you are testing)", + production_context, + ] + ) + parts.extend( + [ + "", + "INSTRUCTIONS:", + f"1. You are {persona_name} — speak only as this person", + "2. Follow the scenario and work toward the goal", + "3. Speak naturally as if on a phone call", + "4. Keep responses concise (1-3 sentences)", + "5. Respond ONLY with spoken words — no stage directions or markdown", + f"6. The human on the other end is the production agent ({agent_name})", + f"7. After about {effective_max_turns} exchanges, wrap up politely.", + ] + ) + return "\n".join(parts) + + def format_scenarios_reference_appendix(scenarios: Sequence[Scenario]) -> str: """Markdown appendix listing linked scenarios using @scenario{uuid} reference tokens.""" if not scenarios: diff --git a/app/services/testing/test_agent_template.py b/app/services/testing/test_agent_template.py new file mode 100644 index 00000000..ecb419e6 --- /dev/null +++ b/app/services/testing/test_agent_template.py @@ -0,0 +1,249 @@ +"""Structured test agent template: sections, first-message config, and assembly.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Sequence + +CANONICAL_SECTION_KEYS: tuple[str, ...] = ( + "complementary_goal", + "talking_style", + "questions_to_ask", + "information_to_relay", + "constraints", +) + +CANONICAL_SECTION_TITLES: dict[str, str] = { + "complementary_goal": "Role and Goal", + "talking_style": "Talking Style", + "questions_to_ask": "Questions to Ask", + "information_to_relay": "Information to Relay", + "constraints": "Constraints", +} + +PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST = "assistant_speaks_first" +PRODUCTION_MODE_ASSISTANT_WAITS = "assistant_waits_for_user" +PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL = "assistant_speaks_first_model_generated" + +PRODUCTION_MODE_LABELS: dict[str, str] = { + PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST: "Assistant speaks first", + PRODUCTION_MODE_ASSISTANT_WAITS: "Assistant waits for user", + PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL: "Assistant speaks first with model-generated message", +} + +VALID_PRODUCTION_MODES: frozenset[str] = frozenset(PRODUCTION_MODE_LABELS) + +CALLER_MODE_WAIT = "wait" +CALLER_MODE_SPEAK_FIRST = "speak_first" + +DEFAULT_CALLER_MESSAGE = "Hello, I'm calling because I need some help." + +SPOKEN_IDENTITY_GUARDRAIL = ( + "Never say you are a test agent, AI, bot, simulator, or that you are testing or evaluating anything. " + "Never mention prompts, scenarios, personas, or automation. Speak only as the named caller." +) + + +def ensure_opening_includes_persona_name(opening: str, persona_name: str) -> str: + """Ensure the caller's first spoken line identifies as the persona when missing.""" + text = (opening or "").strip() + name = (persona_name or "").strip() + if not name: + return text + if text and name.lower() in text.lower(): + return text + if not text: + return f"Hello, this is {name} calling." + return f"Hi, this is {name}. {text}" + + +@dataclass +class TestAgentPromptSection: + key: str + title: str + content: str + + +@dataclass +class TestAgentFirstMessage: + production_mode: str = PRODUCTION_MODE_ASSISTANT_WAITS + production_message: Optional[str] = None + caller_mode: str = CALLER_MODE_SPEAK_FIRST + caller_message: Optional[str] = DEFAULT_CALLER_MESSAGE + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass +class TestAgentTemplate: + sections: List[TestAgentPromptSection] + first_message: TestAgentFirstMessage + + def to_dict(self) -> Dict[str, Any]: + return { + "sections": [ + {"key": s.key, "title": s.title, "content": s.content} + for s in self.sections + ], + "first_message": self.first_message.to_dict(), + } + + +def default_first_message() -> TestAgentFirstMessage: + """Legacy default: production waits, caller speaks first (matches Retell/Vapi bridge).""" + return TestAgentFirstMessage( + production_mode=PRODUCTION_MODE_ASSISTANT_WAITS, + production_message=None, + caller_mode=CALLER_MODE_SPEAK_FIRST, + caller_message=DEFAULT_CALLER_MESSAGE, + ) + + +def derive_caller_first_message(production_mode: str, production_message: Optional[str] = None) -> TestAgentFirstMessage: + """Invert production who-speaks-first into complementary caller behavior.""" + mode = production_mode if production_mode in VALID_PRODUCTION_MODES else PRODUCTION_MODE_ASSISTANT_WAITS + prod_msg = (production_message or "").strip() or None + + if mode in (PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST, PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST_MODEL): + return TestAgentFirstMessage( + production_mode=mode, + production_message=prod_msg if mode == PRODUCTION_MODE_ASSISTANT_SPEAKS_FIRST else None, + caller_mode=CALLER_MODE_WAIT, + caller_message=None, + ) + + return TestAgentFirstMessage( + production_mode=PRODUCTION_MODE_ASSISTANT_WAITS, + production_message=None, + caller_mode=CALLER_MODE_SPEAK_FIRST, + caller_message=DEFAULT_CALLER_MESSAGE, + ) + + +def assemble_test_agent_prompt(sections: Sequence[TestAgentPromptSection]) -> str: + """Deterministically assemble canonical sections into markdown.""" + by_key = {section.key: section for section in sections} + parts: list[str] = [] + for key in CANONICAL_SECTION_KEYS: + section = by_key.get(key) + title = (section.title.strip() if section else "") or CANONICAL_SECTION_TITLES[key] + content = (section.content.strip() if section and section.content else "") or "Not specified in source prompt." + parts.append(f"## {title}\n\n{content}") + return "\n\n".join(parts) + + +def empty_prompt_sections() -> List[TestAgentPromptSection]: + return [ + TestAgentPromptSection(key=key, title=CANONICAL_SECTION_TITLES[key], content="") + for key in CANONICAL_SECTION_KEYS + ] + + +def normalize_sections(raw_sections: Any) -> List[TestAgentPromptSection]: + if not isinstance(raw_sections, list): + raise ValueError("LLM response sections must be a list") + + by_key: dict[str, TestAgentPromptSection] = {} + for item in raw_sections: + if not isinstance(item, dict): + continue + key = str(item.get("key") or "").strip() + if key not in CANONICAL_SECTION_KEYS: + continue + title = str(item.get("title") or CANONICAL_SECTION_TITLES[key]).strip() + content = str(item.get("content") or "").strip() + by_key[key] = TestAgentPromptSection(key=key, title=title, content=content) + + sections: list[TestAgentPromptSection] = [] + for key in CANONICAL_SECTION_KEYS: + if key in by_key: + sections.append(by_key[key]) + else: + sections.append( + TestAgentPromptSection( + key=key, + title=CANONICAL_SECTION_TITLES[key], + content="Not specified in source prompt.", + ) + ) + return sections + + +def normalize_first_message(raw: Any) -> TestAgentFirstMessage: + if not isinstance(raw, dict): + return default_first_message() + + production_mode = str(raw.get("production_mode") or PRODUCTION_MODE_ASSISTANT_WAITS).strip() + production_message = raw.get("production_message") + production_message_str = str(production_message).strip() if production_message else None + + derived = derive_caller_first_message(production_mode, production_message_str) + + caller_mode = str(raw.get("caller_mode") or derived.caller_mode).strip() + if caller_mode not in (CALLER_MODE_WAIT, CALLER_MODE_SPEAK_FIRST): + caller_mode = derived.caller_mode + + caller_message = raw.get("caller_message") + caller_message_str = str(caller_message).strip() if caller_message else derived.caller_message + + return TestAgentFirstMessage( + production_mode=derived.production_mode, + production_message=derived.production_message, + caller_mode=caller_mode, + caller_message=caller_message_str if caller_mode == CALLER_MODE_SPEAK_FIRST else None, + ) + + +def parse_test_agent_template(raw: Any) -> Optional[TestAgentTemplate]: + if not isinstance(raw, dict): + return None + sections_raw = raw.get("sections") + if not isinstance(sections_raw, list): + return None + sections = normalize_sections(sections_raw) + first_message = normalize_first_message(raw.get("first_message")) + return TestAgentTemplate(sections=sections, first_message=first_message) + + +def template_from_generation( + sections: Sequence[TestAgentPromptSection], + first_message: TestAgentFirstMessage, +) -> TestAgentTemplate: + return TestAgentTemplate(sections=list(sections), first_message=first_message) + + +def resolve_first_message_from_agent(agent: Any) -> TestAgentFirstMessage: + """Read first-message config from agent row, with legacy default.""" + raw = getattr(agent, "test_agent_template", None) + parsed = parse_test_agent_template(raw) + if parsed is not None: + return parsed.first_message + return default_first_message() + + +def resolve_caller_opening_text( + *, + first_message: TestAgentFirstMessage, + persona_name: str, + scenario_first_message: Optional[str] = None, +) -> Optional[str]: + """Return caller opening line when caller speaks first; None when caller waits.""" + if first_message.caller_mode == CALLER_MODE_WAIT: + return None + + name = (persona_name or "Test Caller").strip() + opening: Optional[str] = None + + if scenario_first_message and str(scenario_first_message).strip(): + opening = str(scenario_first_message).strip() + elif first_message.caller_message and str(first_message.caller_message).strip(): + opening = str(first_message.caller_message).strip() + else: + opening = f"Hello, this is {name} calling." + + return ensure_opening_includes_persona_name(opening, name) + + +def should_caller_speak_first(first_message: TestAgentFirstMessage) -> bool: + return first_message.caller_mode == CALLER_MODE_SPEAK_FIRST diff --git a/app/services/voice_agent/bot_fast_api.py b/app/services/voice_agent/bot_fast_api.py index 3bcd92a7..4acffeff 100644 --- a/app/services/voice_agent/bot_fast_api.py +++ b/app/services/voice_agent/bot_fast_api.py @@ -98,7 +98,7 @@ def _get_imports(): """ -async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None): +async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None, persona=None): """ Run the voice agent bot with the provided Google API key. @@ -132,6 +132,11 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str transport_out_sample_rate = resolve_websocket_audio_out_sample_rate_hz( telephony_mode=telephony_mode, ) + ambient_mixer = None + if persona is not None: + from app.services.audio.ambient_catalog import resolve_ambient_mixer + + ambient_mixer = await resolve_ambient_mixer(persona, transport_out_sample_rate) ws_transport = imports["FastAPIWebsocketTransport"]( websocket=websocket_client, params=imports["FastAPIWebsocketParams"]( @@ -142,6 +147,7 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str audio_out_sample_rate=transport_out_sample_rate if telephony_mode else None, vad_analyzer=imports["SileroVADAnalyzer"](), serializer=transport_serializer, + audio_out_mixer=ambient_mixer, ), ) diff --git a/app/services/voice_agent/llm_voice_providers.py b/app/services/voice_agent/llm_voice_providers.py new file mode 100644 index 00000000..59bec721 --- /dev/null +++ b/app/services/voice_agent/llm_voice_providers.py @@ -0,0 +1,339 @@ +"""Live voice pipeline LLM provider registry and service factory. + +Mirrors the LLM-capable providers exposed in Voice Bundles / Integrations +(see ``app/services/judge_alignment/model_catalog.py``). +""" + +from __future__ import annotations + +import json +import os +from typing import Any, Callable, Dict, Optional + +from loguru import logger + +# Providers selectable for the LLM leg of STT+LLM+TTS voice bundles. +LLM_VOICE_PROVIDER_KEYS = frozenset( + { + "openai", + "anthropic", + "google", + "xai", + "fireworks", + "cohere", + "mistral", + "meta", + "together", + "perplexity", + "azure", + "aws", + "openrouter", + "custom", + "sarvam", + } +) + +_DEFAULT_LLM_MODELS: Dict[str, str] = { + "openai": "gpt-4.1", + "google": "gemini-2.5-flash", + "anthropic": "claude-sonnet-4.6", + "xai": "grok-3-beta", + "fireworks": "accounts/fireworks/models/llama-v3p1-8b-instruct", + "cohere": "command-r-plus-08-2024", + "mistral": "mistral-small-latest", + "meta": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "together": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "perplexity": "sonar", + "azure": "gpt-4.1", + "aws": "amazon.nova-lite-v1:0", + "openrouter": "openai/gpt-4o-2024-11-20", + "custom": "gpt-4o-mini", + "sarvam": "sarvam-30b", +} + +_ENV_KEYS: Dict[str, str] = { + "openai": "OPENAI_API_KEY", + "google": "GOOGLE_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "xai": "XAI_API_KEY", + "fireworks": "FIREWORKS_API_KEY", + "cohere": "COHERE_API_KEY", + "mistral": "MISTRAL_API_KEY", + "meta": "TOGETHER_API_KEY", + "together": "TOGETHER_API_KEY", + "perplexity": "PERPLEXITY_API_KEY", + "azure": "AZURE_OPENAI_API_KEY", + "aws": "AWS_ACCESS_KEY_ID", + "openrouter": "OPENROUTER_API_KEY", + "custom": "OPENAI_API_KEY", + "sarvam": "SARVAM_API_KEY", +} + + +def normalize_llm_model(provider: str, model: str) -> str: + """Normalize catalog model ids for provider-specific APIs.""" + provider_key = (provider or "").strip().lower() + if not model: + return model + if provider_key == "fireworks" and not model.startswith("accounts/"): + return f"accounts/fireworks/models/{model}" + if provider_key == "azure": + from app.services.ai.llm_service import _azure_deployment_name + + return _azure_deployment_name(model) + return model + + +def default_llm_model(provider: str) -> str: + return _DEFAULT_LLM_MODELS.get((provider or "").strip().lower(), "gpt-4.1") + + +def llm_env_key(provider: str) -> str: + return _ENV_KEYS.get((provider or "").strip().lower(), "OPENAI_API_KEY") + + +def _parse_aws_credentials(api_key: str) -> Dict[str, Any]: + """Parse AWS credential JSON or fall back to access key + env secret.""" + try: + parsed = json.loads(api_key) + if isinstance(parsed, dict): + access = ( + parsed.get("aws_access_key_id") + or parsed.get("access_key_id") + or parsed.get("aws_access_key") + ) + secret = ( + parsed.get("aws_secret_access_key") + or parsed.get("secret_access_key") + or parsed.get("aws_secret_key") + ) + return { + "aws_access_key": access, + "aws_secret_key": secret, + "aws_session_token": parsed.get("aws_session_token") + or parsed.get("session_token"), + "aws_region": parsed.get("aws_region") + or parsed.get("region") + or os.getenv("AWS_REGION", "us-east-1"), + } + except (json.JSONDecodeError, TypeError): + pass + return { + "aws_access_key": api_key, + "aws_secret_key": os.getenv("AWS_SECRET_ACCESS_KEY"), + "aws_region": os.getenv("AWS_REGION", "us-east-1"), + } + + +def get_llm_provider_registry(get_service: Callable[[str], Any]) -> Dict[str, Dict[str, Any]]: + """Build the LLM provider registry used by ``run_voice_bundle_fastapi``.""" + + def _openai_factory(api_key, model, params=None, base_url=None): + kwargs: Dict[str, Any] = {"api_key": api_key, "model": normalize_llm_model("openai", model)} + if params: + kwargs["params"] = params + if base_url: + kwargs["base_url"] = base_url + return get_service("OpenAILLMService")(**kwargs) + + registry: Dict[str, Dict[str, Any]] = {} + + for provider in sorted(LLM_VOICE_PROVIDER_KEYS): + env_key = llm_env_key(provider) + default_model = default_llm_model(provider) + + if provider == "openai": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, _f=_openai_factory: _f( + api_key, model, params + ), + } + elif provider == "google": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("GoogleLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "anthropic": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("AnthropicLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "fireworks": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("FireworksLLMService")( + api_key=api_key, + model=normalize_llm_model("fireworks", model), + **({"params": params} if params else {}), + ), + } + elif provider == "xai": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("GrokLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "mistral": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("MistralLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider in ("together", "meta"): + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("TogetherLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "perplexity": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("PerplexityLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "openrouter": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenRouterLLMService")( + api_key=api_key, + model=model, + **({"params": params} if params else {}), + ), + } + elif provider == "aws": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("AWSBedrockLLMService")( + model=model, + params=params, + **_parse_aws_credentials(api_key), + ), + } + elif provider == "cohere": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenAILLMService")( + api_key=api_key, + model=model, + base_url="https://api.cohere.com/compatibility/v1", + **({"params": params} if params else {}), + ), + } + elif provider == "sarvam": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, gs=get_service: gs("OpenAILLMService")( + api_key=api_key, + model=model, + base_url="https://api.sarvam.ai/v1", + **({"params": params} if params else {}), + ), + } + elif provider == "custom": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + "factory": lambda api_key, model, params=None, base_url=None, _f=_openai_factory: _f( + api_key, model, params, base_url=base_url + ), + "supports_base_url": True, + } + elif provider == "azure": + registry[provider] = { + "env_key": env_key, + "default_model": default_model, + # Azure is instantiated with endpoint metadata in run_voice_bundle_fastapi. + "factory": lambda api_key, model, params=None, _f=_openai_factory: _f( + api_key, normalize_llm_model("azure", model), params + ), + } + + return registry + + +def resolve_voice_llm_base_url(db, organization_id, voice_bundle, llm_provider) -> Optional[str]: + """Resolve an OpenAI-compatible base URL for custom / gateway-routed LLM legs.""" + provider_key = ( + llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) + ).lower() + if provider_key != "custom": + return None + + from app.services.credentials import resolve_ai_provider + + ai_provider = resolve_ai_provider( + provider_key, + db, + organization_id, + credential_id=getattr(voice_bundle, "llm_credential_id", None), + ) + if not ai_provider: + return None + + base_url = getattr(ai_provider, "gateway_base_url", None) + if base_url and str(base_url).strip(): + return str(base_url).strip() + return None + + +def instantiate_llm_service( + provider: str, + *, + get_service: Callable[[str], Any], + api_key: str, + model: str, + params: Optional[Any] = None, + base_url: Optional[str] = None, +): + """Instantiate a streaming LLM service for the live voice pipeline.""" + registry = get_llm_provider_registry(get_service) + provider_key = (provider or "").strip().lower() + cfg = registry.get(provider_key) + if cfg is None: + supported = ", ".join(sorted(registry.keys())) + raise ValueError( + f"Unsupported LLM provider '{provider_key}'. Supported providers: {supported}" + ) + + factory = cfg["factory"] + if provider_key == "custom" or cfg.get("supports_base_url"): + return factory(api_key, model, params, base_url=base_url) + if base_url: + logger.debug( + "Ignoring llm base_url for provider '{}' (not OpenAI-compatible custom routing)", + provider_key, + ) + return factory(api_key, model, params) diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index edebc4d0..d38049c0 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -153,6 +153,30 @@ def _get_service(service_name: str): elif service_name == "AzureLLMService": from efficientai.services.azure.llm import AzureLLMService service_class = AzureLLMService + elif service_name == "AnthropicLLMService": + from efficientai.services.anthropic.llm import AnthropicLLMService + service_class = AnthropicLLMService + elif service_name == "FireworksLLMService": + from efficientai.services.fireworks.llm import FireworksLLMService + service_class = FireworksLLMService + elif service_name == "GrokLLMService": + from efficientai.services.grok.llm import GrokLLMService + service_class = GrokLLMService + elif service_name == "MistralLLMService": + from efficientai.services.mistral.llm import MistralLLMService + service_class = MistralLLMService + elif service_name == "TogetherLLMService": + from efficientai.services.together.llm import TogetherLLMService + service_class = TogetherLLMService + elif service_name == "PerplexityLLMService": + from efficientai.services.perplexity.llm import PerplexityLLMService + service_class = PerplexityLLMService + elif service_name == "OpenRouterLLMService": + from efficientai.services.openrouter.llm import OpenRouterLLMService + service_class = OpenRouterLLMService + elif service_name == "AWSBedrockLLMService": + from efficientai.services.aws.llm import AWSBedrockLLMService + service_class = AWSBedrockLLMService # Optional: Smart Turn Analyzer elif service_name == "LocalSmartTurnAnalyzerV3": @@ -347,40 +371,10 @@ def _instantiate_tts_service( def _get_llm_providers(): - """Get LLM provider registry with truly lazy-loaded service classes. - - Each provider's SDK is only loaded when that provider is actually used. - """ - return { - "openai": { - "env_key": "OPENAI_API_KEY", - "default_model": "gpt-4.1", - "factory": lambda api_key, model, params=None: _get_service("OpenAILLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - "google": { - "env_key": "GOOGLE_API_KEY", - "default_model": "gemini-2.5-flash", - "factory": lambda api_key, model, params=None: _get_service("GoogleLLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - "azure": { - "env_key": "AZURE_OPENAI_API_KEY", - "default_model": "gpt-4.1", - # Azure is instantiated with endpoint metadata in run_voice_bundle_fastapi. - "factory": lambda api_key, model, params=None: _get_service("OpenAILLMService")( - api_key=api_key, - model=model, - **({"params": params} if params else {}), - ), - }, - } + """Get LLM provider registry with truly lazy-loaded service classes.""" + from app.services.voice_agent.llm_voice_providers import get_llm_provider_registry + + return get_llm_provider_registry(_get_service) DEFAULT_STT_PROVIDER = None @@ -522,10 +516,13 @@ async def run_voice_bundle_fastapi( tts_api_key: str | None = None, llm_api_key: str | None = None, llm_endpoint_url: str | None = None, + llm_base_url: str | None = None, serializer=None, telephony_mode: bool = False, call_short_id: str | None = None, silence_hangup_secs: float | None = None, + caller_speaks_first: bool = True, + caller_opening_text: str | None = None, ): """ Run the STT+LLM+TTS voice bundle pipeline over a FastAPI WebSocket. @@ -607,6 +604,11 @@ async def run_voice_bundle_fastapi( transport_out_sample_rate = resolve_websocket_audio_out_sample_rate_hz( telephony_mode=telephony_mode, ) + ambient_mixer = None + if persona is not None: + from app.services.audio.ambient_catalog import resolve_ambient_mixer + + ambient_mixer = await resolve_ambient_mixer(persona, transport_out_sample_rate) ws_transport = imports["FastAPIWebsocketTransport"]( websocket=websocket_client, params=imports["FastAPIWebsocketParams"]( @@ -617,6 +619,7 @@ async def run_voice_bundle_fastapi( audio_out_sample_rate=transport_out_sample_rate, vad_analyzer=imports["SileroVADAnalyzer"](params=imports["VADParams"](stop_secs=0.2)), serializer=transport_serializer, + audio_out_mixer=ambient_mixer, ), ) @@ -696,7 +699,16 @@ async def run_voice_bundle_fastapi( params=llm_params, ) else: - llm = llm_cfg["factory"](api_key=llm_api_key, model=llm_model, params=llm_params) + from app.services.voice_agent.llm_voice_providers import instantiate_llm_service + + llm = instantiate_llm_service( + llm_provider_value, + get_service=_get_service, + api_key=llm_api_key, + model=llm_model, + params=llm_params, + base_url=llm_base_url, + ) # Build context with provided system instruction or a default base_instruction = ( @@ -715,16 +727,15 @@ async def run_voice_bundle_fastapi( "or special Unicode characters. Use only plain spoken words." ) - messages = [ - { - "role": "system", - "content": base_instruction, - }, - { - "role": "user", - "content": "Start by greeting the user warmly and introducing yourself based on the system instruction.", - }, - ] + messages = [{"role": "system", "content": base_instruction}] + if caller_speaks_first: + bootstrap = ( + f"Say your opening line now, in character, word for word if possible: " + f"\"{caller_opening_text.strip()}\"" + if caller_opening_text and caller_opening_text.strip() + else "Introduce yourself using your name from the system prompt, then begin the scenario naturally." + ) + messages.append({"role": "user", "content": bootstrap}) context = imports["LLMContext"](messages) context_aggregator = imports["LLMContextAggregatorPair"](context) @@ -877,7 +888,8 @@ async def on_client_connected(transport, client): if not use_aligned_recorders: await audio_buffer_input.start_recording() await audio_buffer_output.start_recording() - await task.queue_frames([imports["LLMRunFrame"]()]) + if caller_speaks_first: + await task.queue_frames([imports["LLMRunFrame"]()]) @ws_transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): @@ -926,7 +938,10 @@ async def on_client_ready(rtvi): await audio_buffer_input.start_recording() await audio_buffer_output.start_recording() logger.info("AudioBufferProcessors started recording (input + output)") - await task.queue_frames([imports["LLMRunFrame"]()]) + if caller_speaks_first: + await task.queue_frames([imports["LLMRunFrame"]()]) + else: + logger.info("Caller waits for production agent — skipping initial LLM run") @ws_transport.event_handler("on_client_connected") async def on_client_connected(transport, client): diff --git a/app/services/webrtc_bridge/elevenlabs_ws_bridge.py b/app/services/webrtc_bridge/elevenlabs_ws_bridge.py index c5331e21..ac471cc4 100644 --- a/app/services/webrtc_bridge/elevenlabs_ws_bridge.py +++ b/app/services/webrtc_bridge/elevenlabs_ws_bridge.py @@ -102,6 +102,8 @@ def __init__( self._silence_task: Optional[asyncio.Task] = None # Background task that simulates a live microphone by sending silence self._bg_silence_task: Optional[asyncio.Task] = None + # When True, an external ambient mic pump feeds continuous PCM instead. + self._external_mic_feed = False # Suppresses background silence while the test agent is actively sending audio self._user_is_sending = False @@ -157,10 +159,9 @@ async def connect_to_elevenlabs(self) -> bool: # Start background message receiver asyncio.create_task(self._receive_loop()) - # Start continuous silence stream to simulate a live microphone. - # Without this, ElevenLabs' VAD stalls between turns because it - # expects a constant audio input (just like a browser mic provides). - self._bg_silence_task = asyncio.create_task(self._background_silence_loop()) + # Start continuous silence stream unless an external mic feed is active. + if not self._external_mic_feed: + self._bg_silence_task = asyncio.create_task(self._background_silence_loop()) return True @@ -201,6 +202,14 @@ def mark_user_audio_done(self): """ self._user_is_sending = False + def suppress_background_silence(self): + """Stop the zero-silence mic loop when an ambient mic pump is active.""" + self._external_mic_feed = True + if self._bg_silence_task and not self._bg_silence_task.done(): + self._bg_silence_task.cancel() + self._bg_silence_task = None + logger.info("[ElevenLabsWS] Background silence stream suppressed (external mic feed active)") + async def send_silence(self, duration_ms: int = 500): """Send an explicit silence buffer (e.g. trailing silence after an utterance).""" if not self.is_connected or not self._ws: diff --git a/app/services/webrtc_bridge/test_agent_processor.py b/app/services/webrtc_bridge/test_agent_processor.py index 990fa6bd..6272c867 100644 --- a/app/services/webrtc_bridge/test_agent_processor.py +++ b/app/services/webrtc_bridge/test_agent_processor.py @@ -63,6 +63,7 @@ class TestAgentConfig: scenario_description: str = "General inquiry call" scenario_goal: str = "Have a conversation and evaluate the agent" first_message: str = "Hello, I'm calling because I need some help." + caller_speaks_first: bool = True # Context about the voice AI agent being tested agent_name: str = "Voice AI Agent" @@ -211,11 +212,16 @@ async def generate_first_message(self) -> Optional[bytes]: Generate the first message to start the conversation. Returns: - Audio bytes of the first message, or None if failed + Audio bytes of the first message, or None if failed or caller waits """ + if not self.config.caller_speaks_first: + logger.info("[TestAgent] Caller configured to wait for production agent greeting") + return None + try: - # Use configured first message or generate one - first_text = self.config.first_message + first_text = (self.config.first_message or "").strip() + if not first_text: + return None # Add to conversation history self.conversation_history.append({ diff --git a/frontend/src/components/VoiceAgent.tsx b/frontend/src/components/VoiceAgent.tsx index 07253840..935271f0 100644 --- a/frontend/src/components/VoiceAgent.tsx +++ b/frontend/src/components/VoiceAgent.tsx @@ -26,11 +26,31 @@ interface VoiceAgentProps { compact?: boolean sidebarLayout?: boolean agentDisplayName?: string + connectDisabled?: boolean + connectDisabledReason?: string + runEvaluation?: boolean + userTranscriptLabel?: string + botTranscriptLabel?: string } type TranscriptEntry = { role: 'user' | 'agent'; content: string; timestamp: string } -export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpoint, customEndpointLabel, onSessionSaved, compact = false, sidebarLayout = false, agentDisplayName }: VoiceAgentProps) { +export default function VoiceAgent({ + personaId, + scenarioId, + agentId, + customEndpoint, + customEndpointLabel, + onSessionSaved, + compact = false, + sidebarLayout = false, + agentDisplayName, + connectDisabled = false, + connectDisabledReason, + runEvaluation = false, + userTranscriptLabel = 'You', + botTranscriptLabel = 'Agent', +}: VoiceAgentProps) { const { selectedAgent } = useAgentStore() // Use agentId prop if provided, otherwise fall back to selectedAgent from store @@ -419,6 +439,7 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo } if (!customEndpoint && personaId) params.append('persona_id', personaId) if (!customEndpoint && scenarioId) params.append('scenario_id', scenarioId) + if (!customEndpoint && runEvaluation) params.append('run_evaluation', 'true') if (!customEndpoint && params.toString()) { endpointUrl += `?${params.toString()}` @@ -534,7 +555,7 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo entry.role === 'user' ? 'bg-blue-50 text-blue-900 ml-2' : 'bg-gray-100 text-gray-800 mr-2' }`} > - {entry.role === 'user' ? 'You' : 'Agent'}: + {entry.role === 'user' ? userTranscriptLabel : botTranscriptLabel}: {entry.content} )) @@ -567,7 +588,8 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo onClick={connect} isLoading={isConnecting} leftIcon={isConnecting ? : } - disabled={isConnecting} + disabled={isConnecting || connectDisabled} + title={connectDisabled ? connectDisabledReason : undefined} > Start call @@ -658,7 +680,8 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo onClick={connect} isLoading={isConnecting} leftIcon={isConnecting ? : } - disabled={isConnecting} + disabled={isConnecting || connectDisabled} + title={connectDisabled ? connectDisabledReason : undefined} > Connect diff --git a/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx b/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx index 7107a677..c5ff07cf 100644 --- a/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx +++ b/frontend/src/components/call-recordings/TestVoiceAgentResultDetails.tsx @@ -190,7 +190,7 @@ interface TestVoiceAgentResultData { name?: string timestamp?: string duration_seconds?: number | null - status?: 'queued' | 'transcribing' | 'evaluating' | 'completed' | 'failed' + status?: 'queued' | 'transcribing' | 'evaluating' | 'completed' | 'failed' | 'call_ended' transcription?: string | null speaker_segments?: Array<{ speaker: string diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index c432705b..33863840 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1213,6 +1213,21 @@ class ApiClient { ): Promise<{ sections: Array<{ key: string; title: string; content: string }> test_agent_prompt: string + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + test_agent_template: { + sections: Array<{ key: string; title: string; content: string }> + first_message: { + production_mode: string + production_message?: string | null + caller_mode: string + caller_message?: string | null + } + } provider: string model: string }> { @@ -1383,6 +1398,72 @@ class ApiClient { return response.data } + async listAmbientPresets(): Promise<{ presets: { id: string; label: string }[] }> { + const response = await this.client.get('/api/v1/personas/ambient-presets') + return response.data + } + + async listAmbientLibrary(): Promise< + Array<{ + id: string + name: string + original_filename?: string | null + created_at?: string + }> + > { + const response = await this.client.get('/api/v1/personas/ambient-library') + return response.data + } + + async uploadAmbientLibraryAsset(file: File, name?: string): Promise { + const formData = new FormData() + formData.append('file', file) + if (name) { + formData.append('name', name) + } + const response = await this.client.post('/api/v1/personas/ambient-library', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async updateAmbientLibraryAsset(assetId: string, data: { name: string }): Promise { + const response = await this.client.patch(`/api/v1/personas/ambient-library/${assetId}`, data) + return response.data + } + + async deleteAmbientLibraryAsset(assetId: string): Promise { + await this.client.delete(`/api/v1/personas/ambient-library/${assetId}`) + } + + async previewAmbientLibraryAsset(assetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-library/${assetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async previewAmbientPreset(presetId: string): Promise { + const response = await this.client.get(`/api/v1/personas/ambient-presets/${presetId}/preview`, { + responseType: 'blob', + }) + return response.data + } + + async uploadPersonaAmbientAudio(personaId: string, file: File): Promise { + const formData = new FormData() + formData.append('file', file) + const response = await this.client.post(`/api/v1/personas/${personaId}/ambient-audio`, formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }) + return response.data + } + + async deletePersonaAmbientAudio(personaId: string): Promise { + const response = await this.client.delete(`/api/v1/personas/${personaId}/ambient-audio`) + return response.data + } + // Scenarios endpoints async listScenarios(skip = 0, limit = 100, agentId?: string): Promise { const response = await this.client.get('/api/v1/scenarios', { diff --git a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx index feba5879..d8247847 100644 --- a/frontend/src/pages/agents/AgentWorkspaceDetail.tsx +++ b/frontend/src/pages/agents/AgentWorkspaceDetail.tsx @@ -13,6 +13,12 @@ import AgentEditForm from './components/AgentEditForm' import AgentTalkSidebar, { type AgentTalkMode } from './components/AgentTalkSidebar' import { Save, X } from 'lucide-react' import { extractPhoneConflictDetail } from './components/agentPhoneValidation' +import { + TestAgentTemplateDraft, + assembleTestAgentPrompt, + defaultTestAgentTemplate, + templateFromApi, +} from './components/agentTestSetupConstants' const VALID_TABS: AgentDetailTab[] = ['overview', 'test_agent', 'voice_ai_agent'] @@ -33,6 +39,7 @@ interface FormData { phone_number: string language: string description: string + test_agent_template: TestAgentTemplateDraft prompt_variables: Record silence_hangup_secs: number call_type: string @@ -91,6 +98,7 @@ export default function AgentWorkspaceDetail({ phone_number: '', language: 'en', description: '', + test_agent_template: defaultTestAgentTemplate(), prompt_variables: {}, silence_hangup_secs: 15, call_type: 'outbound', @@ -140,6 +148,9 @@ export default function AgentWorkspaceDetail({ phone_number: agent.phone_number || '', language: agent.language, description: agent.description || '', + test_agent_template: agent.test_agent_template + ? templateFromApi(agent.test_agent_template) + : defaultTestAgentTemplate(), prompt_variables: agent.prompt_variables || {}, silence_hangup_secs: agent.silence_hangup_secs ?? 15, call_type: agent.call_type, @@ -160,7 +171,10 @@ export default function AgentWorkspaceDetail({ language: data.language, call_type: data.call_type, call_medium: data.call_medium, - description: data.description?.trim() || null, + description: + data.description?.trim() || + assembleTestAgentPrompt(data.test_agent_template.sections), + test_agent_template: data.test_agent_template, prompt_variables: data.prompt_variables || {}, silence_hangup_secs: data.silence_hangup_secs ?? 15, } @@ -279,6 +293,9 @@ export default function AgentWorkspaceDetail({ phone_number: agent.phone_number || '', language: agent.language, description: agent.description || '', + test_agent_template: agent.test_agent_template + ? templateFromApi(agent.test_agent_template) + : defaultTestAgentTemplate(), prompt_variables: agent.prompt_variables || {}, silence_hangup_secs: agent.silence_hangup_secs ?? 15, call_type: agent.call_type, @@ -370,10 +387,10 @@ export default function AgentWorkspaceDetail({ return (
-
-
+
+

@@ -421,7 +438,7 @@ export default function AgentWorkspaceDetail({

-
+
{!isEditMode ? ( silence_hangup_secs: number call_type: string @@ -71,13 +77,9 @@ export default function AgentEditForm({ agentId, }: AgentEditFormProps) { const navigate = useNavigate() - const [descriptionEditorMode, setDescriptionEditorMode] = useState<'write' | 'preview'>('write') const [providerPromptEditorMode, setProviderPromptEditorMode] = useState<'write' | 'preview'>('write') - const [showAIGeneratePanel, setShowAIGeneratePanel] = useState(false) - const [includeLinkedScenarios, setIncludeLinkedScenarios] = useState(true) - const [aiDescription, setAiDescription] = useState('') - const [aiTone, setAiTone] = useState('professional') - const [aiFormat, setAiFormat] = useState('structured') + const [showGenerateFromProductionPanel, setShowGenerateFromProductionPanel] = useState(false) + const [setupAdditionalContext, setSetupAdditionalContext] = useState('') const [aiCredentialId, setAiCredentialId] = useState('') const [aiModel, setAiModel] = useState('') const [phoneNumberInputMode, setPhoneNumberInputMode] = useState<'provider' | 'custom'>('provider') @@ -175,29 +177,43 @@ export default function AgentEditForm({ telephonyNumbers, ]) - const generateDescriptionMutation = useMutation({ - mutationFn: (data: { - description: string - tone?: string - format_style?: string - provider?: string - model?: string - agent_id?: string - include_linked_scenarios?: boolean - append_scenarios_to_output?: boolean - }) => apiClient.generateAgentDescription(data), + const generateFromProductionMutation = useMutation({ + mutationFn: () => { + if (!formData.provider_prompt?.trim()) { + throw new Error('Production prompt is required') + } + return apiClient.generateTestPromptFromProduction({ + production_prompt: formData.provider_prompt, + agent_name: formData.name, + language: formData.language, + call_type: formData.call_type, + additional_context: setupAdditionalContext.trim() || undefined, + ...(aiProvider ? { provider: aiProvider } : {}), + ...(aiCredentialId ? { credential_id: aiCredentialId } : {}), + ...(aiModel ? { model: aiModel } : {}), + }) + }, onSuccess: (data) => { - onChange({ ...formData, description: data.content }) - setShowAIGeneratePanel(false) - setAiDescription('') - setDescriptionEditorMode('preview') - showToast('Description generated successfully!', 'success') + const nextTemplate = applyGeneratedTemplate(formData.test_agent_template, data) + onChange({ + ...formData, + test_agent_template: nextTemplate, + description: data.test_agent_prompt || assembleTestAgentPrompt(nextTemplate.sections), + }) + setShowGenerateFromProductionPanel(false) + showToast('Test agent template generated from production prompt', 'success') }, onError: (err: any) => { - showToast(err?.response?.data?.detail || 'Failed to generate description with AI', 'error') + showToast( + err?.message || err?.response?.data?.detail || 'Failed to generate from production', + 'error', + ) }, }) + const hasStructuredTemplate = isTemplateFilled(formData.test_agent_template) + const showLegacyPrompt = Boolean(formData.description?.trim() && !hasStructuredTemplate) + const voiceAgentIntegrations = integrations.filter( (integration) => integration.is_active && @@ -212,6 +228,29 @@ export default function AgentEditForm({ ? voiceBundles.find((vb) => vb.id === formData.voice_bundle_id) : undefined + const testAgentConfigured = Boolean( + linkedVoiceBundle && linkedVoiceBundle.is_active !== false, + ) + const voiceAiConfigured = Boolean( + formData.voice_ai_integration_id?.trim() && formData.voice_ai_agent_id?.trim(), + ) + const voiceBundleLabel = linkedVoiceBundle + ? linkedVoiceBundle.name + : formData.voice_bundle_id + ? 'Unknown bundle' + : OVERVIEW_NOT_CONFIGURED + const voiceAiIntegrationLabel = selectedVoiceIntegration + ? (() => { + const platformLabel = getIntegrationPlatformLabel( + selectedVoiceIntegration.platform as IntegrationPlatform, + ) + const name = selectedVoiceIntegration.name?.trim() + return name ? `${name} (${platformLabel})` : platformLabel + })() + : formData.voice_ai_integration_id + ? 'Unknown integration' + : OVERVIEW_NOT_CONFIGURED + const hasPlatformLink = Boolean(formData.voice_ai_integration_id || formData.voice_ai_agent_id) const productionPromptProse = @@ -220,7 +259,16 @@ export default function AgentEditForm({ return (
{activeTab === 'overview' && ( -
+
+
+

Overview

+

+ Identity, call routing, voice stacks, and session behavior. +

+
+ +
+
@@ -460,11 +508,53 @@ export default function AgentEditForm({ Delete agent
+
+ +
+ +
+ } + /> + +
+ + + +
+ } + /> + + + {formData.voice_ai_agent_id.trim()} + + ) : ( + OVERVIEW_NOT_CONFIGURED + ) + } + /> +
+ +
+
)} {activeTab === 'test_agent' && ( -
+
{testAgentSubTab === 'configuration' && ( @@ -494,7 +584,7 @@ export default function AgentEditForm({ {linkedVoiceBundle && ( { const returnPath = agentId ? `/agents/${agentId}?tab=test_agent` : '/agents' navigate( @@ -508,137 +598,69 @@ export default function AgentEditForm({ )} {testAgentSubTab === 'prompt' && ( -
-
- +
+
+
-
- - -
- {/* AI Generate Panel */} - {showAIGeneratePanel && ( -
-
- - Generate Description with AI -
-

- Describe what this agent should do and AI will generate a rich markdown description. + {showGenerateFromProductionPanel && ( +

+

+ Uses the production agent prompt from the Voice AI Agent tab to generate complementary + caller sections and first-message settings.

+ {!formData.provider_prompt?.trim() ? ( +

+ Add a production prompt on the Voice AI Agent tab first. +

+ ) : null}