diff --git a/.gitignore b/.gitignore index cf2f92d4..4a1a6bf6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ MANIFEST secrets/ *.json !package.json +!tests/fixtures/**/*.json .cursorrules # Virtual environments diff --git a/app/api/v1/routes/call_imports.py b/app/api/v1/routes/call_imports.py index 70551508..e0fe902a 100644 --- a/app/api/v1/routes/call_imports.py +++ b/app/api/v1/routes/call_imports.py @@ -1756,7 +1756,7 @@ async def preview_call_import_file( file_bytes = await file.read() if len(file_bytes) > MAX_UPLOAD_BYTES: raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + status_code=status.HTTP_413_CONTENT_TOO_LARGE, detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", ) @@ -1835,7 +1835,7 @@ async def create_call_import( file_bytes = await file.read() if len(file_bytes) > MAX_UPLOAD_BYTES: raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + status_code=status.HTTP_413_CONTENT_TOO_LARGE, detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", ) @@ -2313,7 +2313,7 @@ async def upload_call_import_csv( file_bytes = await file.read() if len(file_bytes) > MAX_UPLOAD_BYTES: raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + status_code=status.HTTP_413_CONTENT_TOO_LARGE, detail=f"File exceeds {MAX_UPLOAD_BYTES} bytes", ) diff --git a/app/api/v1/routes/data_sources.py b/app/api/v1/routes/data_sources.py index ce96497a..647cf3aa 100644 --- a/app/api/v1/routes/data_sources.py +++ b/app/api/v1/routes/data_sources.py @@ -44,7 +44,7 @@ async def test_s3_connection(api_key: str = Depends(get_api_key)): if not enabled: error = s3_service.get_status_message() or "Blob storage is not enabled or not configured." raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=f"Blob storage connection test failed: {error}", ) diff --git a/app/api/v1/routes/iam.py b/app/api/v1/routes/iam.py index 169f82e2..827fdb5d 100644 --- a/app/api/v1/routes/iam.py +++ b/app/api/v1/routes/iam.py @@ -192,7 +192,7 @@ async def update_organization( name = payload.name.strip() if not name: raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="Organization name cannot be empty.", ) diff --git a/app/api/v1/routes/integrations.py b/app/api/v1/routes/integrations.py index 338f295b..0dd54648 100644 --- a/app/api/v1/routes/integrations.py +++ b/app/api/v1/routes/integrations.py @@ -3,6 +3,7 @@ Manage integrations with external voice AI platforms (Retell, Vapi, etc.) """ from fastapi import APIRouter, Depends, HTTPException, status, Query +from celery import chain from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from datetime import datetime, timezone @@ -10,16 +11,29 @@ from uuid import UUID from loguru import logger -from app.dependencies import get_db, get_organization_id, get_api_key -from app.models.database import Integration, IntegrationPlatform, Agent +from app.dependencies import get_db, get_organization_id, get_api_key, get_workspace_id +from app.models.database import ( + Integration, + IntegrationPlatform, + Agent, + ProviderSyncJob, +) from app.models.schemas import ( IntegrationCreate, IntegrationUpdate, IntegrationResponse, PreviewIntegrationAgentPromptRequest, PreviewIntegrationAgentPromptResponse, + ExternalAgentListResponse, + ElevenLabsConversationSyncRequest, + ProviderSyncJobResponse, ) from app.core.encryption import encrypt_api_key, decrypt_api_key from app.services.credentials.resolver import clear_other_defaults from app.services.voice_providers import get_voice_provider from app.services.ai.llm_gateway import get_credential_effective_routing_label +from app.workers.celery_app import ( + sync_elevenlabs_agents_task, + sync_elevenlabs_catalog_task, + sync_elevenlabs_enrich_task, +) router = APIRouter(prefix="/integrations", tags=["Integrations"]) @@ -38,6 +52,31 @@ def _integration_response( return response.model_copy(update={"effective_routing": effective_routing}) +def _provider_sync_job_response(job: ProviderSyncJob) -> ProviderSyncJobResponse: + config = job.config if isinstance(job.config, dict) else {} + cursor_state = job.cursor_state if isinstance(job.cursor_state, dict) else {} + return ProviderSyncJobResponse.model_validate( + { + "id": job.id, + "integration_id": job.integration_id, + "provider_platform": job.provider_platform, + "status": job.status, + "phase": job.phase, + "config": config, + "cursor_state": cursor_state, + "agents_synced": job.agents_synced or 0, + "conversations_cataloged": job.conversations_cataloged or 0, + "conversations_enriched": job.conversations_enriched or 0, + "errors_count": job.errors_count or 0, + "last_error": job.last_error, + "started_at": job.started_at, + "completed_at": job.completed_at, + "created_at": job.created_at, + "updated_at": job.updated_at, + } + ) + + def _validate_smallest_connection(raw_api_key: str): """Validate a Smallest key via GET /atoms/v1/user.""" try: @@ -471,3 +510,256 @@ async def preview_integration_agent_prompt( ) return PreviewIntegrationAgentPromptResponse(provider_prompt=prompt) + + +@router.get( + "/{integration_id}/external-agents", + response_model=ExternalAgentListResponse, + operation_id="listIntegrationExternalAgents", +) +async def list_integration_external_agents( + integration_id: UUID, + search: str | None = Query(None), + cursor: str | None = Query(None), + page_size: int = Query(30, ge=1, le=100), + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """List provider-side agents for an integration credential.""" + del api_key + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + platform_value = integration.platform.value if hasattr(integration.platform, "value") else str(integration.platform).lower() + if platform_value not in { + IntegrationPlatform.ELEVENLABS.value, + IntegrationPlatform.VAPI.value, + IntegrationPlatform.RETELL.value, + }: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="External agent listing is currently supported only for ElevenLabs, Vapi, and Retell integrations", + ) + + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + provider_class = get_voice_provider(platform_value) + provider = provider_class(api_key=decrypted_api_key) + if not hasattr(provider, "list_agents"): + raise ValueError("Selected provider does not support listing agents") + payload = provider.list_agents(page_size=page_size, search=search, cursor=cursor) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to list provider agents: {str(e)}", + ) + + return ExternalAgentListResponse.model_validate(payload) + + +def _load_active_integration(db: Session, organization_id: UUID, integration_id: UUID) -> Integration: + integration = ( + db.query(Integration) + .filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + .first() + ) + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + return integration + + +def _ensure_no_running_sync_job( + db: Session, + *, + organization_id: UUID, + integration_id: UUID, +) -> None: + existing = ( + db.query(ProviderSyncJob) + .filter( + ProviderSyncJob.organization_id == organization_id, + ProviderSyncJob.integration_id == integration_id, + ProviderSyncJob.status.in_(["queued", "running"]), + ) + .order_by(ProviderSyncJob.created_at.desc()) + .first() + ) + if existing: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"A sync job is already active for this integration ({existing.id})", + ) + + +@router.post( + "/{integration_id}/sync/elevenlabs/agents", + response_model=ProviderSyncJobResponse, + operation_id="startElevenLabsAgentSync", +) +async def start_elevenlabs_agent_sync( + integration_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + integration = _load_active_integration(db, organization_id, integration_id) + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + if platform_value != IntegrationPlatform.ELEVENLABS.value: + raise HTTPException(status_code=400, detail="This endpoint only supports ElevenLabs integrations") + _ensure_no_running_sync_job( + db, + organization_id=organization_id, + integration_id=integration.id, + ) + + job = ProviderSyncJob( + organization_id=organization_id, + workspace_id=workspace_id, + integration_id=integration.id, + provider_platform=IntegrationPlatform.ELEVENLABS.value, + status="queued", + phase="agents", + config={"insights_only": True}, + cursor_state={}, + ) + db.add(job) + db.commit() + db.refresh(job) + + sync_elevenlabs_agents_task.delay(str(job.id)) + return _provider_sync_job_response(job) + + +@router.post( + "/{integration_id}/sync/elevenlabs/conversations", + response_model=ProviderSyncJobResponse, + operation_id="startElevenLabsConversationSync", +) +async def start_elevenlabs_conversation_sync( + integration_id: UUID, + body: ElevenLabsConversationSyncRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + integration = _load_active_integration(db, organization_id, integration_id) + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + if platform_value != IntegrationPlatform.ELEVENLABS.value: + raise HTTPException(status_code=400, detail="This endpoint only supports ElevenLabs integrations") + _ensure_no_running_sync_job( + db, + organization_id=organization_id, + integration_id=integration.id, + ) + + since_unix = body.since_unix + if since_unix is None: + since_unix = int((datetime.now(timezone.utc).timestamp()) - (30 * 24 * 60 * 60)) + + job = ProviderSyncJob( + organization_id=organization_id, + workspace_id=workspace_id, + integration_id=integration.id, + provider_platform=IntegrationPlatform.ELEVENLABS.value, + status="queued", + phase="catalog", + config={ + "since_unix": since_unix, + "agent_ids": body.agent_ids or [], + "insights_only": bool(body.insights_only), + }, + cursor_state={}, + ) + db.add(job) + db.commit() + db.refresh(job) + + chain( + sync_elevenlabs_catalog_task.si(str(job.id)), + sync_elevenlabs_enrich_task.si(str(job.id)), + ).delay() + return _provider_sync_job_response(job) + + +@router.get( + "/{integration_id}/sync/jobs/{job_id}", + response_model=ProviderSyncJobResponse, + operation_id="getProviderSyncJob", +) +async def get_provider_sync_job( + integration_id: UUID, + job_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key, workspace_id + job = ( + db.query(ProviderSyncJob) + .filter( + ProviderSyncJob.id == job_id, + ProviderSyncJob.integration_id == integration_id, + ProviderSyncJob.organization_id == organization_id, + ) + .first() + ) + if not job: + raise HTTPException(status_code=404, detail="Sync job not found") + return _provider_sync_job_response(job) + + +@router.post( + "/{integration_id}/sync/jobs/{job_id}/cancel", + response_model=ProviderSyncJobResponse, + operation_id="cancelProviderSyncJob", +) +async def cancel_provider_sync_job( + integration_id: UUID, + job_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key, workspace_id + job = ( + db.query(ProviderSyncJob) + .filter( + ProviderSyncJob.id == job_id, + ProviderSyncJob.integration_id == integration_id, + ProviderSyncJob.organization_id == organization_id, + ) + .first() + ) + if not job: + raise HTTPException(status_code=404, detail="Sync job not found") + if job.status not in {"completed", "failed", "cancelled"}: + job.status = "cancelled" + job.phase = "cancelled" + job.completed_at = datetime.now(timezone.utc) + db.commit() + db.refresh(job) + return _provider_sync_job_response(job) diff --git a/app/api/v1/routes/observability.py b/app/api/v1/routes/observability.py index 4c26a549..bf770860 100644 --- a/app/api/v1/routes/observability.py +++ b/app/api/v1/routes/observability.py @@ -1,28 +1,73 @@ """Observability routes for external call ingestion and retrieval.""" import random -from datetime import datetime +from datetime import UTC, datetime, timedelta from typing import Any, Dict, List, Optional, Union -from uuid import UUID +from uuid import UUID, uuid4 +import httpx from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status -from pydantic import BaseModel, ConfigDict +from loguru import logger +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from app.config import settings from app.dependencies import get_api_key, get_db, get_organization_id, get_workspace_id from app.services.billing.flexprice_service import ( - record_observability_call_evaluated, record_observability_call_ingested, + record_observability_call_evaluated, +) +from app.services.observability.call_ingest import ( + OBSERVABILITY_CALL_SOURCES, + upsert_call_recording, + upsert_live_event_call_recording, ) from app.models.database import ( - Agent, APIKey, CallRecording, CallRecordingStatus, CallRecordingSource, + Agent, APIKey, CallRecording, CallRecordingSource, Integration, IntegrationPlatform, + CallRecordingStatus, Evaluator, EvaluatorResult, EvaluatorResultStatus, Scenario, Workspace, + ObservabilityLiveEventDedup, + ObservabilityLiveSloBreach, +) +from app.core.encryption import decrypt_api_key +from app.services.voice_providers import get_voice_provider +from app.services.observability.elevenlabs_trace import ( + enrich_with_turn_metrics, + extract_trace_id, + normalize_elevenlabs_otlp, ) -from app.utils.call_recordings import generate_unique_call_short_id +from app.services.observability.provider_call_enrichment import ( + is_sparse_provider_call_data, + resolve_observability_provider_platform, +) +from app.services.observability.recording_archive import archive_observability_recording_to_s3 +from app.services.observability.retell_trace import build_retell_synthetic_trace +from app.services.observability.trace_archive import ( + load_provider_trace, + persist_provider_trace, +) +from app.services.observability.vapi_trace import build_vapi_synthetic_trace +from app.services.observability.live_ingest import StaleLiveEventError, parse_live_event_ts +from app.services.observability.live_latency import ( + query_live_latency_metrics, + record_live_latency_samples, +) +from app.services.observability.live_slo import evaluate_llm_p90_slo +from app.services.observability.live_trace import build_live_synthetic_trace from app.workers.celery_app import process_evaluator_result_task router = APIRouter(prefix="/observability", tags=["observability"]) +_LIVE_SYNTHETIC_PLATFORMS = frozenset({"pipecat", "livekit"}) + + +def _should_build_live_synthetic_trace(provider_platform: str, call_data: Dict[str, Any]) -> bool: + platform = (provider_platform or "").strip().lower() + if platform in _LIVE_SYNTHETIC_PLATFORMS: + return True + return isinstance(call_data.get("live_transcript"), list) + class CallIngestionPayload(BaseModel): """Flat payload for ingesting a single call record from an external source.""" @@ -38,6 +83,7 @@ class CallIngestionPayload(BaseModel): endedReason: Optional[str] = None recording_url: Optional[str] = None provider_platform: Optional[str] = None + trace_id: Optional[str] = None model_config = ConfigDict( extra="allow", @@ -65,6 +111,63 @@ class CallIngestionPayload(BaseModel): ) +class LiveEventEnvelope(BaseModel): + """Platform-neutral incremental live event envelope.""" + + event_id: str = Field(..., min_length=3, max_length=128) + call_id: str = Field(..., min_length=1, max_length=255) + event_type: str = Field(..., min_length=3, max_length=64) + seq: Optional[int] = Field(default=None, ge=0) + event_ts: str + platform: str = Field(..., min_length=2, max_length=64) + agent_ref: Optional[str] = Field(default=None, max_length=128) + payload: Dict[str, Any] = Field(default_factory=dict) + trace_id: Optional[str] = Field(default=None, max_length=64) + + model_config = ConfigDict(extra="allow") + + +def _ensure_live_ingest_enabled() -> None: + if not settings.OBSERVABILITY_LIVE_INGEST_ENABLED: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Live ingest is disabled for this deployment", + ) + + +def _ensure_live_aggregates_enabled() -> None: + if not settings.OBSERVABILITY_LIVE_AGGREGATES_ENABLED: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Live aggregates are disabled for this deployment", + ) + + +def _validate_live_event_ts_drift(event_ts: datetime) -> None: + max_drift = max(0, int(settings.OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS)) + now = datetime.now(UTC) + if abs((now - event_ts).total_seconds()) > max_drift: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "event_ts drift exceeds accepted window. " + f"max={max_drift}s" + ), + ) + + +def _cleanup_expired_live_dedup_rows(db: Session, organization_id: UUID) -> None: + db.query(ObservabilityLiveEventDedup).filter( + ObservabilityLiveEventDedup.organization_id == organization_id, + ObservabilityLiveEventDedup.expires_at < datetime.now(UTC), + ).delete(synchronize_session=False) + db.flush() + + +def _issue_efficientai_trace_id() -> str: + return uuid4().hex + + def _serialize_agent_summary(agent: Optional[Agent]) -> Optional[Dict[str, Any]]: """Serialize a minimal agent summary for call list/detail responses.""" if not agent: @@ -119,6 +222,11 @@ def _serialize_call_recording( "source": call_recording.source.value if call_recording.source else None, "provider_platform": call_recording.provider_platform, "provider_call_id": call_recording.provider_call_id, + "trace_id": call_recording.trace_id, + "last_live_event_ts": call_data.get("_live_last_event_ts"), + "evaluator_result_id": ( + str(call_recording.evaluator_result_id) if call_recording.evaluator_result_id else None + ), "agent_id": str(call_recording.agent_id) if call_recording.agent_id else None, "agent": _serialize_agent_summary(agent), "created_at": call_recording.created_at.isoformat() if call_recording.created_at else None, @@ -158,6 +266,31 @@ def _resolve_default_workspace_id(db: Session, organization_id: UUID) -> UUID: return default_ws.id +def _resolve_agent_id_from_ref( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_ref_raw: Optional[str], +) -> Optional[UUID]: + if not agent_ref_raw: + return None + try: + return UUID(str(agent_ref_raw)) + except ValueError: + pass + linked_agent = ( + db.query(Agent) + .filter( + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + Agent.voice_ai_agent_id == str(agent_ref_raw), + ) + .first() + ) + return linked_agent.id if linked_agent else None + + def _upsert_call_recording( *, db: Session, @@ -166,76 +299,41 @@ def _upsert_call_recording( provider_platform: str, provider_call_id: str, call_data_payload: Dict[str, Any], - agent_ref_raw: Optional[str], + agent_ref_raw: Optional[str] = None, explicit_agent_id: Optional[UUID] = None, call_event: Optional[str] = None, + trace_id: Optional[str] = None, source: CallRecordingSource = CallRecordingSource.WEBHOOK, + trigger_auto_evaluate: bool = False, ) -> Dict[str, Any]: - """Create/update a call recording for an organization + workspace. - - If the referenced agent lives in a different workspace, prefer the agent's - workspace so the recording stays co-located with its agent for filtering. - """ - # Attempt to link to an internal agent when a UUID is provided (unless explicit agent provided) - agent_id: Optional[UUID] = explicit_agent_id - if not agent_id and agent_ref_raw: - try: - agent_uuid = UUID(agent_ref_raw) - agent = ( - db.query(Agent) - .filter(Agent.id == agent_uuid, Agent.organization_id == organization_id) - .first() - ) - if agent: - agent_id = agent.id - if agent.workspace_id and agent.workspace_id != workspace_id: - workspace_id = agent.workspace_id - except ValueError: - # Not a UUID; treat as external reference only - agent_id = None - - # Preserve external agent reference alongside provider payload - if agent_ref_raw: - call_data_payload.setdefault("_agent_ref", agent_ref_raw) - - call_recording = ( - db.query(CallRecording) - .filter( - CallRecording.organization_id == organization_id, - CallRecording.provider_call_id == provider_call_id, - CallRecording.provider_platform == provider_platform, - ) - .first() + """Create/update a call recording and return the serialized API payload.""" + call_recording, action = upsert_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + provider_platform=provider_platform, + provider_call_id=provider_call_id, + call_data_payload=call_data_payload, + agent_ref_raw=agent_ref_raw, + explicit_agent_id=explicit_agent_id, + call_event=call_event, + trace_id=trace_id, + source=source, ) - - if call_recording: - call_recording.call_data = call_data_payload - call_recording.status = CallRecordingStatus.UPDATED - call_recording.source = source - if call_event: - call_recording.call_event = call_event - if agent_id: - call_recording.agent_id = agent_id - db.commit() - db.refresh(call_recording) - action = "updated" - else: - call_recording = CallRecording( - organization_id=organization_id, - workspace_id=workspace_id, - call_short_id=generate_unique_call_short_id(db), - status=CallRecordingStatus.UPDATED, - call_event=call_event, - source=source, - call_data=call_data_payload, - provider_call_id=provider_call_id, - provider_platform=provider_platform, - agent_id=agent_id, - ) - db.add(call_recording) - db.commit() - db.refresh(call_recording) - action = "created" + if isinstance(call_recording.call_data, dict): + _maybe_archive_observability_recording(db, call_recording) + payload = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + platform = (call_recording.provider_platform or "").strip().lower() + if platform in {"retell", "vapi", "elevenlabs"} and _is_terminal_observability_call(payload): + if _maybe_persist_provider_trace( + db, + call_recording, + call_data=payload, + provider_platform=platform, + ): + db.commit() + db.refresh(call_recording) + _warn_if_trace_quota_exceeded(db, organization_id, call_recording) agent_obj = None if call_recording.agent_id: @@ -243,16 +341,52 @@ def _upsert_call_recording( response = _serialize_call_recording(call_recording, include_data=True, agent=agent_obj) response["action"] = action - if action == "created": - record_observability_call_ingested( - organization_id, - call_recording.call_short_id, + if trigger_auto_evaluate: + _maybe_auto_evaluate_call_recording( + db=db, + organization_id=organization_id, workspace_id=workspace_id, - provider=provider_platform, + call_recording=call_recording, + agent=agent_obj, ) return response +def _warn_if_trace_quota_exceeded( + db: Session, + organization_id: UUID, + call_recording: CallRecording, +) -> None: + quota = settings.OBSERVABILITY_TRACE_QUOTA_PER_ORG_PER_DAY + if quota is None or quota <= 0: + return + trace_id = call_recording.trace_id + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + if not trace_id: + trace_id = call_data.get("trace_id") + if not trace_id: + return + + day_start = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) + trace_count = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.created_at >= day_start, + CallRecording.trace_id.isnot(None), + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .count() + ) + if trace_count > quota: + logger.warning( + "Observability trace quota exceeded for org {}: count={} quota={}", + organization_id, + trace_count, + quota, + ) + + def _validate_webhook_api_key(api_key: str, db: Session) -> UUID: """Validate an API key from a webhook URL and return the organization ID.""" db_key = db.query(APIKey).filter( @@ -263,13 +397,18 @@ def _validate_webhook_api_key(api_key: str, db: Session) -> UUID: status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or inactive API key", ) - db_key.last_used = datetime.utcnow() + db_key.last_used = datetime.now(UTC) db.commit() return db_key.organization_id -def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Session) -> Dict[str, Any]: - """Process a flat CallIngestionPayload-style body in the org's default workspace.""" +def _process_flat_payload_for_workspace( + body: Dict[str, Any], + organization_id: UUID, + workspace_id: UUID, + db: Session, +) -> Dict[str, Any]: + """Process a flat CallIngestionPayload-style body in a specific workspace.""" payload = CallIngestionPayload.model_validate(body) provider_call_id = payload.id @@ -278,7 +417,7 @@ def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Sessi call_data_payload: Dict[str, Any] = {} for field in ( "startedAt", "endedAt", "to_phone_number", "from_phone_number", - "messages", "metadata", "endedReason", "recording_url", + "messages", "metadata", "endedReason", "recording_url", "trace_id", ): value = getattr(payload, field, None) if value is not None: @@ -288,6 +427,12 @@ def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Sessi call_data_payload.update(payload.model_extra) agent_ref_raw = str(payload.agent_id) if payload.agent_id is not None else None + explicit_agent_id = _resolve_agent_id_from_ref( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_ref_raw=agent_ref_raw, + ) call_event: Optional[str] = None if payload.endedAt: @@ -295,8 +440,6 @@ def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Sessi elif payload.startedAt: call_event = "call_started" - workspace_id = _resolve_default_workspace_id(db, organization_id) - return _upsert_call_recording( db=db, organization_id=organization_id, @@ -305,11 +448,20 @@ def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Sessi provider_call_id=provider_call_id, call_data_payload=call_data_payload, agent_ref_raw=agent_ref_raw, + explicit_agent_id=explicit_agent_id, call_event=call_event, + trace_id=payload.trace_id or call_data_payload.get("trace_id"), source=CallRecordingSource.WEBHOOK, + trigger_auto_evaluate=True, ) +def _process_flat_payload(body: Dict[str, Any], organization_id: UUID, db: Session) -> Dict[str, Any]: + """Process a flat CallIngestionPayload-style body in the org's default workspace.""" + workspace_id = _resolve_default_workspace_id(db, organization_id) + return _process_flat_payload_for_workspace(body, organization_id, workspace_id, db) + + def _process_provider_payload(body: Dict[str, Any], organization_id: UUID, db: Session) -> Dict[str, Any]: """Process a provider webhook payload in the org's default workspace.""" call_payload = body.get("call") or body.get("call_data") @@ -341,6 +493,12 @@ def _process_provider_payload(body: Dict[str, Any], organization_id: UUID, db: S call_event = body.get("event") or call_data_payload.pop("_event", None) workspace_id = _resolve_default_workspace_id(db, organization_id) + explicit_agent_id = _resolve_agent_id_from_ref( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_ref_raw=str(agent_ref_raw) if agent_ref_raw else None, + ) return _upsert_call_recording( db=db, @@ -350,9 +508,414 @@ def _process_provider_payload(body: Dict[str, Any], organization_id: UUID, db: S provider_call_id=provider_call_id, call_data_payload=call_data_payload, agent_ref_raw=str(agent_ref_raw) if agent_ref_raw else None, + explicit_agent_id=explicit_agent_id, call_event=call_event, + trace_id=body.get("trace_id") or call_data_payload.get("trace_id"), source=CallRecordingSource.WEBHOOK, + trigger_auto_evaluate=True, + ) + + +def _process_named_provider_payload( + body: Dict[str, Any], + organization_id: UUID, + db: Session, + provider_platform: str, +) -> Dict[str, Any]: + normalized = dict(body) + normalized["provider_platform"] = provider_platform + if "call" not in normalized and "call_data" not in normalized: + if "call_id" in normalized or "id" in normalized: + normalized["call"] = dict(normalized) + if "event" not in normalized: + status_value = normalized.get("status") or normalized.get("call_status") + if status_value: + normalized["event"] = str(status_value) + return _process_provider_payload(normalized, organization_id, db) + + +def _refresh_call_data_from_provider( + *, + db: Session, + organization_id: UUID, + call_recording: CallRecording, +) -> Dict[str, Any]: + """Fetch latest provider call payload for an existing call recording.""" + provider_call_id = call_recording.provider_call_id + provider_platform = (call_recording.provider_platform or "").strip().lower() + if not provider_platform or provider_platform == "external": + provider_platform = resolve_observability_provider_platform(call_recording, db=db) + if provider_platform and provider_platform != "external": + call_recording.provider_platform = provider_platform + db.commit() + db.refresh(call_recording) + if not provider_call_id or not provider_platform or provider_platform == "external": + raise ValueError("Call does not have provider information") + if not call_recording.agent_id: + raise ValueError("Call is not linked to an internal agent") + + agent = ( + db.query(Agent) + .filter( + Agent.id == call_recording.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == call_recording.workspace_id, + ) + .first() + ) + if not agent or not agent.voice_ai_integration_id: + raise ValueError("Agent or voice integration not found") + + integration = ( + db.query(Integration) + .filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + .first() + ) + if not integration: + raise ValueError("Integration not found") + + decrypted_api_key = decrypt_api_key(integration.api_key) + provider_class = get_voice_provider(provider_platform) + provider_kwargs: Dict[str, Any] = {"api_key": decrypted_api_key} + if provider_platform == IntegrationPlatform.VAPI.value and integration.public_key: + provider_kwargs["public_key"] = integration.public_key + provider = provider_class(**provider_kwargs) + refreshed_call_data = provider.retrieve_call_metrics(str(provider_call_id)) + if not isinstance(refreshed_call_data, dict) or not refreshed_call_data: + raise ValueError("Provider returned empty call metrics payload") + return refreshed_call_data + + +def _resolve_provider_api_key_for_call( + db: Session, + organization_id: UUID, + call_recording: CallRecording, +) -> Optional[str]: + if not call_recording.agent_id: + return None + agent = ( + db.query(Agent) + .filter( + Agent.id == call_recording.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == call_recording.workspace_id, + ) + .first() + ) + if not agent or not agent.voice_ai_integration_id: + return None + integration = ( + db.query(Integration) + .filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + .first() + ) + if not integration or not integration.api_key: + return None + return decrypt_api_key(integration.api_key) + + +def _is_terminal_observability_call(call_data: Dict[str, Any]) -> bool: + status_name = str( + call_data.get("call_status") + or call_data.get("status") + or "" + ).lower().strip() + return bool( + status_name in {"ended", "completed", "done", "failed", "call_ended"} + or call_data.get("endedAt") + or call_data.get("ended_at") + or call_data.get("end_timestamp") + or call_data.get("endedReason") + or call_data.get("ended_reason") + or call_data.get("disconnection_reason") + ) + + +def _maybe_archive_observability_recording( + db: Session, + call_recording: CallRecording, + call_data: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + payload = call_data if isinstance(call_data, dict) else ( + call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + ) + if not payload or not _is_terminal_observability_call(payload): + return payload + + provider_platform = (call_recording.provider_platform or payload.get("provider_platform") or "").strip().lower() + if not provider_platform: + return payload + if ( + provider_platform == IntegrationPlatform.ELEVENLABS.value + and settings.OBSERVABILITY_ELEVENLABS_SKIP_RECORDING_ARCHIVE + and not payload.get("force_recording_archive") + ): + return payload + + provider_api_key = _resolve_provider_api_key_for_call( + db=db, + organization_id=call_recording.organization_id, + call_recording=call_recording, + ) + archived = archive_observability_recording_to_s3( + call_data=payload, + provider_platform=provider_platform, + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + provider_api_key=provider_api_key, + ) + if archived.get("recording_s3_key") and archived.get("recording_s3_key") != payload.get("recording_s3_key"): + call_recording.call_data = archived + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + db.refresh(call_recording) + return archived + + +def _maybe_enrich_sparse_provider_call( + db: Session, + organization_id: UUID, + call_recording: CallRecording, +) -> CallRecording: + """Pull full provider metrics when a hosted call row only has webhook-lite data.""" + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + if not call_data or not call_recording.provider_call_id or not call_recording.agent_id: + return call_recording + + provider_platform = resolve_observability_provider_platform(call_recording, call_data, db=db) + if provider_platform in {"", "external"}: + return call_recording + if provider_platform != (call_recording.provider_platform or "").strip().lower(): + call_recording.provider_platform = provider_platform + + if not _is_terminal_observability_call(call_data): + if call_recording.provider_platform != provider_platform: + db.commit() + db.refresh(call_recording) + return call_recording + + if not is_sparse_provider_call_data(call_data, provider_platform): + if call_recording.provider_platform != provider_platform: + db.commit() + db.refresh(call_recording) + return call_recording + + try: + refreshed = _refresh_call_data_from_provider( + db=db, + organization_id=organization_id, + call_recording=call_recording, + ) + call_recording.call_data = refreshed + call_recording.status = CallRecordingStatus.UPDATED + _maybe_archive_observability_recording(db, call_recording, refreshed) + _maybe_persist_provider_trace( + db, + call_recording, + call_data=refreshed, + provider_platform=provider_platform, + ) + db.commit() + db.refresh(call_recording) + except Exception as exc: + logger.warning( + "Sparse provider call enrichment failed for call_short_id={}: {}", + call_recording.call_short_id, + exc, + ) + db.rollback() + return call_recording + + +def _prepare_observability_call_recording( + db: Session, + organization_id: UUID, + call_recording: CallRecording, + *, + enrich: bool = True, +) -> CallRecording: + """Load sharded call payloads and optionally pull full provider metrics.""" + from app.services.live_entity_storage import hydrate_call_recordings + + hydrate_call_recordings([call_recording]) + if enrich: + call_recording = _maybe_enrich_sparse_provider_call(db, organization_id, call_recording) + return call_recording + + +def _resolve_elevenlabs_integration_for_call( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + call_data: Dict[str, Any], +) -> Optional[Integration]: + integration_id = call_data.get("integration_id") + if integration_id: + try: + parsed = UUID(str(integration_id)) + row = db.query(Integration).filter( + Integration.id == parsed, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + row_platform = row.platform.value if (row and hasattr(row.platform, "value")) else str(getattr(row, "platform", "")).lower() + if row and row_platform == IntegrationPlatform.ELEVENLABS.value: + return row + except Exception: + pass + + if call_recording.agent_id: + agent = db.query(Agent).filter( + Agent.id == call_recording.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + if agent and agent.voice_ai_integration_id: + row = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + row_platform = row.platform.value if (row and hasattr(row.platform, "value")) else str(getattr(row, "platform", "")).lower() + if row and row_platform == IntegrationPlatform.ELEVENLABS.value: + return row + + provider_agent_id = call_data.get("agent_id") + if provider_agent_id: + agent = db.query(Agent).filter( + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + Agent.voice_ai_agent_id == str(provider_agent_id), + ).first() + if agent and agent.voice_ai_integration_id: + row = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + row_platform = row.platform.value if (row and hasattr(row.platform, "value")) else str(getattr(row, "platform", "")).lower() + if row and row_platform == IntegrationPlatform.ELEVENLABS.value: + return row + + return None + + +def _build_elevenlabs_trace_from_stored_data( + call_data: Dict[str, Any], + provider_call_id: str, +) -> Optional[Dict[str, Any]]: + stored_trace = load_provider_trace(call_data) + if stored_trace: + return stored_trace + + provider_trace = call_data.get("provider_trace") + if not isinstance(provider_trace, dict): + return None + otlp_payload = provider_trace.get("otlp_traces") + if not isinstance(otlp_payload, dict): + return None + + normalized = normalize_elevenlabs_otlp( + otlp_payload, + conversation_id=provider_call_id, + fallback_trace_id=provider_trace.get("trace_id"), + ) + transcript = call_data.get("transcript") + if isinstance(transcript, list): + normalized = enrich_with_turn_metrics(normalized, transcript) + return normalized + + +def _maybe_persist_provider_trace( + db: Session, + call_recording: CallRecording, + *, + call_data: Optional[Dict[str, Any]] = None, + provider_platform: Optional[str] = None, + source: Optional[str] = None, + raw_payload: Optional[Dict[str, Any]] = None, +) -> bool: + payload = call_data if isinstance(call_data, dict) else ( + call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + ) + if not payload: + return False + + platform = (provider_platform or call_recording.provider_platform or "").strip().lower() + if not platform: + return False + + existing_provider_trace = payload.get("provider_trace") if isinstance(payload.get("provider_trace"), dict) else {} + existing_source = existing_provider_trace.get("source") if isinstance(existing_provider_trace, dict) else None + explicit_trace_id = payload.get("trace_id") or call_recording.trace_id + + trace_payload: Optional[Dict[str, Any]] = None + if platform == "vapi": + trace_payload = build_vapi_synthetic_trace( + payload, + provider_call_id=str(call_recording.provider_call_id or payload.get("id") or call_recording.call_short_id), + ) + elif platform == "retell": + trace_payload = build_retell_synthetic_trace( + payload, + provider_call_id=str( + call_recording.provider_call_id or payload.get("call_id") or call_recording.call_short_id + ), + ) + elif platform == "elevenlabs": + provider_call_id = str( + call_recording.provider_call_id + or payload.get("conversation_id") + or payload.get("call_id") + or call_recording.call_short_id + ) + trace_payload = _build_elevenlabs_trace_from_stored_data(payload, provider_call_id) + if raw_payload is None: + provider_trace = payload.get("provider_trace") + if isinstance(provider_trace, dict) and isinstance(provider_trace.get("otlp_traces"), dict): + raw_payload = provider_trace.get("otlp_traces") + elif _should_build_live_synthetic_trace(platform, payload): + trace_payload = build_live_synthetic_trace( + payload, + provider_call_id=str( + call_recording.provider_call_id or payload.get("id") or call_recording.call_short_id + ), + provider_platform=platform or "external", + trace_id=str(explicit_trace_id) if explicit_trace_id else None, + ) + + if not trace_payload: + return False + + trace_source = source or str(existing_source or trace_payload.get("trace_source") or f"{platform}_synthetic") + updated = persist_provider_trace( + call_data=payload, + provider_platform=platform, + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=trace_payload, + source=trace_source, + raw_payload=raw_payload if isinstance(raw_payload, dict) else None, ) + call_recording.call_data = updated + trace_id = explicit_trace_id or trace_payload.get("trace_id") or updated.get("trace_id") + if explicit_trace_id: + updated["trace_id"] = explicit_trace_id + if trace_id: + call_recording.trace_id = str(trace_id) + call_recording.status = CallRecordingStatus.UPDATED + db.flush() + return True @router.post("/calls/webhook/retell/{api_key}", status_code=status.HTTP_201_CREATED) @@ -371,7 +934,286 @@ async def ingest_retell_webhook( ``{"event": "call_ended", "call": {...}}`` """ organization_id = _validate_webhook_api_key(api_key, db) - return _process_provider_payload(body, organization_id, db) + normalized = dict(body) + event_value = str(normalized.get("event") or "").lower().strip() + if event_value in {"call_ended", "call_analyzed", "call_completed", "end_of_call_report", "end-of-call-report", "completed"}: + normalized["event"] = "call_ended" + + response = _process_named_provider_payload(normalized, organization_id, db, "retell") + call_data = response.get("call_data") if isinstance(response, dict) else None + call_short_id = response.get("call_short_id") if isinstance(response, dict) else None + + def _is_terminal(payload: Dict[str, Any]) -> bool: + status_name = str(payload.get("call_status") or payload.get("status") or "").lower().strip() + return bool( + status_name in {"ended", "completed", "failed", "call_ended"} + or payload.get("end_timestamp") + or payload.get("endedAt") + or payload.get("disconnection_reason") + ) + + def _is_incomplete(payload: Dict[str, Any]) -> bool: + has_transcript = bool( + (isinstance(payload.get("transcript_object"), list) and len(payload.get("transcript_object")) > 0) + or (isinstance(payload.get("messages"), list) and len(payload.get("messages")) > 0) + or (isinstance(payload.get("transcript"), str) and payload.get("transcript").strip()) + ) + has_analysis = isinstance(payload.get("call_analysis"), dict) and len(payload.get("call_analysis")) > 0 + has_cost = isinstance(payload.get("call_cost"), dict) and len(payload.get("call_cost")) > 0 + return not (has_transcript and has_analysis and has_cost) + + if isinstance(call_data, dict) and call_short_id and _is_terminal(call_data): + row = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if row: + should_refresh = _is_incomplete(call_data) or is_sparse_provider_call_data(call_data, "retell") + if should_refresh: + try: + refreshed = _refresh_call_data_from_provider( + db=db, + organization_id=organization_id, + call_recording=row, + ) + row.call_data = refreshed + row.status = CallRecordingStatus.UPDATED + _maybe_archive_observability_recording(db, row, refreshed) + _maybe_persist_provider_trace( + db, + row, + call_data=refreshed, + provider_platform="retell", + ) + db.commit() + db.refresh(row) + agent = db.query(Agent).filter(Agent.id == row.agent_id).first() if row.agent_id else None + response = _serialize_call_recording(row, include_data=True, agent=agent) + except Exception as exc: + logger.warning(f"[RetellWebhook] Pull fallback refresh failed for {call_short_id}: {exc}") + + return response + + +@router.post("/calls/webhook/elevenlabs/{api_key}", status_code=status.HTTP_201_CREATED) +async def ingest_elevenlabs_webhook( + api_key: str, + body: Dict[str, Any], + db: Session = Depends(get_db), +) -> Dict[str, Any]: + organization_id = _validate_webhook_api_key(api_key, db) + if body.get("type") == "post_call_transcription_otel" and isinstance(body.get("data"), dict): + data = body["data"] + conversation_id = data.get("conversation_id") + otlp_payload = data.get("otlp_traces") + if conversation_id and isinstance(otlp_payload, dict): + workspace_id = _resolve_default_workspace_id(db, organization_id) + provider_agent_id = data.get("agent_id") + integration_id = None + internal_agent_id = None + if provider_agent_id: + linked_agent = db.query(Agent).filter( + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + Agent.voice_ai_agent_id == str(provider_agent_id), + ).first() + if linked_agent: + internal_agent_id = str(linked_agent.id) + if linked_agent.voice_ai_integration_id: + integration_id = str(linked_agent.voice_ai_integration_id) + + extracted_trace_id = extract_trace_id(otlp_payload) + normalized_trace = normalize_elevenlabs_otlp( + otlp_payload, + conversation_id=str(conversation_id), + fallback_trace_id=extracted_trace_id, + ) + transcript_payload = data.get("transcript") + if isinstance(transcript_payload, list): + normalized_trace = enrich_with_turn_metrics(normalized_trace, transcript_payload) + call_data_payload: Dict[str, Any] = { + "id": conversation_id, + "call_id": conversation_id, + "agent_id": provider_agent_id, + "provider_platform": "elevenlabs", + "status": data.get("status") or "done", + "transcript": transcript_payload, + "conversation_id": conversation_id, + } + call_data_payload = persist_provider_trace( + call_data=call_data_payload, + provider_platform="elevenlabs", + organization_id=organization_id, + call_short_id=f"el-{str(conversation_id)[:24]}", + trace_payload=normalized_trace, + source="elevenlabs_post_call_webhook", + raw_payload=otlp_payload, + ) + extracted_trace_id = call_data_payload.get("trace_id") or extracted_trace_id + if integration_id: + call_data_payload["integration_id"] = integration_id + + # Reconcile against a previously-created pending call row (e.g. playground + # web call bootstrapped before ElevenLabs conversation_id was known). + if internal_agent_id: + try: + internal_agent_uuid = UUID(str(internal_agent_id)) + pending_row = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.provider_platform == "elevenlabs", + CallRecording.provider_call_id.is_(None), + CallRecording.agent_id == internal_agent_uuid, + ) + .order_by(CallRecording.created_at.desc()) + .first() + ) + if pending_row: + pending_row.provider_call_id = conversation_id + pending_row.call_data = call_data_payload + pending_row.call_event = "call_ended" + pending_row.status = CallRecordingStatus.UPDATED + pending_row.source = CallRecordingSource.WEBHOOK + if extracted_trace_id: + pending_row.trace_id = extracted_trace_id + _maybe_persist_provider_trace( + db, + pending_row, + call_data=call_data_payload, + provider_platform="elevenlabs", + source="elevenlabs_post_call_webhook", + raw_payload=otlp_payload, + ) + db.commit() + db.refresh(pending_row) + agent_obj = db.query(Agent).filter(Agent.id == pending_row.agent_id).first() + response = _serialize_call_recording( + pending_row, + include_data=True, + agent=agent_obj, + ) + response["action"] = "updated" + _maybe_auto_evaluate_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=pending_row, + agent=agent_obj, + ) + return response + except Exception: + db.rollback() + + return _process_provider_payload( + { + "provider_platform": "elevenlabs", + "provider_call_id": conversation_id, + "event": "call_ended", + "trace_id": extracted_trace_id, + "agent_id": internal_agent_id or provider_agent_id, + "call": call_data_payload, + }, + organization_id, + db, + ) + return _process_named_provider_payload(body, organization_id, db, "elevenlabs") + + +@router.post("/calls/webhook/vapi/{api_key}", status_code=status.HTTP_201_CREATED) +async def ingest_vapi_webhook( + api_key: str, + body: Dict[str, Any], + db: Session = Depends(get_db), +) -> Dict[str, Any]: + organization_id = _validate_webhook_api_key(api_key, db) + normalized = dict(body) + status_value = str( + normalized.get("status") + or normalized.get("call_status") + or (normalized.get("call") or {}).get("status") + or "", + ).lower().strip() + if status_value in {"ended", "completed", "end-of-call-report", "done", "failed"}: + normalized["event"] = "call_ended" + + response = _process_named_provider_payload(normalized, organization_id, db, "vapi") + call_data = response.get("call_data") if isinstance(response, dict) else None + call_short_id = response.get("call_short_id") if isinstance(response, dict) else None + + def _is_terminal(payload: Dict[str, Any]) -> bool: + status_name = str(payload.get("status") or payload.get("call_status") or "").lower().strip() + return bool( + status_name in {"ended", "completed", "end-of-call-report", "done", "failed"} + or payload.get("endedAt") + or payload.get("ended_at") + or payload.get("endedReason") + or payload.get("ended_reason") + ) + + def _is_incomplete(payload: Dict[str, Any]) -> bool: + has_messages = bool( + (isinstance(payload.get("messages"), list) and len(payload.get("messages")) > 0) + or ( + isinstance(payload.get("artifact"), dict) + and isinstance(payload.get("artifact", {}).get("messages"), list) + and len(payload.get("artifact", {}).get("messages")) > 0 + ) + ) + has_analysis = isinstance(payload.get("analysis"), dict) and len(payload.get("analysis")) > 0 + has_cost = bool( + payload.get("cost") is not None + or ( + isinstance(payload.get("costBreakdown"), dict) + and len(payload.get("costBreakdown")) > 0 + ) + or ( + isinstance(payload.get("cost_breakdown"), dict) + and len(payload.get("cost_breakdown")) > 0 + ) + ) + return not (has_messages and has_analysis and has_cost) + + if isinstance(call_data, dict) and call_short_id and _is_terminal(call_data) and _is_incomplete(call_data): + row = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if row: + try: + refreshed = _refresh_call_data_from_provider( + db=db, + organization_id=organization_id, + call_recording=row, + ) + row.call_data = refreshed + row.status = CallRecordingStatus.UPDATED + _maybe_archive_observability_recording(db, row, refreshed) + _maybe_persist_provider_trace( + db, + row, + call_data=refreshed, + provider_platform="vapi", + ) + db.commit() + db.refresh(row) + agent = db.query(Agent).filter(Agent.id == row.agent_id).first() if row.agent_id else None + response = _serialize_call_recording(row, include_data=True, agent=agent) + except Exception as exc: + logger.warning(f"[VapiWebhook] Pull fallback refresh failed for {call_short_id}: {exc}") + + return response @router.post("/calls/webhook/{api_key}", status_code=status.HTTP_201_CREATED) @@ -393,6 +1235,218 @@ async def ingest_call_via_webhook_url( return _process_flat_payload(body, organization_id, db) +@router.post("/observe", status_code=status.HTTP_201_CREATED) +async def observe_call( + body: Dict[str, Any], + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Header-auth flat call ingest endpoint for SDK observation.""" + del api_key + return _process_flat_payload_for_workspace(body, organization_id, workspace_id, db) + + +@router.post("/live/events", status_code=status.HTTP_202_ACCEPTED) +async def ingest_live_event( + body: Dict[str, Any], + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Ingest an incremental live event with idempotent merge semantics.""" + del api_key + _ensure_live_ingest_enabled() + envelope = LiveEventEnvelope.model_validate(body) + + try: + event_ts_dt = parse_live_event_ts(envelope.event_ts) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + _validate_live_event_ts_drift(event_ts_dt) + + provider_platform = envelope.platform.strip().lower() + provider_call_id = envelope.call_id.strip() + trace_id = envelope.trace_id or _issue_efficientai_trace_id() + now = datetime.now(UTC) + + _cleanup_expired_live_dedup_rows(db, organization_id) + duplicate_row = ( + db.query(ObservabilityLiveEventDedup) + .filter( + ObservabilityLiveEventDedup.organization_id == organization_id, + ObservabilityLiveEventDedup.event_id == envelope.event_id, + ObservabilityLiveEventDedup.expires_at >= now, + ) + .first() + ) + if duplicate_row: + duplicate_trace_id = envelope.trace_id + if not duplicate_trace_id and duplicate_row.call_short_id: + existing_call = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == duplicate_row.call_short_id, + CallRecording.organization_id == organization_id, + ) + .first() + ) + if existing_call: + existing_payload = existing_call.call_data if isinstance(existing_call.call_data, dict) else {} + duplicate_trace_id = existing_call.trace_id or existing_payload.get("trace_id") + return { + "accepted": True, + "duplicate": True, + "event_id": envelope.event_id, + "trace_id": duplicate_trace_id, + "call_short_id": duplicate_row.call_short_id, + } + + live_event = envelope.model_dump() + live_event["platform"] = provider_platform + live_event["event_ts"] = event_ts_dt.isoformat() + live_event["trace_id"] = trace_id + explicit_agent_id = _resolve_agent_id_from_ref( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_ref_raw=envelope.agent_ref, + ) + + try: + call_recording, action = upsert_live_event_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + provider_platform=provider_platform, + provider_call_id=provider_call_id, + live_event=live_event, + max_out_of_order_seq=settings.OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ, + agent_ref_raw=envelope.agent_ref, + explicit_agent_id=explicit_agent_id, + source=CallRecordingSource.WEBHOOK, + persist=False, + ) + except StaleLiveEventError as exc: + db.rollback() + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + + dedup_ttl = max(60, int(settings.OBSERVABILITY_LIVE_EVENT_IDEMPOTENCY_TTL_SECONDS)) + db.add( + ObservabilityLiveEventDedup( + organization_id=organization_id, + workspace_id=workspace_id, + event_id=envelope.event_id, + provider_platform=provider_platform, + provider_call_id=provider_call_id, + call_short_id=call_recording.call_short_id, + seq=envelope.seq, + event_ts=event_ts_dt, + expires_at=now + timedelta(seconds=dedup_ttl), + ) + ) + + latency_samples_written = 0 + if settings.OBSERVABILITY_LIVE_AGGREGATES_ENABLED: + latency_samples_written = record_live_latency_samples( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + provider_platform=provider_platform, + provider_call_id=provider_call_id, + event_payload=envelope.payload, + event_ts=event_ts_dt, + ) + + slo_breach = None + if settings.OBSERVABILITY_LIVE_AGGREGATES_ENABLED and settings.OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED: + slo_breach = evaluate_llm_p90_slo( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + provider_platform=provider_platform, + ) + + try: + db.commit() + except IntegrityError: + db.rollback() + return { + "accepted": True, + "duplicate": True, + "event_id": envelope.event_id, + "trace_id": trace_id, + "call_short_id": call_recording.call_short_id, + } + + if action == "created": + record_observability_call_ingested( + organization_id, + call_recording.call_short_id, + workspace_id=workspace_id, + provider=provider_platform, + ) + + if call_recording.call_event in {"call_ended", "call_failed"}: + db.refresh(call_recording) + terminal_call_data = ( + call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + ) + trace_persisted = _maybe_persist_provider_trace( + db, + call_recording, + call_data=terminal_call_data, + provider_platform=provider_platform, + raw_payload=( + envelope.payload.get("otlp_traces") + if isinstance(envelope.payload.get("otlp_traces"), dict) + else None + ), + ) + _maybe_archive_observability_recording(db, call_recording) + if trace_persisted: + db.commit() + db.refresh(call_recording) + + evaluator_hook_queued = False + if ( + settings.OBSERVABILITY_LIVE_SLO_AUTOMATION_ENABLED + and slo_breach is not None + and call_recording.call_event in {"call_ended", "call_failed"} + ): + try: + _maybe_auto_evaluate_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + agent=None, + ) + slo_breach.evaluator_queued = True + db.commit() + evaluator_hook_queued = True + except Exception: + db.rollback() + + return { + "accepted": True, + "duplicate": False, + "action": action, + "event_id": envelope.event_id, + "call_short_id": call_recording.call_short_id, + "provider_platform": provider_platform, + "provider_call_id": provider_call_id, + "trace_id": trace_id, + "latency_samples_written": latency_samples_written, + "slo_breach_detected": bool(slo_breach), + "evaluator_hook_queued": evaluator_hook_queued, + } + + @router.get("/calls", response_model=List[Dict[str, Any]]) async def list_calls( @@ -411,7 +1465,7 @@ async def list_calls( .filter( CallRecording.organization_id == organization_id, CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .order_by(CallRecording.created_at.desc()) .offset(skip) @@ -432,6 +1486,241 @@ async def list_calls( ] +@router.get("/calls/summary", response_model=Dict[str, Any]) +async def calls_summary( + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + del api_key + call_recordings = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .all() + ) + total_calls = len(call_recordings) + trace_linked_calls = 0 + trace_available_calls = 0 + evaluated_calls = 0 + ended_calls = 0 + failed_calls = 0 + started_calls = 0 + other_calls = 0 + for recording in call_recordings: + call_data = recording.call_data if isinstance(recording.call_data, dict) else {} + trace_id = recording.trace_id or call_data.get("trace_id") + if trace_id: + trace_linked_calls += 1 + provider_trace = call_data.get("provider_trace") + has_stored_provider_trace = isinstance(provider_trace, dict) and bool( + provider_trace.get("normalized_trace") or provider_trace.get("otlp_traces") or provider_trace.get("trace_s3_key") + ) + has_retell_signal = bool( + ( + isinstance(call_data.get("transcript_object"), list) + and len(call_data.get("transcript_object")) > 0 + ) + or ( + isinstance(call_data.get("transcript"), str) + and call_data.get("transcript", "").strip() + ) + or ( + isinstance(call_data.get("latency"), dict) + and len(call_data.get("latency")) > 0 + ) + ) + has_vapi_signal = bool( + ( + isinstance(call_data.get("messages"), list) + and len(call_data.get("messages")) > 0 + ) + or ( + isinstance(call_data.get("artifact"), dict) + and isinstance(call_data.get("artifact", {}).get("messages"), list) + and len(call_data.get("artifact", {}).get("messages")) > 0 + ) + or ( + isinstance(call_data.get("analysis"), dict) + and len(call_data.get("analysis")) > 0 + ) + or ( + isinstance(call_data.get("artifact"), dict) + and isinstance(call_data.get("artifact", {}).get("performanceMetrics"), dict) + and len(call_data.get("artifact", {}).get("performanceMetrics")) > 0 + ) + ) + platform = (recording.provider_platform or "").strip().lower() + has_synthetic_candidate = ( + platform == "retell" and has_retell_signal + ) or ( + platform == "vapi" and has_vapi_signal + ) + if trace_id or has_stored_provider_trace or has_synthetic_candidate: + trace_available_calls += 1 + if recording.evaluator_result_id: + evaluated_calls += 1 + event = (recording.call_event or "").strip().lower() + if event == "call_ended": + ended_calls += 1 + elif event in {"call_failed", "failed"}: + failed_calls += 1 + elif event == "call_started": + started_calls += 1 + elif event: + other_calls += 1 + + duration_seconds = [ + value for value in (_extract_duration_seconds(recording) for recording in call_recordings) if value + ] + total_minutes = sum(duration_seconds) / 60.0 if duration_seconds else 0.0 + avg_duration_ms = (sum(duration_seconds) / len(duration_seconds) * 1000.0) if duration_seconds else 0.0 + trace_link_rate = (trace_linked_calls / total_calls * 100.0) if total_calls else 0.0 + trace_available_rate = (trace_available_calls / total_calls * 100.0) if total_calls else 0.0 + evaluated_rate = (evaluated_calls / total_calls * 100.0) if total_calls else 0.0 + return { + "total_calls": total_calls, + "total_minutes": round(total_minutes, 2), + "avg_duration_ms": round(avg_duration_ms, 2), + "avg_latency_ms": round(avg_duration_ms, 2), + "trace_linked_calls": trace_linked_calls, + "trace_link_rate_pct": round(trace_link_rate, 2), + "trace_available_calls": trace_available_calls, + "trace_available_rate_pct": round(trace_available_rate, 2), + "evaluated_calls": evaluated_calls, + "evaluated_rate_pct": round(evaluated_rate, 2), + "event_breakdown": { + "call_ended": ended_calls, + "call_failed": failed_calls, + "call_started": started_calls, + "other": other_calls, + }, + "live_feature_flags": { + "live_ingest_enabled": settings.OBSERVABILITY_LIVE_INGEST_ENABLED, + "live_aggregates_enabled": settings.OBSERVABILITY_LIVE_AGGREGATES_ENABLED, + "live_dashboard_enabled": settings.OBSERVABILITY_LIVE_DASHBOARD_ENABLED, + }, + } + + +@router.get("/live/metrics/latency", response_model=Dict[str, Any]) +async def get_live_latency_metrics( + platform: Optional[str] = None, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Return rolling live latency percentiles across all agents.""" + del api_key + _ensure_live_aggregates_enabled() + normalized_platform = platform.strip().lower() if isinstance(platform, str) and platform.strip() else None + return { + "scope": "workspace", + "platform": normalized_platform, + "windows": { + "60s": query_live_latency_metrics( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + window_seconds=60, + provider_platform=normalized_platform, + ), + "300s": query_live_latency_metrics( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + window_seconds=300, + provider_platform=normalized_platform, + ), + }, + } + + +@router.get("/live/agents/{agent_id}/latency", response_model=Dict[str, Any]) +async def get_live_agent_latency_metrics( + agent_id: str, + platform: Optional[str] = None, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Return rolling live latency percentiles for a specific agent.""" + del api_key + _ensure_live_aggregates_enabled() + try: + agent_uuid = UUID(agent_id) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid agent_id") from exc + normalized_platform = platform.strip().lower() if isinstance(platform, str) and platform.strip() else None + return { + "scope": "agent", + "agent_id": str(agent_uuid), + "platform": normalized_platform, + "windows": { + "60s": query_live_latency_metrics( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + window_seconds=60, + agent_id=agent_uuid, + provider_platform=normalized_platform, + ), + "300s": query_live_latency_metrics( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + window_seconds=300, + agent_id=agent_uuid, + provider_platform=normalized_platform, + ), + }, + } + + +@router.get("/live/slo/breaches", response_model=List[Dict[str, Any]]) +async def list_live_slo_breaches( + limit: int = 50, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> List[Dict[str, Any]]: + """List recent live SLO breaches for operational triage.""" + del api_key + rows = ( + db.query(ObservabilityLiveSloBreach) + .filter( + ObservabilityLiveSloBreach.organization_id == organization_id, + ObservabilityLiveSloBreach.workspace_id == workspace_id, + ) + .order_by(ObservabilityLiveSloBreach.created_at.desc()) + .limit(max(1, min(limit, 200))) + .all() + ) + return [ + { + "id": str(row.id), + "call_short_id": row.call_short_id, + "provider_platform": row.provider_platform, + "agent_id": str(row.agent_id) if row.agent_id else None, + "metric_name": row.metric_name, + "window_seconds": row.window_seconds, + "p90_ms": row.p90_ms, + "threshold_ms": row.threshold_ms, + "sample_count": row.sample_count, + "evaluator_queued": row.evaluator_queued, + "created_at": row.created_at.isoformat() if row.created_at else None, + } + for row in rows + ] + + @router.get("/calls/{call_short_id}", response_model=Dict[str, Any]) async def get_call( call_short_id: str, @@ -449,7 +1738,7 @@ async def get_call( CallRecording.call_short_id == call_short_id, CallRecording.organization_id == organization_id, CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .first() ) @@ -460,14 +1749,26 @@ async def get_call( detail="Call not found", ) - from app.services.live_entity_storage import hydrate_call_recordings - - hydrate_call_recordings([call_recording]) + call_recording = _prepare_observability_call_recording(db, organization_id, call_recording) + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + provider_platform = resolve_observability_provider_platform(call_recording, call_data, db=db) + if provider_platform == "elevenlabs" and not load_provider_trace(call_data): + try: + await _query_elevenlabs_trace_for_call( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + call_data=call_data, + ) + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else call_data + except HTTPException: + pass # region agent log from app.utils.debug_agent_log import agent_debug_log - call_data_dbg = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + call_data_dbg = call_data agent_debug_log( "observability.py:get_call", "call detail fetched", @@ -487,7 +1788,106 @@ async def get_call( if call_recording.agent_id: agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - return _serialize_call_recording(call_recording, include_data=True, agent=agent) + return _serialize_call_recording(call_recording, include_data=True, agent=agent) + + +@router.post("/calls/{call_short_id}/refresh", response_model=Dict[str, Any]) +async def refresh_observability_call( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Refresh an observability call by pulling latest metrics from the provider.""" + del api_key + + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") + + try: + refreshed_call_data = _refresh_call_data_from_provider( + db=db, + organization_id=organization_id, + call_recording=call_recording, + ) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to refresh provider call metrics: {exc}", + ) from exc + + call_recording.call_data = refreshed_call_data + call_recording.status = CallRecordingStatus.UPDATED + _maybe_archive_observability_recording(db, call_recording, refreshed_call_data) + provider_platform = (call_recording.provider_platform or "").strip().lower() + if provider_platform == "elevenlabs" and call_recording.provider_call_id: + integration = _resolve_elevenlabs_integration_for_call( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + call_data=refreshed_call_data, + ) + if integration is not None: + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + provider_class = get_voice_provider("elevenlabs") + provider = provider_class(api_key=decrypted_api_key) + provider_trace_payload = provider.retrieve_provider_trace(str(call_recording.provider_call_id)) + otlp_payload = provider_trace_payload.get("otlp_traces") + if isinstance(otlp_payload, dict): + normalized_trace = normalize_elevenlabs_otlp( + otlp_payload, + conversation_id=str(call_recording.provider_call_id), + fallback_trace_id=extract_trace_id(otlp_payload), + ) + transcript = provider_trace_payload.get("transcript") + if isinstance(transcript, list): + normalized_trace = enrich_with_turn_metrics(normalized_trace, transcript) + call_recording.call_data = persist_provider_trace( + call_data=call_recording.call_data if isinstance(call_recording.call_data, dict) else {}, + provider_platform="elevenlabs", + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=normalized_trace, + source="elevenlabs_refresh_fetch", + raw_payload=otlp_payload, + ) + call_recording.trace_id = normalized_trace.get("trace_id") or call_recording.trace_id + except Exception as exc: + logger.warning( + "ElevenLabs trace refresh fetch failed for call_short_id={}: {}", + call_recording.call_short_id, + exc, + ) + else: + _maybe_persist_provider_trace( + db, + call_recording, + call_data=refreshed_call_data, + provider_platform=provider_platform, + ) + db.commit() + db.refresh(call_recording) + + refreshed_agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + return _serialize_call_recording(call_recording, include_data=True, agent=refreshed_agent) @router.get("/calls/{call_short_id}/live-events") @@ -512,7 +1912,7 @@ async def stream_call_live_events( CallRecording.call_short_id == call_short_id, CallRecording.organization_id == organization_id, CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .first() ) @@ -546,7 +1946,7 @@ async def event_generator(): CallRecording.call_short_id == call_short_id, CallRecording.organization_id == bound_org_id, CallRecording.workspace_id == bound_workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .first() ) @@ -575,6 +1975,71 @@ async def event_generator(): ) +@router.get("/calls/{call_short_id}/live-audio") +async def stream_observability_live_call_audio( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Return partial merged mono WAV for an in-progress telephony call.""" + from io import BytesIO + + from fastapi.responses import StreamingResponse + + from app.services.telephony.call_recording_lifecycle import LIVE_CALL_EVENTS + from app.services.telephony.live_recording import merge_live_tracks_mono + + del api_key + + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") + + call_event = (call_recording.call_event or "").lower() + if call_event not in LIVE_CALL_EVENTS: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Live audio is only available while the call is in progress", + ) + + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + user_path = call_data.get("live_user_audio_path") + bot_path = call_data.get("live_bot_audio_path") + if not user_path or not bot_path: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Live recording paths are not registered for this call yet", + ) + + wav_bytes, duration_sec, _sample_rate = merge_live_tracks_mono(str(user_path), str(bot_path)) + if not wav_bytes: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No live audio captured yet", + ) + + return StreamingResponse( + BytesIO(wav_bytes), + media_type="audio/wav", + headers={ + "Content-Disposition": f'inline; filename="call_{call_short_id}_live.wav"', + "Cache-Control": "no-store", + "X-Audio-Duration-Sec": f"{duration_sec:.3f}", + }, + ) + + @router.get("/calls/{call_short_id}/audio") async def stream_observability_call_audio( call_short_id: str, @@ -583,10 +2048,10 @@ async def stream_observability_call_audio( api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): - """Stream call recording audio for observability calls (S3 or provider URL).""" + """Stream call recording audio for observability calls (S3-backed).""" from io import BytesIO - from fastapi.responses import RedirectResponse, StreamingResponse + from fastapi.responses import StreamingResponse del api_key @@ -596,17 +2061,38 @@ async def stream_observability_call_audio( CallRecording.call_short_id == call_short_id, CallRecording.organization_id == organization_id, CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .first() ) if not call_recording: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") + call_recording = _prepare_observability_call_recording( + db, organization_id, call_recording, enrich=False + ) call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} - recording_url = call_data.get("recording_url") - if recording_url: - return RedirectResponse(recording_url) + provider_platform = ( + call_recording.provider_platform + or call_data.get("provider_platform") + or "" + ).strip().lower() + + if provider_platform == IntegrationPlatform.ELEVENLABS.value: + from app.services.observability.provider_audio_proxy import ( + stream_elevenlabs_audio_proxy, + ) + + return stream_elevenlabs_audio_proxy( + db=db, + organization_id=organization_id, + call_recording=call_recording, + call_data=call_data, + filename_prefix="call", + ) + + if not call_data.get("recording_s3_key"): + call_data = _maybe_archive_observability_recording(db, call_recording, call_data) s3_key = call_data.get("recording_s3_key") if not s3_key: @@ -621,7 +2107,6 @@ async def stream_observability_call_audio( { "call_short_id": call_short_id, "has_recording_s3_key": True, - "has_recording_url": bool(recording_url), }, "H6", run_id="post-fix", @@ -668,7 +2153,7 @@ async def delete_call( CallRecording.call_short_id == call_short_id, CallRecording.organization_id == organization_id, CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), ) .first() ) @@ -761,36 +2246,20 @@ def _messages_to_speaker_segments(messages: List[Dict[str, Any]]) -> List[Dict[s return segments -@router.post("/calls/{call_short_id}/evaluate", status_code=status.HTTP_201_CREATED) -async def evaluate_call( - call_short_id: str, - payload: EvaluateCallPayload, - background_tasks: BackgroundTasks, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -) -> Dict[str, Any]: - """Trigger an LLM evaluation on an ingested call in the active workspace. - - Both the call recording and the evaluator must already live in the same - workspace as the caller. - """ - del api_key +def _extract_duration_seconds(call_recording: CallRecording) -> Optional[float]: + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + return _resolve_call_duration_seconds(call_data) - call_recording = ( - db.query(CallRecording) - .filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.WEBHOOK, - ) - .first() - ) - if not call_recording: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") +def _queue_call_evaluation( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + evaluator_id: str, + background_tasks: Optional[BackgroundTasks] = None, +) -> Dict[str, Any]: call_data = dict(call_recording.call_data or {}) call_data.setdefault("call_short_id", call_recording.call_short_id) messages = call_data.get("messages") @@ -808,7 +2277,7 @@ async def evaluate_call( ) try: - evaluator_uuid = UUID(payload.evaluator_id) + evaluator_uuid = UUID(evaluator_id) except ValueError: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid evaluator_id") @@ -838,7 +2307,10 @@ async def evaluate_call( result_id = candidate break if not result_id: - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to generate unique result ID") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to generate unique result ID", + ) transcript = _messages_to_transcript(messages) duration_seconds = _resolve_call_duration_seconds(call_data) @@ -877,12 +2349,22 @@ async def evaluate_call( except Exception: pass - background_tasks.add_task( - record_observability_call_evaluated, - organization_id, - call_short_id, - workspace_id=workspace_id, - ) + if background_tasks is not None: + background_tasks.add_task( + record_observability_call_evaluated, + organization_id, + call_recording.call_short_id, + workspace_id=workspace_id, + ) + else: + try: + record_observability_call_evaluated( + organization_id, + call_recording.call_short_id, + workspace_id=workspace_id, + ) + except Exception: + pass return { "evaluator_result_id": str(evaluator_result.id), @@ -892,6 +2374,500 @@ async def evaluate_call( } +def _maybe_auto_evaluate_call_recording( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + agent: Optional[Agent] = None, +) -> None: + if call_recording.call_event != "call_ended": + return + if call_recording.evaluator_result_id: + return + if not call_recording.agent_id: + return + + agent_obj = agent + if agent_obj is None: + agent_obj = ( + db.query(Agent) + .filter( + Agent.id == call_recording.agent_id, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ) + .first() + ) + if not agent_obj or not agent_obj.observability_auto_evaluator_id: + return + + try: + _queue_call_evaluation( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + evaluator_id=str(agent_obj.observability_auto_evaluator_id), + background_tasks=None, + ) + except Exception: + # Best-effort auto-eval should not block webhook ingest. + return + + +def _normalize_span_attributes(raw_attrs: Any) -> Dict[str, Any]: + if isinstance(raw_attrs, dict): + return raw_attrs + if isinstance(raw_attrs, list): + result: Dict[str, Any] = {} + for item in raw_attrs: + if not isinstance(item, dict): + continue + key = item.get("key") + if not key: + continue + value = item.get("value") + if isinstance(value, dict) and "stringValue" in value: + result[key] = value.get("stringValue") + elif isinstance(value, dict) and "intValue" in value: + result[key] = value.get("intValue") + elif isinstance(value, dict) and "doubleValue" in value: + result[key] = value.get("doubleValue") + elif isinstance(value, dict) and "boolValue" in value: + result[key] = value.get("boolValue") + else: + result[key] = value + return result + return {} + + +def _to_epoch_ms(value: Any) -> Optional[float]: + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if stripped.isdigit() or ( + stripped.replace(".", "", 1).isdigit() and stripped.count(".") <= 1 + ): + return _to_epoch_ms(float(stripped)) + try: + parsed = datetime.fromisoformat(stripped.replace("Z", "+00:00")) + return parsed.timestamp() * 1000.0 + except Exception: + return None + if isinstance(value, (int, float)): + numeric = float(value) + if numeric > 1e16: # ns + return numeric / 1_000_000.0 + if numeric > 1e13: # us + return numeric / 1000.0 + if numeric > 1e10: # ms + return numeric + if numeric > 0: + return numeric * 1000.0 + return None + return None + + +def _collect_spans(raw: Any, out: List[Dict[str, Any]]) -> None: + if isinstance(raw, dict): + if any(k in raw for k in ("spanId", "span_id", "id")) and any( + k in raw for k in ("name", "operationName") + ): + out.append(raw) + for value in raw.values(): + _collect_spans(value, out) + elif isinstance(raw, list): + for item in raw: + _collect_spans(item, out) + + +def _normalize_trace_payload(trace_id: str, payload: Dict[str, Any]) -> Dict[str, Any]: + raw_spans: List[Dict[str, Any]] = [] + _collect_spans(payload, raw_spans) + + spans: List[Dict[str, Any]] = [] + root_span_id: Optional[str] = None + for raw in raw_spans: + span_id = raw.get("span_id") or raw.get("spanId") or raw.get("id") + parent_span_id = raw.get("parent_span_id") or raw.get("parentSpanId") + if not parent_span_id: + references = raw.get("references") + if isinstance(references, list): + for ref in references: + if isinstance(ref, dict) and ( + ref.get("refType") == "CHILD_OF" or ref.get("type") == "CHILD_OF" + ): + parent_span_id = ref.get("spanID") or ref.get("spanId") + break + + start_ms = _to_epoch_ms( + raw.get("start_time") + or raw.get("startTime") + or raw.get("startTimeUnixNano") + or raw.get("start_time_unix_nano") + ) + end_ms = _to_epoch_ms( + raw.get("end_time") + or raw.get("endTime") + or raw.get("endTimeUnixNano") + or raw.get("end_time_unix_nano") + ) + duration_ms = raw.get("duration_ms") + if duration_ms is None: + duration = raw.get("duration") + if isinstance(duration, (int, float)): + duration_ms = duration / 1_000_000.0 if duration > 1e10 else float(duration) + elif start_ms is not None and end_ms is not None: + duration_ms = max(end_ms - start_ms, 0.0) + + attrs = _normalize_span_attributes(raw.get("attributes") or raw.get("tags")) + status_obj = raw.get("status") + if isinstance(status_obj, dict): + status_value = status_obj.get("code") or status_obj.get("status_code") + else: + status_value = status_obj + + normalized = { + "span_id": span_id, + "parent_span_id": parent_span_id, + "name": raw.get("name") or raw.get("operationName") or "unknown", + "start_time": start_ms, + "end_time": end_ms, + "duration_ms": duration_ms, + "attributes": attrs, + "status": status_value, + } + spans.append(normalized) + if not parent_span_id and span_id and root_span_id is None: + root_span_id = span_id + + if root_span_id is None and spans: + root_span_id = spans[0].get("span_id") + + return { + "trace_id": trace_id, + "root_span_id": root_span_id, + "spans": spans, + } + + +async def _query_trace_cloud(trace_id: str, api_key: str) -> Dict[str, Any]: + url = settings.EFFICIENT_AI_TRACE_QUERY_URL + headers: Dict[str, str] = {} + if api_key: + headers["X-API-Key"] = api_key + if settings.EFFICIENT_AI_API_KEY: + headers["x-efficient-ai-api-key"] = settings.EFFICIENT_AI_API_KEY + + request_url = url.format(trace_id=trace_id) if "{trace_id}" in url else url + params = {} if "{trace_id}" in url else {"trace_id": trace_id} + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(request_url, params=params, headers=headers) + response.raise_for_status() + payload = response.json() + return _normalize_trace_payload(trace_id, payload) + + +async def _query_trace_tempo(trace_id: str) -> Dict[str, Any]: + base = settings.TEMPO_QUERY_URL.rstrip("/") + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(f"{base}/api/traces/{trace_id}") + response.raise_for_status() + payload = response.json() + return _normalize_trace_payload(trace_id, payload) + + +async def _query_elevenlabs_trace_for_call( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + call_data: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + provider_call_id = call_recording.provider_call_id or call_data.get("conversation_id") + if not provider_call_id: + return None + + stored = _build_elevenlabs_trace_from_stored_data(call_data, str(provider_call_id)) + if stored: + return stored + + integration = _resolve_elevenlabs_integration_for_call( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + call_data=call_data, + ) + if integration is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "No ElevenLabs integration could be resolved for this call. " + "Link the provider agent to an integration first." + ), + ) + + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + platform_value = integration.platform.value if hasattr(integration.platform, "value") else str(integration.platform).lower() + provider_class = get_voice_provider(platform_value) + provider = provider_class(api_key=decrypted_api_key) + payload = provider.retrieve_conversation_trace(str(provider_call_id)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Failed to query ElevenLabs trace API: {exc}", + ) from exc + + status_value = str(payload.get("status") or "").lower().strip() + if status_value and status_value not in {"done", "failed"}: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Trace is not ready yet. ElevenLabs exposes OTLP after conversation completion.", + ) + + otlp_payload = payload.get("otlp_traces") + if not isinstance(otlp_payload, dict): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No OpenTelemetry trace found in ElevenLabs conversation payload.", + ) + + normalized = normalize_elevenlabs_otlp( + otlp_payload, + conversation_id=str(provider_call_id), + fallback_trace_id=extract_trace_id(otlp_payload), + ) + transcript = payload.get("transcript") + if isinstance(transcript, list): + normalized = enrich_with_turn_metrics(normalized, transcript) + persisted = persist_provider_trace( + call_data=call_data, + provider_platform="elevenlabs", + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=normalized, + source="elevenlabs_api_fetch", + raw_payload=otlp_payload, + ) + call_recording.call_data = persisted + if normalized.get("trace_id"): + call_recording.trace_id = str(normalized.get("trace_id")) + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + db.refresh(call_recording) + return normalized + + +@router.get("/calls/{call_short_id}/trace", response_model=Dict[str, Any]) +async def get_call_trace( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") + + call_recording = _prepare_observability_call_recording(db, organization_id, call_recording) + + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + provider_platform = resolve_observability_provider_platform(call_recording, call_data, db=db) + backend = (settings.TRACING_QUERY_BACKEND or "cloud").strip().lower() + stored_provider_trace = load_provider_trace(call_data) + if stored_provider_trace: + return stored_provider_trace + + elevenlabs_fallback_to_tempo = False + if provider_platform == "elevenlabs": + try: + elevenlabs_trace = await _query_elevenlabs_trace_for_call( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + call_data=call_data, + ) + if elevenlabs_trace: + return elevenlabs_trace + except HTTPException as elevenlabs_exc: + trace_id = call_recording.trace_id or call_data.get("trace_id") + if backend == "tempo" and trace_id: + elevenlabs_fallback_to_tempo = True + else: + raise elevenlabs_exc + + if provider_platform == "vapi": + synthetic_trace = build_vapi_synthetic_trace( + call_data, + provider_call_id=str(call_recording.provider_call_id or call_data.get("id") or call_short_id), + ) + if synthetic_trace: + call_recording.call_data = persist_provider_trace( + call_data=call_data, + provider_platform="vapi", + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=synthetic_trace, + source="vapi_synthetic", + ) + call_recording.trace_id = synthetic_trace.get("trace_id") or call_recording.trace_id + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + return synthetic_trace + + if provider_platform == "retell": + synthetic_trace = build_retell_synthetic_trace( + call_data, + provider_call_id=str(call_recording.provider_call_id or call_data.get("call_id") or call_short_id), + ) + if synthetic_trace: + call_recording.call_data = persist_provider_trace( + call_data=call_data, + provider_platform="retell", + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=synthetic_trace, + source="retell_synthetic", + ) + call_recording.trace_id = synthetic_trace.get("trace_id") or call_recording.trace_id + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + return synthetic_trace + + if _should_build_live_synthetic_trace(provider_platform, call_data): + synthetic_trace = build_live_synthetic_trace( + call_data, + provider_call_id=str(call_recording.provider_call_id or call_data.get("id") or call_short_id), + provider_platform=provider_platform or "external", + trace_id=str(call_recording.trace_id or call_data.get("trace_id") or "") or None, + ) + if synthetic_trace: + call_recording.call_data = persist_provider_trace( + call_data=call_data, + provider_platform=provider_platform or "external", + organization_id=call_recording.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=synthetic_trace, + source=synthetic_trace.get("trace_source") or "live_synthetic", + ) + call_recording.trace_id = synthetic_trace.get("trace_id") or call_recording.trace_id + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + return synthetic_trace + + if provider_platform in {"retell", "vapi"}: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + f"No synthetic {provider_platform} trace could be built from stored call data. " + "Try Refresh on the call to pull the latest provider report." + ), + ) + + if provider_platform == "elevenlabs" and not elevenlabs_fallback_to_tempo: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No provider trace linked to this call yet", + ) + + trace_id = call_recording.trace_id or call_data.get("trace_id") + if not trace_id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No trace linked to this call") + + try: + if backend == "tempo": + trace_payload = await _query_trace_tempo(trace_id) + if provider_platform == "elevenlabs": + trace_payload = {**trace_payload, "trace_source": "efficientai"} + return trace_payload + return await _query_trace_cloud(trace_id, api_key) + except httpx.HTTPStatusError as exc: + upstream_status = exc.response.status_code + if upstream_status == status.HTTP_404_NOT_FOUND: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "Trace not found in the trace store. Spans may have expired, been purged, " + "or never exported (check OTLP export and Tempo retention)." + ), + ) from exc + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=f"Trace backend returned {upstream_status}", + ) from exc + except httpx.HTTPError as exc: + backend_label = "Tempo" if backend == "tempo" else "cloud trace API" + backend_url = ( + settings.TEMPO_QUERY_URL.rstrip("/") + if backend == "tempo" + else settings.EFFICIENT_AI_TRACE_QUERY_URL + ) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=( + f"Could not reach {backend_label} at {backend_url}: {exc}. " + "For local dev with query_backend=tempo, ensure Tempo is running on port 3200." + ), + ) from exc + + +@router.post("/calls/{call_short_id}/evaluate", status_code=status.HTTP_201_CREATED) +async def evaluate_call( + call_short_id: str, + payload: EvaluateCallPayload, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """Trigger an LLM evaluation on an ingested call in the active workspace. + + Both the call recording and the evaluator must already live in the same + workspace as the caller. + """ + del api_key + + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source.in_(OBSERVABILITY_CALL_SOURCES), + ) + .first() + ) + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call not found") + + return _queue_call_evaluation( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + call_recording=call_recording, + evaluator_id=payload.evaluator_id, + background_tasks=background_tasks, + ) + + from app.core.auth.capabilities import REPORTS_GENERATE, REPORTS_VIEW from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities diff --git a/app/api/v1/routes/personas.py b/app/api/v1/routes/personas.py index de07c715..6476aa44 100644 --- a/app/api/v1/routes/personas.py +++ b/app/api/v1/routes/personas.py @@ -644,7 +644,7 @@ async def update_persona( incoming = update_data.get("tts_provider") if incoming and str(incoming).strip().lower() != str(db_persona.tts_provider).strip().lower(): raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="tts_provider cannot be changed after persona creation. Change the voice instead.", ) update_data.pop("tts_provider", None) @@ -653,7 +653,7 @@ async def update_persona( try: validate_persona_tts_config(provider, update_data["tts_config"]) except ValueError as exc: - raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)) update_data["tts_config"] = _normalized_persona_tts_config(provider, update_data["tts_config"]) for field, value in update_data.items(): setattr(db_persona, field, value) diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 31b1a3c2..c44b45d4 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -1239,8 +1239,6 @@ async def stream_call_audio( Proxy endpoint to stream playground call recording audio in the active workspace. Required for providers like ElevenLabs whose audio URLs need auth headers. """ - import requests as http_requests - call_recording = db.query(CallRecording).filter( CallRecording.call_short_id == call_short_id, CallRecording.organization_id == organization_id, @@ -1278,43 +1276,16 @@ async def stream_call_audio( # ElevenLabs requires API key header – proxy the stream if platform == "elevenlabs": - audio_url = recording_urls.get("conversation_audio") - if not audio_url: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording URL available") - - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - if not agent or not agent.voice_ai_integration_id: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") - - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - ).first() - if not integration: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") - - decrypted_key = decrypt_api_key(integration.api_key) - - upstream = http_requests.get( - audio_url, - headers={"xi-api-key": decrypted_key}, - stream=True, - timeout=60, + from app.services.observability.provider_audio_proxy import ( + stream_elevenlabs_audio_proxy, ) - if upstream.status_code != 200: - raise HTTPException( - status_code=upstream.status_code, - detail=f"ElevenLabs audio fetch failed ({upstream.status_code})", - ) - content_type = upstream.headers.get("content-type", "audio/mpeg") - - return StreamingResponse( - upstream.iter_content(chunk_size=8192), - media_type=content_type, - headers={ - "Content-Disposition": f'inline; filename="call_{call_short_id}.mp3"', - }, + return stream_elevenlabs_audio_proxy( + db=db, + organization_id=organization_id, + call_recording=call_recording, + call_data=call_data, + filename_prefix="call", ) # Custom WebSocket sessions store audio in S3 diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index e6d553b5..139d53cb 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import Session from uuid import UUID from typing import Dict, Any, Optional, List +from datetime import UTC, datetime from loguru import logger from app.database import get_db @@ -139,39 +140,32 @@ async def websocket_endpoint( use_voice_bundle_pipeline = bool(voice_bundle and voice_bundle.bundle_type == "stt_llm_tts") - def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: - """Resolve API key from AIProvider (preferred) or Integration for given provider.""" - from sqlalchemy import func - # 1) AIProvider (handle both string and enum comparisons) - provider_value = provider.value if hasattr(provider, 'value') else provider - - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == provider_value, - AIProvider.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not ai_provider_rec: - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.is_active == True, - ).first() + def resolve_api_key_for_provider( + provider: ModelProvider, + credential_id: Optional[UUID] = None, + ) -> str | None: + """Resolve API key using shared credential resolver (pinned id -> default -> latest).""" + import os + + from app.services.credentials import resolve_ai_provider, resolve_integration + + provider_value = provider.value if hasattr(provider, "value") else provider + + ai_provider_rec = resolve_ai_provider( + provider, db, organization_id, credential_id=credential_id + ) if ai_provider_rec: try: key = decrypt_api_key(ai_provider_rec.api_key) logger.debug( - f"[resolve_api_key] Found AIProvider key for '{provider_value}': " + f"[resolve_api_key] Found AIProvider key for '{provider_value}' " + f"(credential_id={credential_id or ai_provider_rec.id}): " f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" ) return key except Exception as e: logger.error(f"Failed to decrypt AIProvider key for {provider}: {e}", exc_info=True) - else: - logger.debug(f"[resolve_api_key] No AIProvider record found for '{provider_value}'") - # 2) Integration mapping (only for platforms that exist in IntegrationPlatform) platform_map = { ModelProvider.DEEPGRAM: IntegrationPlatform.DEEPGRAM, ModelProvider.CARTESIA: IntegrationPlatform.CARTESIA, @@ -183,38 +177,44 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: } plat = platform_map.get(provider) if plat: - # Handle both string and enum comparisons for platform - plat_value = plat.value if hasattr(plat, 'value') else plat - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - Integration.platform == plat_value, - Integration.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not integ: - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - func.lower(Integration.platform) == plat_value.lower(), - Integration.is_active == True, - ).first() - + integ = resolve_integration(plat, db, organization_id, credential_id=credential_id) if integ: try: key = decrypt_api_key(integ.api_key) + plat_value = plat.value if hasattr(plat, "value") else plat logger.debug( - f"[resolve_api_key] Found Integration key for '{provider_value}' (platform={plat_value}): " + f"[resolve_api_key] Found Integration key for '{provider_value}' " + f"(platform={plat_value}, credential_id={credential_id or integ.id}): " f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" ) return key except Exception as e: logger.error(f"Failed to decrypt Integration key for {provider}: {e}", exc_info=True) - else: - logger.debug(f"[resolve_api_key] No Integration record found for platform '{plat_value}'") - else: - logger.debug(f"[resolve_api_key] No platform mapping for provider '{provider_value}'") - logger.warning(f"[resolve_api_key] Could not resolve any API key for provider '{provider_value}'") + env_map = { + ModelProvider.OPENAI: "OPENAI_API_KEY", + ModelProvider.CARTESIA: "CARTESIA_API_KEY", + ModelProvider.ELEVENLABS: "ELEVENLABS_API_KEY", + ModelProvider.DEEPGRAM: "DEEPGRAM_API_KEY", + ModelProvider.GOOGLE: "GOOGLE_API_KEY", + ModelProvider.SARVAM: "SARVAM_API_KEY", + ModelProvider.MURF: "MURF_API_KEY", + ModelProvider.SMALLEST: "SMALLEST_API_KEY", + ModelProvider.VOICEMAKER: "VOICEMAKER_API_KEY", + } + env_var = env_map.get(provider) + if env_var: + env_key = os.getenv(env_var) + if env_key: + logger.debug( + f"[resolve_api_key] Using environment variable {env_var} for '{provider_value}'" + ) + return env_key + + logger.warning( + f"[resolve_api_key] Could not resolve any API key for provider '{provider_value}' " + f"(credential_id={credential_id})" + ) return None def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: @@ -481,6 +481,55 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs agent_silence_hangup_secs = resolve_agent_silence_hangup_secs(agent) + observability_call_short_id: Optional[str] = None + live_observability_emitter = None + if agent_id and workspace_id and result_id: + try: + from app.services.observability.live_event_emitter import LiveObservabilityEmitter + + live_observability_emitter = LiveObservabilityEmitter( + organization_id=organization_id, + workspace_id=workspace_id, + provider_call_id=result_id, + provider_platform="pipecat", + agent_ref=str(agent_id), + explicit_agent_id=UUID(agent_id), + ) + observability_call_short_id = live_observability_emitter.start_call() + except Exception as observability_start_error: + logger.warning( + f"Failed to start live observability ingest for voice session: {observability_start_error}", + exc_info=True, + ) + live_observability_emitter = None + + if use_voice_bundle_pipeline and live_observability_emitter is None: + try: + from app.models.database import CallRecordingSource + from app.services.observability.call_ingest import upsert_call_recording + + started_recording, _ = upsert_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + provider_platform="efficientai", + provider_call_id=result_id, + call_data_payload={ + "direction": "inbound", + "startedAt": datetime.now(UTC).isoformat(), + "live_transcript": [], + }, + explicit_agent_id=UUID(agent_id), + call_event="call_started", + source=CallRecordingSource.PLAYGROUND, + ) + observability_call_short_id = started_recording.call_short_id + except Exception as observability_start_error: + logger.warning( + f"Failed to create observability call record at session start: {observability_start_error}", + exc_info=True, + ) + try: if use_voice_bundle_pipeline: # Resolve per-provider keys for voice bundle @@ -488,9 +537,30 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: tts_provider = voice_bundle.tts_provider if voice_bundle else None llm_provider = voice_bundle.llm_provider if voice_bundle else None - stt_api_key = resolve_api_key_for_provider(stt_provider) if stt_provider else None - tts_api_key = resolve_api_key_for_provider(tts_provider) if tts_provider else None - llm_api_key = resolve_api_key_for_provider(llm_provider) if llm_provider else None + stt_api_key = ( + resolve_api_key_for_provider( + stt_provider, + credential_id=getattr(voice_bundle, "stt_credential_id", None), + ) + if stt_provider + else None + ) + tts_api_key = ( + resolve_api_key_for_provider( + tts_provider, + credential_id=getattr(voice_bundle, "tts_credential_id", None), + ) + if tts_provider + else None + ) + llm_api_key = ( + resolve_api_key_for_provider( + llm_provider, + credential_id=getattr(voice_bundle, "llm_credential_id", None), + ) + if llm_provider + else None + ) llm_endpoint_url = ( resolve_azure_endpoint_for_provider(llm_provider) if llm_provider and ( @@ -530,6 +600,9 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: llm_api_key=llm_api_key, llm_endpoint_url=llm_endpoint_url, silence_hangup_secs=agent_silence_hangup_secs, + workspace_id=str(workspace_id) if workspace_id else None, + call_short_id=observability_call_short_id, + live_observability_emitter=live_observability_emitter, ) else: call_metadata = await run_bot( @@ -544,6 +617,8 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: result_id=result_id, model_name=model_name, # Pass model name from voice bundle silence_hangup_secs=agent_silence_hangup_secs, + workspace_id=str(workspace_id) if workspace_id else None, + live_observability_emitter=live_observability_emitter, ) except Exception as bot_error: logger.error(f"Error in run_bot: {bot_error}", exc_info=True) @@ -551,6 +626,7 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: # Create evaluator result if we have the required data (only if no error) # Also create if we have a live transcript even without S3 audio + evaluator_result = None has_audio = call_metadata and call_metadata.get("s3_key") has_transcript = call_metadata and call_metadata.get("transcription") has_usable_data = has_audio or has_transcript @@ -682,6 +758,45 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: logger.info(f"✅ Created evaluator result {result_id} and triggered processing task") except Exception as e: logger.error(f"❌ Error creating evaluator result: {e}", exc_info=True) + + if call_metadata and agent_id and result_id and workspace_id: + if live_observability_emitter is not None: + try: + live_observability_emitter.end_call( + recording_s3_key=call_metadata.get("s3_key"), + duration_seconds=call_metadata.get("duration"), + trace_id=call_metadata.get("trace_id"), + ) + if not observability_call_short_id: + observability_call_short_id = live_observability_emitter.call_short_id + except Exception as live_end_error: + logger.warning( + f"Failed to finalize live observability ingest: {live_end_error}", + exc_info=True, + ) + has_observability_payload = any( + call_metadata.get(key) + for key in ("s3_key", "transcription", "speaker_segments", "trace_id") + ) + if has_observability_payload or observability_call_short_id: + try: + from app.services.observability.call_ingest import persist_playground_voice_call + + persist_playground_voice_call( + db, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=UUID(agent_id), + result_id=result_id, + call_metadata=call_metadata, + evaluator_result_id=evaluator_result.id if evaluator_result else None, + provider_platform="efficientai" if use_voice_bundle_pipeline else "efficientai_legacy", + ) + except Exception as persist_error: + logger.warning( + f"Failed to persist voice call to observability: {persist_error}", + exc_info=True, + ) except WebSocketDisconnect: print("WebSocket disconnected by client") diff --git a/app/cli.py b/app/cli.py index 7fbefbff..e6f3db20 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,6 +1,7 @@ """CLI for EfficientAI platform.""" import click +import platform import yaml import os import sys @@ -11,6 +12,21 @@ from typing import Optional +def _is_macos() -> bool: + return platform.system() == "Darwin" + + +def _local_dev_celery_pool() -> str: + """Prefork forks after threads/ObjC init and crashes psycopg2 on macOS.""" + return "threads" if _is_macos() else "prefork" + + +def _local_dev_celery_concurrency(pool: str) -> int: + if pool == "threads": + return 4 if _is_macos() else 8 + return 8 + + @click.group() def main(): """EfficientAI - Voice AI Evaluation Platform CLI.""" @@ -395,12 +411,16 @@ def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optio click.echo(f"🚀 Starting Celery worker...") click.echo(f" Log level: {loglevel}") + resolved_pool = pool or _local_dev_celery_pool() + resolved_concurrency = ( + concurrency if concurrency is not None else _local_dev_celery_concurrency(resolved_pool) + ) if queues: click.echo(f" Queues: {queues}") - if concurrency is not None: - click.echo(f" Concurrency: {concurrency}") - if pool: - click.echo(f" Pool: {pool}") + click.echo(f" Concurrency: {resolved_concurrency}") + click.echo(f" Pool: {resolved_pool}") + if _is_macos() and pool is None: + click.echo(" macOS: defaulting to pool=threads (avoids prefork/psycopg2 crashes)") # Start Celery worker try: @@ -408,10 +428,8 @@ def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optio cmd = ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"] if queues: cmd.append(f"--queues={queues}") - if concurrency is not None: - cmd.append(f"--concurrency={concurrency}") - if pool: - cmd.append(f"--pool={pool}") + cmd.append(f"--pool={resolved_pool}") + cmd.append(f"--concurrency={resolved_concurrency}") subprocess.run(cmd, check=True) except KeyboardInterrupt: click.echo("\n👋 Celery worker stopped") @@ -616,8 +634,19 @@ def _handle_signal(sig, frame): [sys.executable, "-m", "app.cli", "telephony-worker", "--config", str(config_path), "--port", str(port)], env=telephony_env, ) + default_pool = _local_dev_celery_pool() celery_proc = subprocess.Popen( - ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"], + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={loglevel}", + "-P", + default_pool, + "-c", + str(_local_dev_celery_concurrency(default_pool)), + ], ) click.echo(f"🚀 Started media server (pid={media_proc.pid}) and Celery worker (pid={celery_proc.pid})") @@ -951,6 +980,8 @@ def _stream_telephony(): # Start Celery workers as subprocess(es) with output streaming try: + default_pool = _local_dev_celery_pool() + default_concurrency = _local_dev_celery_concurrency(default_pool) worker_process = _spawn_worker( [ "celery", @@ -960,12 +991,21 @@ def _stream_telephony(): f"--loglevel={worker_loglevel}", "-Q", "celery,audio-metrics", + "-P", + default_pool, "-c", - "8", + str(default_concurrency), ], - label="Celery worker (celery + audio-metrics queues, concurrency=8)", + label=( + f"Celery worker (celery + audio-metrics queues, " + f"pool={default_pool}, concurrency={default_concurrency})" + ), prefix="[WORKER]", ) + if _is_macos(): + click.echo( + " macOS: Celery default worker uses pool=threads to avoid prefork/psycopg2 crashes" + ) if imports_worker: from app.workers.config import IMPORTS_WORKER_QUEUES diff --git a/app/config.py b/app/config.py index 8242409c..78d87307 100644 --- a/app/config.py +++ b/app/config.py @@ -162,6 +162,31 @@ class Settings(BaseSettings): # Observability / Loki OBSERVABILITY_ENABLED: bool = False + OBSERVABILITY_TRACING_ENABLED: bool = False + OBSERVABILITY_TRACING_EXPORTER: str = "efficientai_http" + OBSERVABILITY_TRACING_SAMPLE_RATE: float = 1.0 + OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS: bool = True + OBSERVABILITY_TRACE_QUOTA_PER_ORG_PER_DAY: Optional[int] = None + OBSERVABILITY_LIVE_INGEST_ENABLED: bool = False + OBSERVABILITY_LIVE_AGGREGATES_ENABLED: bool = False + OBSERVABILITY_LIVE_DASHBOARD_ENABLED: bool = False + OBSERVABILITY_LIVE_EVENT_IDEMPOTENCY_TTL_SECONDS: int = 86400 + OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ: int = 5 + OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS: int = 300 + OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED: bool = False + OBSERVABILITY_LIVE_SLO_AUTOMATION_ENABLED: bool = False + OBSERVABILITY_LIVE_SLO_P90_LLM_MS: int = 1800 + OBSERVABILITY_LIVE_SLO_MIN_SAMPLE_COUNT: int = 20 + OBSERVABILITY_ELEVENLABS_SKIP_RECORDING_ARCHIVE: bool = True + ELEVENLABS_SYNC_MAX_RPS: float = 5.0 + ELEVENLABS_MONITOR_MAX_CONCURRENCY: int = 50 + OTEL_EXPORTER_OTLP_ENDPOINT: str = "https://otel-http.efficientai.ai/v1/traces" + TRACING_QUERY_BACKEND: str = "cloud" # cloud | tempo + EFFICIENT_AI_API_KEY: Optional[str] = None + EFFICIENT_AI_AGENT_ID: Optional[str] = None + EFFICIENT_AI_PROJECT_ID: Optional[str] = None + EFFICIENT_AI_TRACE_QUERY_URL: str = "https://api.efficientai.ai/observability/v1/traces" + TEMPO_QUERY_URL: str = "http://tempo:3200" LOKI_ENABLED: bool = False LOKI_URL: str = "http://loki:3100" LOKI_STORAGE: str = "filesystem" # "filesystem" or "s3" @@ -209,6 +234,7 @@ class Settings(BaseSettings): "vobiz.ai", "amazonaws.com", "cloudfront.net", + "elevenlabs.io", ] # Live telephony pipeline recording merge (dual-track → natural mono) @@ -735,6 +761,55 @@ def load_config_from_file(config_path: str) -> None: obs_config = config_data["observability"] if "enabled" in obs_config: settings.OBSERVABILITY_ENABLED = bool(obs_config["enabled"]) + if "tracing" in obs_config: + tracing_config = obs_config["tracing"] or {} + if "enabled" in tracing_config: + settings.OBSERVABILITY_TRACING_ENABLED = bool(tracing_config["enabled"]) + if "exporter" in tracing_config: + settings.OBSERVABILITY_TRACING_EXPORTER = str(tracing_config["exporter"]) + if "sample_rate" in tracing_config: + settings.OBSERVABILITY_TRACING_SAMPLE_RATE = float(tracing_config["sample_rate"]) + if "include_transcripts" in tracing_config: + settings.OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS = bool( + tracing_config["include_transcripts"] + ) + if "trace_quota_per_org_per_day" in tracing_config: + quota_value = tracing_config["trace_quota_per_org_per_day"] + settings.OBSERVABILITY_TRACE_QUOTA_PER_ORG_PER_DAY = ( + int(quota_value) if quota_value is not None else None + ) + if "endpoint" in tracing_config and tracing_config["endpoint"]: + settings.OTEL_EXPORTER_OTLP_ENDPOINT = str(tracing_config["endpoint"]) + if "query_backend" in tracing_config and tracing_config["query_backend"]: + settings.TRACING_QUERY_BACKEND = str(tracing_config["query_backend"]) + if "trace_query_url" in tracing_config and tracing_config["trace_query_url"]: + settings.EFFICIENT_AI_TRACE_QUERY_URL = str(tracing_config["trace_query_url"]) + if "tempo_query_url" in tracing_config and tracing_config["tempo_query_url"]: + settings.TEMPO_QUERY_URL = str(tracing_config["tempo_query_url"]) + if "live" in obs_config: + live_config = obs_config["live"] or {} + if "ingest_enabled" in live_config: + settings.OBSERVABILITY_LIVE_INGEST_ENABLED = bool(live_config["ingest_enabled"]) + if "aggregates_enabled" in live_config: + settings.OBSERVABILITY_LIVE_AGGREGATES_ENABLED = bool(live_config["aggregates_enabled"]) + if "dashboard_enabled" in live_config: + settings.OBSERVABILITY_LIVE_DASHBOARD_ENABLED = bool(live_config["dashboard_enabled"]) + if "slo_alerts_enabled" in live_config: + settings.OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED = bool(live_config["slo_alerts_enabled"]) + if "slo_automation_enabled" in live_config: + settings.OBSERVABILITY_LIVE_SLO_AUTOMATION_ENABLED = bool( + live_config["slo_automation_enabled"] + ) + if "elevenlabs_skip_recording_archive" in live_config: + settings.OBSERVABILITY_ELEVENLABS_SKIP_RECORDING_ARCHIVE = bool( + live_config["elevenlabs_skip_recording_archive"] + ) + if "elevenlabs_sync_max_rps" in live_config: + settings.ELEVENLABS_SYNC_MAX_RPS = float(live_config["elevenlabs_sync_max_rps"]) + if "elevenlabs_monitor_max_concurrency" in live_config: + settings.ELEVENLABS_MONITOR_MAX_CONCURRENCY = int( + live_config["elevenlabs_monitor_max_concurrency"] + ) if "loki" in obs_config: loki_config = obs_config["loki"] if "enabled" in loki_config: diff --git a/app/migrations/059_call_recordings_trace_id.py b/app/migrations/059_call_recordings_trace_id.py new file mode 100644 index 00000000..33e9f778 --- /dev/null +++ b/app/migrations/059_call_recordings_trace_id.py @@ -0,0 +1,53 @@ +""" +Migration: add trace_id on call_recordings. + +Links OpenTelemetry traces to observability call records. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add trace_id column and index to call_recordings" + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name + AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "call_recordings", "trace_id"): + db.execute( + text( + """ + ALTER TABLE call_recordings + ADD COLUMN trace_id VARCHAR(64) NULL + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_call_recordings_trace_id + ON call_recordings (trace_id) + """ + ) + ) + db.commit() + print("Added call_recordings.trace_id") + + +def downgrade(db: Session): + db.execute(text("DROP INDEX IF EXISTS ix_call_recordings_trace_id")) + db.execute(text("ALTER TABLE call_recordings DROP COLUMN IF EXISTS trace_id")) + db.commit() diff --git a/app/migrations/060_agent_auto_evaluator_id.py b/app/migrations/060_agent_auto_evaluator_id.py new file mode 100644 index 00000000..7d31ea2e --- /dev/null +++ b/app/migrations/060_agent_auto_evaluator_id.py @@ -0,0 +1,65 @@ +""" +Migration: add observability_auto_evaluator_id to agents. + +Allows auto-queueing evaluator runs for observability call_ended ingestion. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add observability_auto_evaluator_id column to agents" + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = :table_name + AND column_name = :column_name + """ + ), + {"table_name": table_name, "column_name": column_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _column_exists(db, "agents", "observability_auto_evaluator_id"): + db.execute( + text( + """ + ALTER TABLE agents + ADD COLUMN observability_auto_evaluator_id UUID NULL + """ + ) + ) + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_agents_observability_auto_evaluator_id + ON agents (observability_auto_evaluator_id) + """ + ) + ) + db.execute( + text( + """ + ALTER TABLE agents + ADD CONSTRAINT fk_agents_observability_auto_evaluator_id + FOREIGN KEY (observability_auto_evaluator_id) + REFERENCES evaluators (id) + ON DELETE SET NULL + """ + ) + ) + db.commit() + print("Added agents.observability_auto_evaluator_id") + + +def downgrade(db: Session): + db.execute(text("ALTER TABLE agents DROP CONSTRAINT IF EXISTS fk_agents_observability_auto_evaluator_id")) + db.execute(text("DROP INDEX IF EXISTS ix_agents_observability_auto_evaluator_id")) + db.execute(text("ALTER TABLE agents DROP COLUMN IF EXISTS observability_auto_evaluator_id")) + db.commit() diff --git a/app/migrations/061_provider_sync_jobs.py b/app/migrations/061_provider_sync_jobs.py new file mode 100644 index 00000000..87011a6a --- /dev/null +++ b/app/migrations/061_provider_sync_jobs.py @@ -0,0 +1,87 @@ +"""Migration: add provider_sync_jobs tables for ElevenLabs migration workflow.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add provider sync job tables" + + +def _table_exists(db: Session, table_name: str) -> bool: + row = db.execute( + text( + """ + SELECT 1 + FROM information_schema.tables + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ).first() + return row is not None + + +def upgrade(db: Session): + if not _table_exists(db, "provider_sync_jobs"): + db.execute( + text( + """ + CREATE TABLE provider_sync_jobs ( + id UUID PRIMARY KEY, + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + integration_id UUID NOT NULL REFERENCES integrations(id) ON DELETE CASCADE, + provider_platform VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'queued', + phase VARCHAR(32) NOT NULL DEFAULT 'queued', + config JSON, + cursor_state JSON, + agents_synced INTEGER NOT NULL DEFAULT 0, + conversations_cataloged INTEGER NOT NULL DEFAULT 0, + conversations_enriched INTEGER NOT NULL DEFAULT 0, + errors_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + started_at TIMESTAMPTZ NULL, + completed_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + + if not _table_exists(db, "provider_sync_job_errors"): + db.execute( + text( + """ + CREATE TABLE provider_sync_job_errors ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES provider_sync_jobs(id) ON DELETE CASCADE, + provider_call_id VARCHAR(255) NULL, + provider_agent_id VARCHAR(255) NULL, + phase VARCHAR(32) NOT NULL, + error_message TEXT NOT NULL, + payload JSON, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_org_id ON provider_sync_jobs (organization_id)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_workspace_id ON provider_sync_jobs (workspace_id)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_integration_id ON provider_sync_jobs (integration_id)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_status ON provider_sync_jobs (status)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_phase ON provider_sync_jobs (phase)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_jobs_platform ON provider_sync_jobs (provider_platform)")) + + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_job_errors_job_id ON provider_sync_job_errors (job_id)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_job_errors_call_id ON provider_sync_job_errors (provider_call_id)")) + db.execute(text("CREATE INDEX IF NOT EXISTS ix_provider_sync_job_errors_phase ON provider_sync_job_errors (phase)")) + db.commit() + print("Added provider sync job tables") + + +def downgrade(db: Session): + db.execute(text("DROP TABLE IF EXISTS provider_sync_job_errors")) + db.execute(text("DROP TABLE IF EXISTS provider_sync_jobs")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 92350139..e8d75948 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -521,6 +521,12 @@ class Agent(Base): # Voice AI agent integration (Retell, Vapi, etc.) voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + observability_auto_evaluator_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluators.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) prompt_variables = Column(JSON, nullable=True) silence_hangup_secs = Column(Integer, nullable=False, server_default="15") @@ -617,6 +623,61 @@ class Integration(Base): last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated +class ProviderSyncJob(Base): + """Background sync job for provider-side agent/conversation migration.""" + + __tablename__ = "provider_sync_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + integration_id = Column( + UUID(as_uuid=True), + ForeignKey("integrations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + provider_platform = Column(String(64), nullable=False, index=True) + status = Column(String(32), nullable=False, default="queued", index=True) # queued|running|completed|failed|cancelled + phase = Column(String(32), nullable=False, default="queued", index=True) # agents|catalog|enrich|complete|failed + config = Column(JSON, nullable=True) # since_unix, agent_ids, insights_only + cursor_state = Column(JSON, nullable=True) # pagination cursors and runtime checkpoints + agents_synced = Column(Integer, nullable=False, default=0) + conversations_cataloged = Column(Integer, nullable=False, default=0) + conversations_enriched = Column(Integer, nullable=False, default=0) + errors_count = Column(Integer, nullable=False, default=0) + last_error = Column(Text, nullable=True) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False) + + +class ProviderSyncJobError(Base): + """Per-item sync errors captured for diagnostics/retry.""" + + __tablename__ = "provider_sync_job_errors" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + job_id = Column( + UUID(as_uuid=True), + ForeignKey("provider_sync_jobs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + provider_call_id = Column(String(255), nullable=True, index=True) + provider_agent_id = Column(String(255), nullable=True, index=True) + phase = Column(String(32), nullable=False, index=True) + error_message = Column(Text, nullable=False) + payload = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + class ManualTranscription(Base): """Manual transcription model for storing transcriptions from S3 audio files.""" @@ -1139,6 +1200,7 @@ class CallRecording(Base): call_data = Column(JSON, nullable=True) # JSON blob for provider response provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + trace_id = Column(String(64), nullable=True, index=True) agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent # Link to EvaluatorResult for metric evaluations @@ -1183,6 +1245,111 @@ class CallRecordingPayload(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) +class ObservabilityLiveEventDedup(Base): + """Idempotency ledger for live observability events.""" + + __tablename__ = "observability_live_event_dedup" + __table_args__ = ( + UniqueConstraint( + "organization_id", + "event_id", + name="uq_observability_live_event_dedup_org_event", + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + event_id = Column(String(128), nullable=False, index=True) + provider_platform = Column(String(64), nullable=False, index=True) + provider_call_id = Column(String(255), nullable=False, index=True) + call_short_id = Column(String(6), nullable=True, index=True) + seq = Column(BigInteger, nullable=True) + event_ts = Column(DateTime(timezone=True), nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + +class ObservabilityLiveLatencySample(Base): + """Latency samples captured from live events for rolling percentile queries.""" + + __tablename__ = "observability_live_latency_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_recording_id = Column( + UUID(as_uuid=True), + ForeignKey("call_recordings.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), nullable=False, index=True) + provider_platform = Column(String(64), nullable=False, index=True) + provider_call_id = Column(String(255), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True, index=True) + metric_name = Column(String(64), nullable=False, index=True) + latency_ms = Column(Float, nullable=False) + event_ts = Column(DateTime(timezone=True), nullable=False, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + +class ObservabilityLiveSloBreach(Base): + """Records live SLO breaches for alerting and automation hooks.""" + + __tablename__ = "observability_live_slo_breaches" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_recording_id = Column( + UUID(as_uuid=True), + ForeignKey("call_recordings.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + call_short_id = Column(String(6), nullable=True, index=True) + provider_platform = Column(String(64), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True, index=True) + metric_name = Column(String(64), nullable=False, index=True) + window_seconds = Column(Integer, nullable=False) + p90_ms = Column(Float, nullable=False) + threshold_ms = Column(Float, nullable=False) + sample_count = Column(Integer, nullable=False) + evaluator_queued = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + class Alert(Base): """Alert model for configuring monitoring alerts.""" __tablename__ = "alerts" diff --git a/app/models/schemas.py b/app/models/schemas.py index 40e988e9..af367260 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -841,6 +841,60 @@ def convert_routing_mode(cls, v): model_config = ConfigDict(from_attributes=True) +class ExternalProviderAgent(BaseModel): + """Normalized provider agent metadata for integration picker UIs.""" + + id: str + name: str + archived: bool = False + created_at: Optional[datetime] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class ExternalAgentListResponse(BaseModel): + agents: List[ExternalProviderAgent] + has_more: bool = False + next_cursor: Optional[str] = None + + +class ElevenLabsConversationSyncRequest(BaseModel): + """Start configuration for ElevenLabs conversation catalog/enrich sync.""" + + since_unix: Optional[int] = Field( + default=None, + description="Unix timestamp lower bound; defaults to last 30 days when omitted.", + ) + agent_ids: Optional[List[str]] = Field( + default=None, + description="Optional subset of ElevenLabs agent IDs to sync.", + ) + insights_only: bool = Field( + default=True, + description="When true, store transcript/analysis/cost pointers without downloading audio.", + ) + + +class ProviderSyncJobResponse(BaseModel): + id: UUID + integration_id: UUID + provider_platform: str + status: Literal["queued", "running", "completed", "failed", "cancelled"] + phase: str + config: Dict[str, Any] = Field(default_factory=dict) + cursor_state: Dict[str, Any] = Field(default_factory=dict) + agents_synced: int = 0 + conversations_cataloged: int = 0 + conversations_enriched: int = 0 + errors_count: int = 0 + last_error: Optional[str] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + # ============================================ # DATA SOURCES SCHEMAS # ============================================ diff --git a/app/services/ai/llm_service.py b/app/services/ai/llm_service.py index 95e64488..b8e66cc9 100644 --- a/app/services/ai/llm_service.py +++ b/app/services/ai/llm_service.py @@ -63,6 +63,34 @@ r"(?:^|[/-])gemini-3(?:\.\d+)?(?:[-.]|$)", re.IGNORECASE ) +# OpenAI/Azure reasoning families reject non-default temperature (must be 1 +# or omitted). Excludes ``gpt-5-chat*`` which supports flexible sampling. +_OPENAI_FIXED_TEMPERATURE_RE = re.compile( + r"(?:^|[/-])(?:gpt-5(?!-chat)|o[134])(?:[-.]|$)", + re.IGNORECASE, +) + + +def _model_only_supports_default_temperature(model: str) -> bool: + """Return True when the provider rejects custom ``temperature`` values.""" + model_name = (model or "").rsplit("/", 1)[-1] + return bool(_OPENAI_FIXED_TEMPERATURE_RE.search(model_name)) + + +def _normalize_temperature_for_model(model: str, call_kwargs: Dict[str, Any]) -> None: + """Drop ``temperature`` when the target model only supports the default.""" + if not _model_only_supports_default_temperature(model): + return + temp = call_kwargs.get("temperature") + if temp is None or temp == 1: + return + logger.debug( + "[LLMService] Omitting temperature={} for {} (model only supports default)", + temp, + model, + ) + call_kwargs.pop("temperature", None) + def _gemini_family(model: str) -> Optional[str]: """Return ``"2.5"``, ``"3"``, or ``None`` for the given model name. @@ -489,6 +517,7 @@ def generate_response( model=model_str, credential=credential_ctx, ) + _normalize_temperature_for_model(model_str, call_kwargs) try: response = litellm.completion(**call_kwargs) diff --git a/app/services/ai/transcription_service.py b/app/services/ai/transcription_service.py index 9620e1ca..07412c10 100644 --- a/app/services/ai/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -49,7 +49,8 @@ from app.models.database import ModelProvider, AIProvider, Integration from app.core.encryption import decrypt_api_key from app.services.credentials import resolve_ai_provider, resolve_integration -from app.services.storage.s3_service import s3_service +from app.config import settings +from app.services.storage.blob_storage_service import blob_storage_service from app.core.exceptions import StorageError from sqlalchemy.orm import Session @@ -137,11 +138,12 @@ def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = N """ import os - # First, try S3 if enabled - if s3_service.is_enabled(): + # First, try cloud blob storage when configured + if blob_storage_service.is_enabled() or settings.S3_ENABLED: try: - # Download from S3 - audio_bytes = s3_service.download_file_by_key(audio_file_key) + if not blob_storage_service.is_enabled(): + blob_storage_service.reset_connection() + audio_bytes = blob_storage_service.download_file_by_key(audio_file_key) # Determine file extension from key file_ext = Path(audio_file_key).suffix.lstrip(".") or "wav" diff --git a/app/services/observability/call_ingest.py b/app/services/observability/call_ingest.py new file mode 100644 index 00000000..0c0f47dd --- /dev/null +++ b/app/services/observability/call_ingest.py @@ -0,0 +1,309 @@ +"""Call recording ingest helpers for observability.""" + +from datetime import UTC, datetime +from typing import Any, Dict, Optional, Tuple +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import ( + Agent, + CallRecording, + CallRecordingSource, + CallRecordingStatus, +) +from app.services.observability.live_ingest import ( + derive_live_call_event, + merge_live_event_call_data, +) +from app.services.billing.flexprice_service import record_observability_call_ingested +from app.utils.call_recordings import generate_unique_call_short_id + +# Sources surfaced in the Observability UI/API (webhooks + live playground/voice bundle). +OBSERVABILITY_CALL_SOURCES: Tuple[CallRecordingSource, ...] = ( + CallRecordingSource.WEBHOOK, + CallRecordingSource.PLAYGROUND, +) + + +def upsert_call_recording( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + provider_platform: str, + provider_call_id: str, + call_data_payload: Dict[str, Any], + agent_ref_raw: Optional[str] = None, + explicit_agent_id: Optional[UUID] = None, + call_event: Optional[str] = None, + trace_id: Optional[str] = None, + evaluator_result_id: Optional[UUID] = None, + source: CallRecordingSource = CallRecordingSource.WEBHOOK, +) -> tuple[CallRecording, str]: + """Create or update a call recording for an organization + workspace.""" + agent_id: Optional[UUID] = explicit_agent_id + if not agent_id and agent_ref_raw: + try: + agent_uuid = UUID(agent_ref_raw) + agent = ( + db.query(Agent) + .filter(Agent.id == agent_uuid, Agent.organization_id == organization_id) + .first() + ) + if agent: + agent_id = agent.id + if agent.workspace_id and agent.workspace_id != workspace_id: + workspace_id = agent.workspace_id + except ValueError: + agent_id = None + + if agent_ref_raw: + call_data_payload.setdefault("_agent_ref", agent_ref_raw) + + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.provider_call_id == provider_call_id, + CallRecording.provider_platform == provider_platform, + ) + .first() + ) + + created = call_recording is None + if call_recording: + call_recording.call_data = call_data_payload + call_recording.status = CallRecordingStatus.UPDATED + call_recording.source = source + if trace_id: + call_recording.trace_id = trace_id + if call_event: + call_recording.call_event = call_event + if agent_id: + call_recording.agent_id = agent_id + if evaluator_result_id: + call_recording.evaluator_result_id = evaluator_result_id + else: + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=generate_unique_call_short_id(db), + status=CallRecordingStatus.UPDATED, + call_event=call_event, + source=source, + call_data=call_data_payload, + provider_call_id=provider_call_id, + provider_platform=provider_platform, + trace_id=trace_id, + agent_id=agent_id, + evaluator_result_id=evaluator_result_id, + ) + db.add(call_recording) + + db.commit() + db.refresh(call_recording) + + if created: + record_observability_call_ingested( + organization_id, + call_recording.call_short_id, + workspace_id=workspace_id, + provider=provider_platform, + ) + + return call_recording, ("created" if created else "updated") + + +def persist_playground_voice_call( + db: Session, + *, + organization_id: UUID, + workspace_id: UUID, + agent_id: UUID, + result_id: str, + call_metadata: Dict[str, Any], + evaluator_result_id: Optional[UUID] = None, + provider_platform: str = "efficientai", +) -> Optional[CallRecording]: + """Upsert a playground/live voice call into observability with trace linkage.""" + if not call_metadata: + return None + + existing = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.provider_call_id == result_id, + CallRecording.provider_platform == provider_platform, + ) + .first() + ) + existing_data = existing.call_data if existing and isinstance(existing.call_data, dict) else {} + + call_data_payload: Dict[str, Any] = { + "direction": "inbound", + "evaluator_result_id": result_id, + } + + started_at = existing_data.get("startedAt") or existing_data.get("started_at") + if started_at: + call_data_payload["startedAt"] = started_at + call_data_payload["endedAt"] = datetime.now(UTC).isoformat() + + duration = call_metadata.get("duration") + if duration is not None: + call_data_payload["duration_seconds"] = duration + + s3_key = call_metadata.get("s3_key") + if s3_key: + call_data_payload["recording_s3_key"] = s3_key + elif existing_data.get("recording_s3_key"): + call_data_payload["recording_s3_key"] = existing_data["recording_s3_key"] + + transcription = call_metadata.get("transcription") + speaker_segments = call_metadata.get("speaker_segments") + if transcription: + call_data_payload["transcription"] = transcription + if speaker_segments: + call_data_payload["speaker_segments"] = speaker_segments + + live_transcript = list(existing_data.get("live_transcript") or []) + if live_transcript: + call_data_payload["live_transcript"] = live_transcript + + from app.services.telephony.call_recording_lifecycle import resolve_telephony_messages + + resolved_messages = resolve_telephony_messages( + live_transcript=live_transcript, + conversation_turns=speaker_segments, + ) + if resolved_messages: + call_data_payload["messages"] = resolved_messages + elif transcription: + call_data_payload["messages"] = [{"role": "user", "content": transcription}] + if not live_transcript: + call_data_payload["live_transcript"] = call_data_payload["messages"] + elif existing_data.get("messages"): + call_data_payload["messages"] = existing_data["messages"] + + if call_metadata.get("error"): + call_data_payload["error"] = call_metadata["error"] + call_data_payload["endedReason"] = str(call_metadata["error"]) + + trace_id = call_metadata.get("trace_id") or existing_data.get("trace_id") + if trace_id: + call_data_payload["trace_id"] = trace_id + call_event = "call_failed" if call_metadata.get("error") else "call_ended" + + call_recording, _action = upsert_call_recording( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + provider_platform=provider_platform, + provider_call_id=result_id, + call_data_payload=call_data_payload, + explicit_agent_id=agent_id, + call_event=call_event, + trace_id=trace_id, + evaluator_result_id=evaluator_result_id, + source=CallRecordingSource.PLAYGROUND, + ) + return call_recording + + +def upsert_live_event_call_recording( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + provider_platform: str, + provider_call_id: str, + live_event: Dict[str, Any], + max_out_of_order_seq: int, + agent_ref_raw: Optional[str] = None, + explicit_agent_id: Optional[UUID] = None, + source: CallRecordingSource = CallRecordingSource.WEBHOOK, + persist: bool = True, +) -> tuple[CallRecording, str]: + """Upsert a call recording by incrementally merging a live event envelope.""" + agent_id: Optional[UUID] = explicit_agent_id + if not agent_id and agent_ref_raw: + try: + agent_uuid = UUID(agent_ref_raw) + agent = ( + db.query(Agent) + .filter(Agent.id == agent_uuid, Agent.organization_id == organization_id) + .first() + ) + if agent: + agent_id = agent.id + if agent.workspace_id and agent.workspace_id != workspace_id: + workspace_id = agent.workspace_id + except ValueError: + agent_id = None + + event_type = str(live_event.get("event_type") or "").strip().lower() + call_event = derive_live_call_event(event_type) + trace_id = live_event.get("trace_id") + + call_recording = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == organization_id, + CallRecording.provider_call_id == provider_call_id, + CallRecording.provider_platform == provider_platform, + ) + .first() + ) + + created = call_recording is None + if call_recording is None: + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=generate_unique_call_short_id(db), + status=CallRecordingStatus.UPDATED, + call_event=call_event, + source=source, + call_data={}, + provider_call_id=provider_call_id, + provider_platform=provider_platform, + trace_id=str(trace_id) if trace_id else None, + agent_id=agent_id, + ) + db.add(call_recording) + db.flush() + else: + call_recording.status = CallRecordingStatus.UPDATED + call_recording.call_event = call_event + call_recording.source = source + if trace_id: + call_recording.trace_id = str(trace_id) + if agent_id: + call_recording.agent_id = agent_id + + merged_call_data = merge_live_event_call_data( + existing_call_data=call_recording.call_data if isinstance(call_recording.call_data, dict) else {}, + event=live_event, + max_out_of_order_seq=max_out_of_order_seq, + ) + if agent_ref_raw: + merged_call_data.setdefault("_agent_ref", agent_ref_raw) + if trace_id and not merged_call_data.get("trace_id"): + merged_call_data["trace_id"] = trace_id + call_recording.call_data = merged_call_data + + if persist: + db.commit() + db.refresh(call_recording) + + if created and persist: + record_observability_call_ingested( + organization_id, + call_recording.call_short_id, + workspace_id=workspace_id, + provider=provider_platform, + ) + + return call_recording, ("created" if created else "updated") diff --git a/app/services/observability/elevenlabs_monitor_bridge.py b/app/services/observability/elevenlabs_monitor_bridge.py new file mode 100644 index 00000000..cde3b54f --- /dev/null +++ b/app/services/observability/elevenlabs_monitor_bridge.py @@ -0,0 +1,163 @@ +"""Bridge ElevenLabs monitor websocket events into EfficientAI live ingest.""" + +from __future__ import annotations + +import json +import uuid +from datetime import UTC, datetime +from typing import Any, Dict, Optional + +import httpx +from loguru import logger + +try: + import websockets +except Exception: # pragma: no cover - optional dependency + websockets = None + + +class ElevenLabsMonitorBridge: + """Stream ElevenLabs monitor events into /observability/live/events.""" + + def __init__( + self, + *, + conversation_id: str, + elevenlabs_api_key: str, + efficientai_api_key: str, + workspace_id: Optional[str] = None, + efficientai_base_url: str = "http://localhost:8000", + provider_platform: str = "elevenlabs", + ) -> None: + self.conversation_id = conversation_id + self.elevenlabs_api_key = elevenlabs_api_key + self.efficientai_api_key = efficientai_api_key + self.workspace_id = workspace_id + self.efficientai_base_url = efficientai_base_url.rstrip("/") + self.provider_platform = provider_platform + self.trace_id: Optional[str] = None + self._seq = 0 + self._started = False + self._ended = False + + async def run(self) -> None: + if websockets is None: + raise RuntimeError("websockets is required. Install with: pip install websockets") + + if not self._started: + await self._post_event("call.started", {"startedAt": self._now_iso(), "status": "in_progress"}) + self._started = True + + monitor_url = ( + "wss://api.elevenlabs.io/v1/convai/conversations/" + f"{self.conversation_id}/monitor" + ) + logger.info("Connecting ElevenLabs monitor websocket for conversation={}", self.conversation_id) + headers = {"xi-api-key": self.elevenlabs_api_key} + + connect_kwargs = {"max_size": 16 * 1024 * 1024} + try: + async with websockets.connect( + monitor_url, + additional_headers=headers, + **connect_kwargs, + ) as ws: + await self._recv_loop(ws) + except TypeError: + async with websockets.connect( + monitor_url, + extra_headers=headers, + **connect_kwargs, + ) as ws: + await self._recv_loop(ws) + finally: + if not self._ended: + await self._post_event("call.ended", {"endedAt": self._now_iso(), "status": "ended"}) + self._ended = True + + async def _recv_loop(self, ws: Any) -> None: + async for raw in ws: + try: + event = json.loads(raw) + except Exception: + continue + await self._handle_event(event) + + async def _handle_event(self, event: Dict[str, Any]) -> None: + event_type = str(event.get("type") or "").strip() + + if event_type == "user_transcript": + evt = event.get("user_transcription_event") or {} + text = evt.get("user_transcript") + if isinstance(text, str) and text.strip(): + await self._post_event( + "turn.user", + {"content": text.strip(), "role": "user"}, + ) + return + + if event_type == "agent_response": + evt = event.get("agent_response_event") or {} + text = evt.get("agent_response") + if isinstance(text, str) and text.strip(): + await self._post_event( + "turn.assistant", + {"content": text.strip(), "role": "assistant"}, + ) + return + + if event_type == "agent_response_correction": + evt = event.get("agent_response_correction_event") or {} + text = evt.get("corrected_agent_response") + if isinstance(text, str) and text.strip(): + await self._post_event( + "turn.assistant", + { + "content": text.strip(), + "role": "assistant", + "replace_last_by_role": True, + }, + ) + return + + if event_type in {"conversation_ended", "call_ended"}: + await self._post_event("call.ended", {"endedAt": self._now_iso(), "status": "ended"}) + self._ended = True + return + + async def _post_event(self, event_type: str, payload: Dict[str, Any]) -> Dict[str, Any]: + self._seq += 1 + body: Dict[str, Any] = { + "event_id": f"el-monitor-{self.conversation_id}-{self._seq}-{uuid.uuid4().hex[:8]}", + "call_id": self.conversation_id, + "event_type": event_type, + "seq": self._seq, + "event_ts": self._now_iso(), + "platform": self.provider_platform, + "payload": payload, + } + if self.trace_id: + body["trace_id"] = self.trace_id + + headers = { + "X-API-Key": self.efficientai_api_key, + "Content-Type": "application/json", + } + if self.workspace_id: + headers["X-Workspace-Id"] = self.workspace_id + + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post( + f"{self.efficientai_base_url}/api/v1/observability/live/events", + headers=headers, + content=json.dumps(body), + ) + resp.raise_for_status() + data = resp.json() + if data.get("trace_id") and not self.trace_id: + self.trace_id = str(data["trace_id"]) + return data + + @staticmethod + def _now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/app/services/observability/elevenlabs_trace.py b/app/services/observability/elevenlabs_trace.py new file mode 100644 index 00000000..4c2800f1 --- /dev/null +++ b/app/services/observability/elevenlabs_trace.py @@ -0,0 +1,265 @@ +"""Normalize ElevenLabs OTLP payloads for observability trace UI.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional + + +def _normalize_span_attributes(raw_attrs: Any) -> Dict[str, Any]: + if isinstance(raw_attrs, dict): + return dict(raw_attrs) + if isinstance(raw_attrs, list): + result: Dict[str, Any] = {} + for item in raw_attrs: + if not isinstance(item, dict): + continue + key = item.get("key") + if not key: + continue + value = item.get("value") + if isinstance(value, dict): + for scalar_key in ("stringValue", "intValue", "doubleValue", "boolValue"): + if scalar_key in value: + result[key] = value[scalar_key] + break + else: + result[key] = value + else: + result[key] = value + return result + return {} + + +def _to_epoch_ms(value: Any) -> Optional[float]: + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if stripped.isdigit() or (stripped.replace(".", "", 1).isdigit() and stripped.count(".") <= 1): + return _to_epoch_ms(float(stripped)) + try: + return datetime.fromisoformat(stripped.replace("Z", "+00:00")).timestamp() * 1000.0 + except Exception: + return None + if isinstance(value, (int, float)): + numeric = float(value) + if numeric > 1e16: + return numeric / 1_000_000.0 + if numeric > 1e13: + return numeric / 1000.0 + if numeric > 1e10: + return numeric + if numeric > 0: + return numeric * 1000.0 + return None + + +def _collect_spans(raw: Any, out: List[Dict[str, Any]]) -> None: + if isinstance(raw, dict): + if any(k in raw for k in ("spanId", "span_id", "id")) and any(k in raw for k in ("name", "operationName")): + out.append(raw) + for value in raw.values(): + _collect_spans(value, out) + elif isinstance(raw, list): + for item in raw: + _collect_spans(item, out) + + +def extract_trace_id(otlp_payload: Dict[str, Any]) -> Optional[str]: + spans: List[Dict[str, Any]] = [] + _collect_spans(otlp_payload, spans) + for span in spans: + trace_id = span.get("traceId") or span.get("trace_id") + if trace_id: + return str(trace_id) + return None + + +def normalize_elevenlabs_otlp( + otlp_payload: Dict[str, Any], + *, + conversation_id: str, + fallback_trace_id: Optional[str] = None, +) -> Dict[str, Any]: + raw_spans: List[Dict[str, Any]] = [] + _collect_spans(otlp_payload, raw_spans) + + trace_id = extract_trace_id(otlp_payload) or fallback_trace_id or conversation_id + root_span_id: Optional[str] = None + spans: List[Dict[str, Any]] = [] + + for raw in raw_spans: + span_id = raw.get("span_id") or raw.get("spanId") or raw.get("id") + parent_span_id = raw.get("parent_span_id") or raw.get("parentSpanId") + + start_ms = _to_epoch_ms( + raw.get("start_time") + or raw.get("startTime") + or raw.get("startTimeUnixNano") + or raw.get("start_time_unix_nano") + ) + end_ms = _to_epoch_ms( + raw.get("end_time") + or raw.get("endTime") + or raw.get("endTimeUnixNano") + or raw.get("end_time_unix_nano") + ) + + duration_ms = raw.get("duration_ms") + if duration_ms is None: + duration = raw.get("duration") + if isinstance(duration, (int, float)): + duration_ms = duration / 1_000_000.0 if duration > 1e10 else float(duration) + elif start_ms is not None and end_ms is not None: + duration_ms = max(end_ms - start_ms, 0.0) + + attrs = _normalize_span_attributes(raw.get("attributes") or raw.get("tags")) + attrs.setdefault("trace.provider", "elevenlabs") + attrs.setdefault("elevenlabs.conversation_id", conversation_id) + + status_obj = raw.get("status") + if isinstance(status_obj, dict): + status_value = status_obj.get("code") or status_obj.get("status_code") + else: + status_value = status_obj + + spans.append({ + "span_id": span_id, + "parent_span_id": parent_span_id, + "name": raw.get("name") or raw.get("operationName") or "unknown", + "start_time": start_ms, + "end_time": end_ms, + "duration_ms": duration_ms, + "attributes": attrs, + "status": str(status_value) if status_value is not None else None, + }) + + if not parent_span_id and span_id and root_span_id is None: + root_span_id = span_id + + if root_span_id is None and spans: + root_span_id = spans[0].get("span_id") + + return { + "trace_id": trace_id, + "root_span_id": root_span_id, + "spans": spans, + "trace_source": "elevenlabs", + } + + +def enrich_with_turn_metrics(trace_payload: Dict[str, Any], transcript: List[Dict[str, Any]]) -> Dict[str, Any]: + """Attach synthetic elevenlabs.metric.* spans derived from conversation_turn_metrics.""" + spans = list(trace_payload.get("spans") or []) + if not spans or not transcript: + return trace_payload + + by_turn = [s for s in spans if isinstance(s.get("name"), str) and s.get("name", "").startswith("elevenlabs.recv.")] + if not by_turn: + return trace_payload + + metric_spans: List[Dict[str, Any]] = [] + for idx, entry in enumerate(transcript): + if not isinstance(entry, dict): + continue + metrics = entry.get("conversation_turn_metrics") + if not isinstance(metrics, dict): + continue + nested_metrics = metrics.get("metrics") if isinstance(metrics.get("metrics"), dict) else {} + parent = by_turn[idx] if idx < len(by_turn) else None + if not parent: + continue + base_start = parent.get("start_time") + base_end = parent.get("end_time") + if not isinstance(base_start, (int, float)): + continue + + def _first_elapsed_ms(metric_keys: List[str]) -> Optional[float]: + for metric_key in metric_keys: + metric_value = nested_metrics.get(metric_key) + if not isinstance(metric_value, dict): + continue + elapsed_s = metric_value.get("elapsed_time") + if isinstance(elapsed_s, (int, float)): + return float(elapsed_s) * 1000.0 + return None + + asr_ms = _first_elapsed_ms( + [ + "convai_turn_asr_latency", + "convai_asr_trailing_service_latency", + ] + ) + llm_ms = _first_elapsed_ms( + [ + "convai_llm_service_tt_last_sentence", + "convai_llm_service_ttf_sentence", + "convai_llm_service_ttfb", + ] + ) + tts_ms = _first_elapsed_ms( + [ + "convai_tts_service_ttfb", + ] + ) + + entries = [ + ( + "elevenlabs.metric.asr", + asr_ms, + { + "convai_asr_provider": metrics.get("convai_asr_provider"), + "convai_metric_key": ( + "convai_turn_asr_latency" + if "convai_turn_asr_latency" in nested_metrics + else "convai_asr_trailing_service_latency" + ), + }, + ), + ( + "elevenlabs.metric.llm", + llm_ms, + { + "llm_usage": entry.get("llm_usage"), + "convai_metric_key": ( + "convai_llm_service_tt_last_sentence" + if "convai_llm_service_tt_last_sentence" in nested_metrics + else "convai_llm_service_ttf_sentence" + if "convai_llm_service_ttf_sentence" in nested_metrics + else "convai_llm_service_ttfb" + ), + }, + ), + ( + "elevenlabs.metric.tts", + tts_ms, + { + "convai_tts_model": metrics.get("convai_tts_model"), + "convai_tts_cascade": metrics.get("convai_tts_cascade"), + "convai_metric_key": "convai_tts_service_ttfb", + }, + ), + ] + for metric_name, elapsed_ms, extra in entries: + if elapsed_ms is None: + continue + end_time = base_start + elapsed_ms + if isinstance(base_end, (int, float)): + end_time = min(end_time, base_end) + attrs = {"trace.provider": "elevenlabs", **{k: v for k, v in extra.items() if v is not None}} + metric_spans.append({ + "span_id": f"{parent.get('span_id')}-{metric_name}", + "parent_span_id": parent.get("span_id"), + "name": metric_name, + "start_time": base_start, + "end_time": end_time, + "duration_ms": elapsed_ms, + "attributes": attrs, + "status": "1", + }) + + if metric_spans: + trace_payload = dict(trace_payload) + trace_payload["spans"] = spans + metric_spans + return trace_payload diff --git a/app/services/observability/live_event_emitter.py b/app/services/observability/live_event_emitter.py new file mode 100644 index 00000000..7d186769 --- /dev/null +++ b/app/services/observability/live_event_emitter.py @@ -0,0 +1,199 @@ +"""Emit incremental live observability events from Pipecat/voice pipelines.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime +from typing import Any, Callable, Dict, Optional +from uuid import UUID + +from loguru import logger + +from app.config import settings +from app.models.database import CallRecordingSource +from app.models.enums import CallRecordingStatus +from app.services.observability.call_ingest import upsert_live_event_call_recording +from app.services.observability.live_trace import build_live_synthetic_trace +from app.services.observability.trace_archive import persist_provider_trace + + +class LiveObservabilityEmitter: + """Push transcript turns and terminal metadata into live observability ingest.""" + + def __init__( + self, + *, + organization_id: UUID, + workspace_id: UUID, + provider_call_id: str, + provider_platform: str = "pipecat", + agent_ref: Optional[str] = None, + explicit_agent_id: Optional[UUID] = None, + trace_id: Optional[str] = None, + db_factory: Optional[Callable[[], Any]] = None, + ) -> None: + self.organization_id = organization_id + self.workspace_id = workspace_id + self.provider_call_id = provider_call_id + self.provider_platform = (provider_platform or "pipecat").strip().lower() + self.agent_ref = agent_ref + self.explicit_agent_id = explicit_agent_id + self.trace_id = trace_id + self._db_factory = db_factory + self._seq = 0 + self.call_short_id: Optional[str] = None + + @classmethod + def enabled(cls) -> bool: + return bool(settings.OBSERVABILITY_LIVE_INGEST_ENABLED) + + def start_call(self, *, direction: str = "inbound") -> Optional[str]: + if not self.enabled(): + return None + started_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") + ack = self._emit( + "call.started", + {"startedAt": started_at, "direction": direction, "status": "in_progress"}, + ) + self.call_short_id = ack.get("call_short_id") + if ack.get("trace_id") and not self.trace_id: + self.trace_id = str(ack["trace_id"]) + return self.call_short_id + + def emit_turn( + self, + role: str, + content: str, + *, + latency: Optional[Dict[str, Any]] = None, + start_time: Optional[float] = None, + end_time: Optional[float] = None, + ) -> None: + if not self.enabled() or not content.strip(): + return + normalized_role = (role or "").strip().lower() + if normalized_role in {"assistant", "agent", "bot"}: + event_type = "turn.assistant" + payload_role = "assistant" + else: + event_type = "turn.user" + payload_role = "user" + payload: Dict[str, Any] = {"content": content.strip(), "role": payload_role} + if isinstance(latency, dict) and latency: + payload["latency"] = latency + if start_time is not None: + payload["start_time"] = start_time + if end_time is not None: + payload["end_time"] = end_time + self._emit(event_type, payload) + + def end_call( + self, + *, + recording_url: Optional[str] = None, + recording_s3_key: Optional[str] = None, + duration_seconds: Optional[float] = None, + trace_id: Optional[str] = None, + ended_reason: Optional[str] = None, + ) -> None: + if not self.enabled(): + return + if trace_id: + self.trace_id = trace_id + ended_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") + payload: Dict[str, Any] = {"endedAt": ended_at, "status": "ended"} + if ended_reason: + payload["endedReason"] = ended_reason + if recording_url: + payload["recording_url"] = recording_url + if recording_s3_key: + payload["recording_s3_key"] = recording_s3_key + if duration_seconds is not None: + payload["duration_seconds"] = float(duration_seconds) + if self.trace_id: + payload["trace_id"] = self.trace_id + self._emit("call.ended", payload) + + def _emit(self, event_type: str, payload: Dict[str, Any]) -> Dict[str, Any]: + self._seq += 1 + live_event = { + "event_id": f"live-{self.provider_call_id}-{self._seq}-{uuid.uuid4().hex[:8]}", + "call_id": self.provider_call_id, + "event_type": event_type, + "seq": self._seq, + "event_ts": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "platform": self.provider_platform, + "payload": payload, + } + if self.trace_id: + live_event["trace_id"] = self.trace_id + if self.agent_ref: + live_event["agent_ref"] = self.agent_ref + + db = self._open_db() + close_db = self._db_factory is None + try: + call_recording, _action = upsert_live_event_call_recording( + db=db, + organization_id=self.organization_id, + workspace_id=self.workspace_id, + provider_platform=self.provider_platform, + provider_call_id=self.provider_call_id, + live_event=live_event, + max_out_of_order_seq=settings.OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ, + agent_ref_raw=self.agent_ref, + explicit_agent_id=self.explicit_agent_id, + source=CallRecordingSource.WEBHOOK, + persist=False, + ) + self.call_short_id = call_recording.call_short_id + if call_recording.trace_id: + self.trace_id = call_recording.trace_id + + if event_type in {"call.ended", "call.failed"}: + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + synthetic_trace = build_live_synthetic_trace( + call_data, + provider_call_id=str(call_recording.provider_call_id or self.provider_call_id), + provider_platform=self.provider_platform, + trace_id=str(call_recording.trace_id or self.trace_id or "") or None, + ) + if synthetic_trace: + call_recording.call_data = persist_provider_trace( + call_data=call_data, + provider_platform=self.provider_platform, + organization_id=self.organization_id, + call_short_id=call_recording.call_short_id, + trace_payload=synthetic_trace, + source=str(synthetic_trace.get("trace_source") or "live_synthetic"), + ) + if synthetic_trace.get("trace_id"): + call_recording.trace_id = str(synthetic_trace["trace_id"]) + call_recording.status = CallRecordingStatus.UPDATED + + db.commit() + db.refresh(call_recording) + return { + "call_short_id": call_recording.call_short_id, + "trace_id": call_recording.trace_id, + } + except Exception as exc: + db.rollback() + logger.warning( + "Live observability emit failed platform={} call_id={} event={}: {}", + self.provider_platform, + self.provider_call_id, + event_type, + exc, + ) + return {} + finally: + if close_db: + db.close() + + def _open_db(self): + if self._db_factory is not None: + return self._db_factory() + from app.database import SessionLocal + + return SessionLocal() diff --git a/app/services/observability/live_ingest.py b/app/services/observability/live_ingest.py new file mode 100644 index 00000000..e2fcc30c --- /dev/null +++ b/app/services/observability/live_ingest.py @@ -0,0 +1,180 @@ +"""Helpers for incremental live observability event ingest.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from typing import Any, Dict, Optional + + +class StaleLiveEventError(ValueError): + """Raised when an event falls outside the accepted out-of-order window.""" + + +def parse_live_event_ts(raw_value: Any) -> datetime: + """Parse live event timestamp to a UTC-aware datetime.""" + if isinstance(raw_value, datetime): + parsed = raw_value + elif isinstance(raw_value, str): + parsed = datetime.fromisoformat(raw_value.replace("Z", "+00:00")) + else: + raise ValueError("event_ts must be an ISO-8601 string") + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def derive_live_call_event(event_type: str) -> str: + normalized = (event_type or "").strip().lower() + if normalized == "call.started": + return "call_started" + if normalized in {"call.ended", "session.ended"}: + return "call_ended" + if normalized in {"call.failed", "session.failed"}: + return "call_failed" + if normalized.startswith("turn."): + return "call_in_progress" + return normalized.replace(".", "_") or "call_in_progress" + + +def _upsert_live_turn(turns: list[Dict[str, Any]], entry: Dict[str, Any]) -> None: + turn_id = entry.get("turn_id") + role = str(entry.get("role") or "").strip().lower() or "unknown" + replace_last_by_role = bool(entry.get("replace_last_by_role")) + + if turn_id: + for idx, item in enumerate(turns): + if str(item.get("turn_id")) == str(turn_id): + turns[idx] = entry + return + + if replace_last_by_role: + for idx in range(len(turns) - 1, -1, -1): + if str(turns[idx].get("role") or "").strip().lower() == role: + turns[idx] = entry + return + + turns.append(entry) + + +def _sync_messages_from_live_turns(turns: list[Dict[str, Any]]) -> list[Dict[str, Any]]: + messages: list[Dict[str, Any]] = [] + for turn in turns: + content = turn.get("content") + if not isinstance(content, str) or not content.strip(): + continue + messages.append( + { + "role": turn.get("role") or "unknown", + "content": content, + "timestamp": turn.get("event_ts"), + "start_time": turn.get("start_time"), + "end_time": turn.get("end_time"), + } + ) + return messages + + +def merge_live_event_call_data( + *, + existing_call_data: Dict[str, Any], + event: Dict[str, Any], + max_out_of_order_seq: int, +) -> Dict[str, Any]: + """Merge a live event into existing call_data while preserving prior keys.""" + merged = deepcopy(existing_call_data) if isinstance(existing_call_data, dict) else {} + payload = event.get("payload") + payload = payload if isinstance(payload, dict) else {} + event_type = str(event.get("event_type") or "").strip().lower() + seq = event.get("seq") + + live_state = merged.get("live_state") + if not isinstance(live_state, dict): + live_state = {} + watermark = live_state.get("max_seq") + if isinstance(watermark, (int, float)) and isinstance(seq, (int, float)): + if int(watermark) - int(seq) > max_out_of_order_seq: + raise StaleLiveEventError("event sequence is older than accepted out-of-order window") + + event_ts = str(event.get("event_ts")) + merged["_live_last_event_ts"] = event_ts + if isinstance(seq, (int, float)): + merged["_live_last_event_seq"] = int(seq) + if not isinstance(watermark, (int, float)) or int(seq) > int(watermark): + live_state["max_seq"] = int(seq) + live_state["last_event_type"] = event_type + live_state["last_platform"] = event.get("platform") + merged["live_state"] = live_state + + if event_type == "call.started": + merged["startedAt"] = payload.get("startedAt") or payload.get("started_at") or event_ts + merged["status"] = payload.get("status") or "in_progress" + elif event_type in {"call.ended", "call.failed", "session.ended", "session.failed"}: + merged["endedAt"] = payload.get("endedAt") or payload.get("ended_at") or event_ts + if payload.get("endedReason"): + merged["endedReason"] = payload.get("endedReason") + merged["status"] = payload.get("status") or ("failed" if "failed" in event_type else "ended") + for recording_key in ( + "recording_url", + "recording_s3_key", + "recording_multi_channel_url", + "recordingUrl", + ): + value = payload.get(recording_key) + if isinstance(value, str) and value.strip(): + merged[recording_key if recording_key != "recordingUrl" else "recording_url"] = value.strip() + recording_urls = payload.get("recording_urls") + if isinstance(recording_urls, dict) and recording_urls: + merged["recording_urls"] = recording_urls + if isinstance(payload.get("duration_seconds"), (int, float)): + merged["duration_seconds"] = float(payload.get("duration_seconds")) + provider_trace = payload.get("provider_trace") + if isinstance(provider_trace, dict) and provider_trace: + merged["provider_trace"] = provider_trace + elif isinstance(payload.get("otlp_traces"), dict) and payload.get("otlp_traces"): + merged["provider_trace"] = { + **(merged.get("provider_trace") if isinstance(merged.get("provider_trace"), dict) else {}), + "otlp_traces": payload.get("otlp_traces"), + "source": payload.get("trace_source") or event.get("platform"), + } + elif isinstance(payload.get("normalized_trace"), dict) and payload.get("normalized_trace"): + merged["provider_trace"] = { + **(merged.get("provider_trace") if isinstance(merged.get("provider_trace"), dict) else {}), + "normalized_trace": payload.get("normalized_trace"), + "source": payload.get("trace_source") or event.get("platform"), + } + elif event_type.startswith("turn."): + live_turns = merged.get("live_transcript") + if not isinstance(live_turns, list): + live_turns = [] + turn_entry = { + "turn_id": payload.get("turn_id"), + "role": payload.get("role") or event_type.split(".", 1)[1], + "content": payload.get("content") or payload.get("text") or "", + "event_ts": event_ts, + "seq": seq, + "start_time": payload.get("start_time"), + "end_time": payload.get("end_time"), + "replace_last_by_role": bool(payload.get("replace_last_by_role")), + } + if isinstance(payload.get("latency"), dict): + turn_entry["latency"] = payload.get("latency") + _upsert_live_turn(live_turns, turn_entry) + merged["live_transcript"] = live_turns + merged["messages"] = _sync_messages_from_live_turns(live_turns) + merged["status"] = payload.get("status") or "in_progress" + + # Keep a small event ledger in payload for easier debugging. + recent_events = merged.get("live_recent_events") + if not isinstance(recent_events, list): + recent_events = [] + recent_events.append( + { + "event_id": event.get("event_id"), + "event_type": event_type, + "seq": seq, + "event_ts": event_ts, + } + ) + merged["live_recent_events"] = recent_events[-30:] + return merged diff --git a/app/services/observability/live_latency.py b/app/services/observability/live_latency.py new file mode 100644 index 00000000..08d8e414 --- /dev/null +++ b/app/services/observability/live_latency.py @@ -0,0 +1,137 @@ +"""Live latency sample capture and rolling percentile helpers.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any, Dict, Iterable, List, Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.models.database import CallRecording, ObservabilityLiveLatencySample + + +def _extract_latency_samples(payload: Dict[str, Any]) -> List[Dict[str, float]]: + samples: List[Dict[str, float]] = [] + + direct_latency = payload.get("latency") + if isinstance(direct_latency, dict): + for key, value in direct_latency.items(): + if isinstance(value, (int, float)): + metric = f"{str(key).strip().lower()}_ms" + samples.append({"metric_name": metric, "latency_ms": float(value)}) + + metric_name = payload.get("latency_metric") + latency_ms = payload.get("latency_ms") + if isinstance(metric_name, str) and isinstance(latency_ms, (int, float)): + samples.append( + {"metric_name": f"{metric_name.strip().lower()}_ms", "latency_ms": float(latency_ms)} + ) + + totals = payload.get("metrics") + if isinstance(totals, dict): + for key, value in totals.items(): + if isinstance(value, (int, float)) and "latency" in str(key).lower(): + samples.append({"metric_name": str(key).strip().lower(), "latency_ms": float(value)}) + + return [item for item in samples if item["latency_ms"] >= 0] + + +def record_live_latency_samples( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + provider_platform: str, + provider_call_id: str, + event_payload: Dict[str, Any], + event_ts: datetime, +) -> int: + """Persist latency samples parsed from a live event payload.""" + extracted = _extract_latency_samples(event_payload) + if not extracted: + return 0 + for sample in extracted: + db.add( + ObservabilityLiveLatencySample( + organization_id=organization_id, + workspace_id=workspace_id, + call_recording_id=call_recording.id, + call_short_id=call_recording.call_short_id, + provider_platform=provider_platform, + provider_call_id=provider_call_id, + agent_id=call_recording.agent_id, + metric_name=sample["metric_name"], + latency_ms=sample["latency_ms"], + event_ts=event_ts.astimezone(UTC), + ) + ) + db.flush() + return len(extracted) + + +def _percentile(sorted_values: List[float], q: float) -> Optional[float]: + if not sorted_values: + return None + if len(sorted_values) == 1: + return sorted_values[0] + rank = (len(sorted_values) - 1) * q + lo = int(rank) + hi = min(lo + 1, len(sorted_values) - 1) + weight = rank - lo + return sorted_values[lo] * (1 - weight) + sorted_values[hi] * weight + + +def _build_percentiles(values: Iterable[float]) -> Dict[str, Optional[float]]: + sorted_values = sorted(float(v) for v in values) + return { + "p50_ms": _percentile(sorted_values, 0.50), + "p90_ms": _percentile(sorted_values, 0.90), + "p95_ms": _percentile(sorted_values, 0.95), + } + + +def query_live_latency_metrics( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + window_seconds: int, + agent_id: Optional[UUID] = None, + provider_platform: Optional[str] = None, +) -> Dict[str, Any]: + """Compute rolling latency percentiles for a time window.""" + cutoff = datetime.now(UTC) - timedelta(seconds=window_seconds) + query = db.query(ObservabilityLiveLatencySample).filter( + ObservabilityLiveLatencySample.organization_id == organization_id, + ObservabilityLiveLatencySample.workspace_id == workspace_id, + ObservabilityLiveLatencySample.event_ts >= cutoff, + ) + if agent_id: + query = query.filter(ObservabilityLiveLatencySample.agent_id == agent_id) + if provider_platform: + query = query.filter( + ObservabilityLiveLatencySample.provider_platform == provider_platform.strip().lower() + ) + + rows = query.all() + all_values = [row.latency_ms for row in rows] + + by_metric: Dict[str, List[float]] = {} + for row in rows: + by_metric.setdefault(row.metric_name, []).append(row.latency_ms) + + metric_payload = { + metric_name: { + **_build_percentiles(values), + "sample_count": len(values), + } + for metric_name, values in by_metric.items() + } + return { + "window_seconds": window_seconds, + "sample_count": len(all_values), + **_build_percentiles(all_values), + "metrics": metric_payload, + } diff --git a/app/services/observability/live_slo.py b/app/services/observability/live_slo.py new file mode 100644 index 00000000..33e88a65 --- /dev/null +++ b/app/services/observability/live_slo.py @@ -0,0 +1,83 @@ +"""Live SLO evaluation + automation hook helpers.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Optional +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.config import settings +from app.models.database import ( + CallRecording, + ObservabilityLiveLatencySample, + ObservabilityLiveSloBreach, +) +from app.services.observability.live_latency import query_live_latency_metrics + + +def evaluate_llm_p90_slo( + *, + db: Session, + organization_id: UUID, + workspace_id: UUID, + call_recording: CallRecording, + provider_platform: str, + window_seconds: int = 300, +) -> Optional[ObservabilityLiveSloBreach]: + """Evaluate rolling LLM p90 and persist a breach marker when violated.""" + if not settings.OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED: + return None + + metrics = query_live_latency_metrics( + db=db, + organization_id=organization_id, + workspace_id=workspace_id, + window_seconds=window_seconds, + agent_id=call_recording.agent_id, + provider_platform=provider_platform, + ) + llm_metrics = metrics.get("metrics", {}).get("llm_ms") or metrics.get("metrics", {}).get("llm_latency_ms") + if not isinstance(llm_metrics, dict): + return None + sample_count = int(llm_metrics.get("sample_count") or 0) + p90_ms = llm_metrics.get("p90_ms") + if sample_count < settings.OBSERVABILITY_LIVE_SLO_MIN_SAMPLE_COUNT or not isinstance(p90_ms, (int, float)): + return None + threshold = float(settings.OBSERVABILITY_LIVE_SLO_P90_LLM_MS) + if float(p90_ms) <= threshold: + return None + + cooldown_cutoff = datetime.now(UTC) - timedelta(minutes=10) + recent = ( + db.query(ObservabilityLiveSloBreach) + .filter( + ObservabilityLiveSloBreach.organization_id == organization_id, + ObservabilityLiveSloBreach.workspace_id == workspace_id, + ObservabilityLiveSloBreach.agent_id == call_recording.agent_id, + ObservabilityLiveSloBreach.metric_name == "llm_ms", + ObservabilityLiveSloBreach.created_at >= cooldown_cutoff, + ) + .first() + ) + if recent: + return None + + breach = ObservabilityLiveSloBreach( + organization_id=organization_id, + workspace_id=workspace_id, + call_recording_id=call_recording.id, + call_short_id=call_recording.call_short_id, + provider_platform=provider_platform, + agent_id=call_recording.agent_id, + metric_name="llm_ms", + window_seconds=window_seconds, + p90_ms=float(p90_ms), + threshold_ms=threshold, + sample_count=sample_count, + evaluator_queued=False, + ) + db.add(breach) + db.flush() + return breach diff --git a/app/services/observability/live_trace.py b/app/services/observability/live_trace.py new file mode 100644 index 00000000..529bbbbe --- /dev/null +++ b/app/services/observability/live_trace.py @@ -0,0 +1,217 @@ +"""Build synthetic traces from incremental live observability call_data.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from app.services.observability.vapi_trace import _first_number, _normalize_role, _to_epoch_ms + + +_LIVE_LAYER_KEYS = { + "stt": ("stt_ms", "stt", "asr_ms", "asr", "transcriber_ms", "transcriber"), + "llm": ("llm_ms", "llm", "model_ms", "model"), + "tts": ("tts_ms", "tts", "voice_ms", "voice"), +} + + +def _latency_from_mapping(raw: Any) -> Dict[str, float]: + if not isinstance(raw, dict): + return {} + resolved: Dict[str, float] = {} + for layer, keys in _LIVE_LAYER_KEYS.items(): + value = _first_number(*[raw.get(key) for key in keys]) + if value is not None: + resolved[layer] = value + return resolved + + +def _extract_live_turns(call_data: Dict[str, Any]) -> List[Dict[str, Any]]: + live_transcript = call_data.get("live_transcript") + if isinstance(live_transcript, list) and live_transcript: + return [entry for entry in live_transcript if isinstance(entry, dict)] + + messages = call_data.get("messages") + if isinstance(messages, list) and messages: + turns: List[Dict[str, Any]] = [] + for entry in messages: + if not isinstance(entry, dict): + continue + content = entry.get("content") or entry.get("text") or entry.get("message") + if not isinstance(content, str) or not content.strip(): + continue + turns.append( + { + "role": entry.get("role"), + "content": content, + "event_ts": entry.get("timestamp") or entry.get("event_ts"), + "latency": entry.get("latency"), + } + ) + return turns + return [] + + +def _turn_window_ms( + entry: Dict[str, Any], + *, + started_at_ms: float, + fallback_start_ms: float, +) -> tuple[float, float]: + event_ts = _to_epoch_ms(entry.get("event_ts") or entry.get("timestamp")) + if event_ts is not None: + end_ts = event_ts + 800.0 + return event_ts, end_ts + + start_offset = _first_number(entry.get("start_time"), entry.get("start")) + if start_offset is not None: + start_ms = start_offset if start_offset > 1e10 else started_at_ms + start_offset * 1000.0 + end_offset = _first_number(entry.get("end_time"), entry.get("end")) + if end_offset is not None: + end_ms = end_offset if end_offset > 1e10 else started_at_ms + end_offset * 1000.0 + else: + end_ms = start_ms + 800.0 + return start_ms, end_ms + + return fallback_start_ms, fallback_start_ms + 800.0 + + +def _average_layer_latencies(turns: List[Dict[str, Any]]) -> Dict[str, float]: + buckets: Dict[str, List[float]] = {"stt": [], "llm": [], "tts": []} + for turn in turns: + role = _normalize_role(turn.get("role")) + layer_latencies = _latency_from_mapping(turn.get("latency")) + if role == "user" and "stt" in layer_latencies: + buckets["stt"].append(layer_latencies["stt"]) + if role == "agent": + for layer in ("llm", "tts"): + if layer in layer_latencies: + buckets[layer].append(layer_latencies[layer]) + return { + layer: sum(values) / len(values) + for layer, values in buckets.items() + if values + } + + +def build_live_synthetic_trace( + call_data: Dict[str, Any], + *, + provider_call_id: str, + provider_platform: str = "external", + trace_id: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Convert live-ingest call_data into a synthetic STT/LLM/TTS trace tree.""" + if not isinstance(call_data, dict): + return None + + turns = _extract_live_turns(call_data) + if not turns: + return None + + platform = (provider_platform or call_data.get("live_state", {}).get("last_platform") or "external").strip().lower() + started_at_ms = _to_epoch_ms(call_data.get("startedAt") or call_data.get("started_at")) + ended_at_ms = _to_epoch_ms(call_data.get("endedAt") or call_data.get("ended_at")) + + if started_at_ms is None: + started_at_ms = _to_epoch_ms(turns[0].get("event_ts")) or 0.0 + if ended_at_ms is None: + last_ts = _to_epoch_ms(turns[-1].get("event_ts")) + ended_at_ms = (last_ts + 800.0) if last_ts is not None else started_at_ms + max(1000.0, len(turns) * 1200.0) + + resolved_trace_id = trace_id or call_data.get("trace_id") or f"live-{provider_call_id}" + root_span_id = f"live-root-{provider_call_id}" + root_span = { + "span_id": root_span_id, + "parent_span_id": None, + "name": "conversation", + "start_time": started_at_ms, + "end_time": ended_at_ms, + "duration_ms": max(ended_at_ms - started_at_ms, 0.0), + "attributes": { + "trace.provider": platform, + "provider.call_id": provider_call_id, + "conversation.id": provider_call_id, + "conversation.type": "live_synthetic", + "call.status": call_data.get("status"), + }, + "status": "1", + } + + spans: List[Dict[str, Any]] = [root_span] + cursor_ms = started_at_ms + turn_span_ids: List[str] = [] + + for idx, entry in enumerate(turns): + role = _normalize_role(entry.get("role")) + text = entry.get("content") or entry.get("text") or "" + msg_start, msg_end = _turn_window_ms(entry, started_at_ms=started_at_ms, fallback_start_ms=cursor_ms) + cursor_ms = max(cursor_ms, msg_end) + + span_id = f"live-turn-{idx}" + turn_span_ids.append(span_id) + spans.append( + { + "span_id": span_id, + "parent_span_id": root_span_id, + "name": "turn", + "start_time": msg_start, + "end_time": msg_end, + "duration_ms": max(msg_end - msg_start, 0.0), + "attributes": { + "trace.provider": platform, + "turn.number": idx + 1, + "turn.role": role, + "turn.text_length": len(str(text)), + **({"turn.user_transcript": text} if role == "user" and isinstance(text, str) and text.strip() else {}), + **({"turn.agent_transcript": text} if role == "agent" and isinstance(text, str) and text.strip() else {}), + }, + "status": "1", + } + ) + + layer_latencies = _latency_from_mapping(entry.get("latency")) + base_start = msg_start + for layer, dur in layer_latencies.items(): + spans.append( + { + "span_id": f"live-turn-{idx}-metric-{layer}", + "parent_span_id": span_id, + "name": layer, + "start_time": base_start, + "end_time": base_start + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": platform, + "metric.layer": layer, + "metric.scope": "turn_reported", + "turn.index": idx, + }, + "status": "1", + } + ) + + call_level = _average_layer_latencies(turns) + for layer, dur in call_level.items(): + spans.append( + { + "span_id": f"live-metric-{layer}", + "parent_span_id": root_span_id, + "name": layer, + "start_time": started_at_ms, + "end_time": started_at_ms + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": platform, + "metric.layer": layer, + "metric.scope": "call_average", + }, + "status": "1", + } + ) + + return { + "trace_id": resolved_trace_id, + "root_span_id": root_span_id, + "spans": spans, + "trace_source": f"{platform}_live_synthetic", + } diff --git a/app/services/observability/provider_audio_proxy.py b/app/services/observability/provider_audio_proxy.py new file mode 100644 index 00000000..5bb62f52 --- /dev/null +++ b/app/services/observability/provider_audio_proxy.py @@ -0,0 +1,126 @@ +"""Provider-hosted audio proxy helpers for observability/playground playback.""" + +from __future__ import annotations + +from typing import Any, Dict, Iterator, Optional +from uuid import UUID + +from fastapi import HTTPException +from fastapi import status +from fastapi.responses import StreamingResponse +from loguru import logger +from sqlalchemy.orm import Session + +from app.core.encryption import decrypt_api_key +from app.models.database import Agent, CallRecording, Integration +from app.services.observability.recording_url_safety import ( + assert_elevenlabs_recording_url, + download_elevenlabs_recording_bytes, +) +from app.services.telephony.exotel_client import ExotelInvalidContentError + + +def resolve_elevenlabs_audio_url(call_data: Dict[str, Any]) -> Optional[str]: + recording_urls = call_data.get("recording_urls") + if isinstance(recording_urls, dict): + conversation_audio = recording_urls.get("conversation_audio") + if isinstance(conversation_audio, str) and conversation_audio.strip(): + return conversation_audio.strip() + for key in ("recording_url", "recordingUrl"): + value = call_data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def stream_elevenlabs_audio_proxy( + *, + db: Session, + organization_id: UUID, + call_recording: CallRecording, + call_data: Dict[str, Any], + filename_prefix: str = "call", +) -> StreamingResponse: + """Proxy ElevenLabs conversation audio by injecting provider auth header.""" + audio_url = resolve_elevenlabs_audio_url(call_data) + if not audio_url: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No recording URL available", + ) + + try: + assert_elevenlabs_recording_url(audio_url) + except ExotelInvalidContentError as exc: + logger.warning( + "Blocked ElevenLabs audio proxy for call_short_id={}: {}", + call_recording.call_short_id, + exc, + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Recording URL is not allowed for provider proxy", + ) from exc + + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Agent or integration not found", + ) + + integration = ( + db.query(Integration) + .filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + .first() + ) + if not integration: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Integration not found", + ) + + decrypted_key = decrypt_api_key(integration.api_key) + try: + audio_bytes, content_type = download_elevenlabs_recording_bytes( + audio_url, + api_key=decrypted_key, + ) + except ExotelInvalidContentError as exc: + logger.warning( + "Blocked ElevenLabs audio download for call_short_id={}: {}", + call_recording.call_short_id, + exc, + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Recording URL is not allowed for provider proxy", + ) from exc + except Exception as exc: + logger.warning( + "ElevenLabs audio fetch failed for call_short_id={}: {}", + call_recording.call_short_id, + exc, + ) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Failed to fetch ElevenLabs recording", + ) from exc + + def _iter_chunks(data: bytes, chunk_size: int = 8192) -> Iterator[bytes]: + for offset in range(0, len(data), chunk_size): + yield data[offset : offset + chunk_size] + + return StreamingResponse( + _iter_chunks(audio_bytes), + media_type=content_type, + headers={ + "Content-Disposition": ( + f'inline; filename="{filename_prefix}_{call_recording.call_short_id}.mp3"' + ), + }, + ) diff --git a/app/services/observability/provider_call_enrichment.py b/app/services/observability/provider_call_enrichment.py new file mode 100644 index 00000000..415641a1 --- /dev/null +++ b/app/services/observability/provider_call_enrichment.py @@ -0,0 +1,124 @@ +"""Helpers to normalize and enrich hosted-provider observability call payloads.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from sqlalchemy.orm import Session + +from app.models.database import Agent, CallRecording, Integration + + +def looks_like_retell_call_data(call_data: Dict[str, Any]) -> bool: + if not isinstance(call_data, dict): + return False + if call_data.get("provider_platform") == "retell": + return True + if call_data.get("call_id") and ( + isinstance(call_data.get("transcript_object"), list) + or isinstance(call_data.get("call_analysis"), dict) + or isinstance(call_data.get("latency"), dict) + or isinstance(call_data.get("call_cost"), dict) + or call_data.get("disconnection_reason") is not None + ): + return True + return False + + +def looks_like_vapi_call_data(call_data: Dict[str, Any]) -> bool: + if not isinstance(call_data, dict): + return False + if call_data.get("provider_platform") == "vapi": + return True + return bool( + call_data.get("assistantId") + or call_data.get("assistant_id") + or isinstance(call_data.get("artifact"), dict) + or call_data.get("endedReason") is not None + ) + + +def resolve_observability_provider_platform( + call_recording: CallRecording, + call_data: Optional[Dict[str, Any]] = None, + *, + db: Optional[Session] = None, +) -> str: + payload = call_data if isinstance(call_data, dict) else ( + call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + ) + stored = (call_recording.provider_platform or payload.get("provider_platform") or "").strip().lower() + if stored and stored not in {"external", "unknown"}: + return stored + + if looks_like_retell_call_data(payload): + return "retell" + if looks_like_vapi_call_data(payload): + return "vapi" + if (payload.get("provider_platform") or "").strip().lower() == "elevenlabs": + return "elevenlabs" + + if db is not None and call_recording.agent_id: + agent = ( + db.query(Agent) + .filter( + Agent.id == call_recording.agent_id, + Agent.organization_id == call_recording.organization_id, + Agent.workspace_id == call_recording.workspace_id, + ) + .first() + ) + if agent and agent.voice_ai_integration_id: + integration = ( + db.query(Integration) + .filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == call_recording.organization_id, + Integration.is_active == True, + ) + .first() + ) + if integration and integration.platform is not None: + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform) + ) + return platform_value.strip().lower() + + return stored or "external" + + +def is_sparse_provider_call_data(call_data: Dict[str, Any], provider_platform: str) -> bool: + platform = provider_platform.strip().lower() + has_transcript = bool( + (isinstance(call_data.get("transcript_object"), list) and len(call_data.get("transcript_object")) > 0) + or (isinstance(call_data.get("messages"), list) and len(call_data.get("messages")) > 0) + or (isinstance(call_data.get("transcript"), str) and call_data.get("transcript", "").strip()) + or (isinstance(call_data.get("transcript"), list) and len(call_data.get("transcript")) > 0) + ) + + if platform == "retell": + has_analysis = isinstance(call_data.get("call_analysis"), dict) and len(call_data.get("call_analysis")) > 0 + has_cost = isinstance(call_data.get("call_cost"), dict) and len(call_data.get("call_cost")) > 0 + has_latency = isinstance(call_data.get("latency"), dict) and len(call_data.get("latency")) > 0 + if not has_transcript: + return True + return not (has_analysis and has_cost and has_latency) + + if platform == "vapi": + has_messages = bool( + (isinstance(call_data.get("messages"), list) and len(call_data.get("messages")) > 0) + or ( + isinstance(call_data.get("artifact"), dict) + and isinstance(call_data.get("artifact", {}).get("messages"), list) + and len(call_data.get("artifact", {}).get("messages")) > 0 + ) + ) + has_analysis = isinstance(call_data.get("analysis"), dict) and len(call_data.get("analysis")) > 0 + has_cost = call_data.get("cost") is not None or isinstance(call_data.get("costBreakdown"), dict) + if not has_messages: + return True + return not (has_analysis and has_cost) + + return False diff --git a/app/services/observability/recording_archive.py b/app/services/observability/recording_archive.py new file mode 100644 index 00000000..a71ef921 --- /dev/null +++ b/app/services/observability/recording_archive.py @@ -0,0 +1,167 @@ +"""Download hosted-provider call recordings into S3 for observability playback.""" + +from __future__ import annotations + +import uuid +from typing import Any, Dict, Optional, Tuple +from uuid import UUID + +from loguru import logger + +from app.services.audio.voice_quality_service import get_recording_url +from app.services.observability.recording_url_safety import ( + build_elevenlabs_conversation_audio_url, + download_elevenlabs_recording_bytes, +) +from app.services.storage.s3_service import s3_service +from app.services.telephony.exotel_client import ExotelInvalidContentError +from app.services.telephony.recording_download import download_recording_url + + +def resolve_observability_recording_url( + call_data: Dict[str, Any], + provider_platform: str, +) -> Optional[str]: + """Resolve a downloadable recording URL from normalized provider call_data.""" + platform = (provider_platform or call_data.get("provider_platform") or "").strip().lower() + url = get_recording_url(call_data, platform) + if url: + return str(url).strip() or None + + if platform == "elevenlabs": + recording_urls = call_data.get("recording_urls") + if isinstance(recording_urls, dict): + conversation_audio = recording_urls.get("conversation_audio") + if isinstance(conversation_audio, str) and conversation_audio.strip(): + return conversation_audio.strip() + + raw_data = call_data.get("raw_data") + if isinstance(raw_data, dict) and raw_data.get("has_audio"): + conversation_id = ( + call_data.get("conversation_id") + or call_data.get("call_id") + or raw_data.get("conversation_id") + ) + if conversation_id: + try: + return build_elevenlabs_conversation_audio_url(str(conversation_id)) + except ExotelInvalidContentError: + return None + + if platform == "retell": + multi_channel = call_data.get("recording_multi_channel_url") + if isinstance(multi_channel, str) and multi_channel.strip(): + return multi_channel.strip() + + return None + + +def _download_recording_bytes( + recording_url: str, + *, + provider_platform: str, + provider_api_key: Optional[str] = None, +) -> Tuple[bytes, str]: + platform = provider_platform.strip().lower() + if platform == "elevenlabs": + if not provider_api_key: + raise ExotelInvalidContentError("ElevenLabs recording download requires an API key") + return download_elevenlabs_recording_bytes( + recording_url, + api_key=provider_api_key, + ) + + return download_recording_url( + recording_url, + timeout_seconds=120.0, + user_supplied=True, + ) + + +def _extension_for_content_type(content_type: str) -> str: + lowered = (content_type or "").lower() + if "wav" in lowered: + return "wav" + if "ogg" in lowered: + return "ogg" + if "webm" in lowered: + return "webm" + if "mpeg" in lowered or "mp3" in lowered: + return "mp3" + return "mp3" + + +def build_observability_recording_s3_key( + *, + organization_id: UUID, + call_short_id: str, + extension: str, +) -> str: + normalized_ext = extension.lstrip(".") or "mp3" + return ( + f"{s3_service.prefix}organizations/{organization_id}/observability/" + f"{call_short_id}/{uuid.uuid4()}.{normalized_ext}" + ) + + +def archive_observability_recording_to_s3( + *, + call_data: Dict[str, Any], + provider_platform: str, + organization_id: UUID, + call_short_id: str, + provider_api_key: Optional[str] = None, +) -> Dict[str, Any]: + """Ensure call_data contains recording_s3_key by archiving provider audio when possible.""" + if not isinstance(call_data, dict): + return call_data + + if call_data.get("recording_s3_key"): + return call_data + + if not s3_service.is_enabled(): + return call_data + + recording_url = resolve_observability_recording_url(call_data, provider_platform) + if not recording_url: + return call_data + + try: + audio_bytes, content_type = _download_recording_bytes( + recording_url, + provider_platform=provider_platform, + provider_api_key=provider_api_key, + ) + except Exception as exc: + logger.warning( + "Observability recording download failed for call_short_id={}: {}", + call_short_id, + exc, + ) + return call_data + + if not audio_bytes: + return call_data + + extension = _extension_for_content_type(content_type) + s3_key = build_observability_recording_s3_key( + organization_id=organization_id, + call_short_id=call_short_id, + extension=extension, + ) + + try: + s3_service.upload_file_by_key(audio_bytes, s3_key, content_type=content_type) + except Exception as exc: + logger.warning( + "Observability recording S3 upload failed for call_short_id={}: {}", + call_short_id, + exc, + ) + return call_data + + updated = dict(call_data) + updated["recording_s3_key"] = s3_key + updated.setdefault("recording_url", recording_url) + updated["recording_source"] = "provider_archive" + return updated diff --git a/app/services/observability/recording_url_safety.py b/app/services/observability/recording_url_safety.py new file mode 100644 index 00000000..0c454aeb --- /dev/null +++ b/app/services/observability/recording_url_safety.py @@ -0,0 +1,73 @@ +"""SSRF-safe recording URL validation for observability provider downloads.""" + +from __future__ import annotations + +import re +from typing import List, Tuple +from urllib.parse import urlparse + +import httpx + +from app.services.telephony.exotel_client import ExotelInvalidContentError +from app.services.telephony.recording_download import assert_recording_url_safe + +_ELEVENLABS_HOST_SUFFIXES: List[str] = ["elevenlabs.io"] +_CONVERSATION_ID_RE = re.compile(r"^conv_[A-Za-z0-9_]+$") + + +def build_elevenlabs_conversation_audio_url(conversation_id: str) -> str: + """Build a trusted ElevenLabs conversation audio URL from a provider call id.""" + normalized = str(conversation_id or "").strip() + if not _CONVERSATION_ID_RE.fullmatch(normalized): + raise ExotelInvalidContentError( + "ElevenLabs conversation id contains unexpected characters" + ) + return f"https://api.elevenlabs.io/v1/convai/conversations/{normalized}/audio" + + +def assert_elevenlabs_recording_url(recording_url: str) -> None: + """Reject non-ElevenLabs destinations before attaching provider credentials.""" + assert_recording_url_safe( + recording_url, + user_supplied=False, + allowed_suffixes=_ELEVENLABS_HOST_SUFFIXES, + ) + parsed = urlparse(recording_url.strip()) + path = parsed.path or "" + if not path.startswith("/v1/convai/conversations/") or not path.endswith("/audio"): + raise ExotelInvalidContentError( + "ElevenLabs recording URL path is not an allowed conversation audio endpoint" + ) + + +def download_elevenlabs_recording_bytes( + recording_url: str, + *, + api_key: str, + timeout_seconds: float = 120.0, +) -> Tuple[bytes, str]: + """Download ElevenLabs audio only after host/path validation.""" + assert_elevenlabs_recording_url(recording_url) + + def _validate_redirect(request: httpx.Request) -> None: + assert_recording_url_safe( + str(request.url), + user_supplied=False, + allowed_suffixes=_ELEVENLABS_HOST_SUFFIXES, + ) + parsed = urlparse(str(request.url)) + path = parsed.path or "" + if not path.startswith("/v1/convai/conversations/") or not path.endswith("/audio"): + raise ExotelInvalidContentError( + "ElevenLabs recording redirect target is not an allowed audio endpoint" + ) + + with httpx.Client( + timeout=timeout_seconds, + follow_redirects=True, + event_hooks={"request": [_validate_redirect]}, + ) as client: + response = client.get(recording_url, headers={"xi-api-key": api_key}) + response.raise_for_status() + content_type = response.headers.get("content-type", "audio/mpeg") + return response.content, content_type diff --git a/app/services/observability/retell_trace.py b/app/services/observability/retell_trace.py new file mode 100644 index 00000000..f05c53c5 --- /dev/null +++ b/app/services/observability/retell_trace.py @@ -0,0 +1,233 @@ +"""Build synthetic traces from Retell call report payloads.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from app.services.observability.vapi_trace import ( + _first_number, + _normalize_role, + _to_epoch_ms, + _to_float, +) + + +def _latency_p50(latency: Dict[str, Any], key: str) -> Optional[float]: + bucket = latency.get(key) + if isinstance(bucket, dict): + return _to_float(bucket.get("p50")) + return None + + +def _extract_transcript_turns(call_data: Dict[str, Any]) -> List[Dict[str, Any]]: + transcript_object = call_data.get("transcript_object") + if isinstance(transcript_object, list) and transcript_object: + return [entry for entry in transcript_object if isinstance(entry, dict)] + + messages = call_data.get("messages") + if isinstance(messages, list) and messages: + turns = [] + for entry in messages: + if not isinstance(entry, dict): + continue + role = entry.get("role") + content = entry.get("content") or entry.get("message") or entry.get("text") + if not content: + continue + turns.append({"role": role, "content": content, **entry}) + if turns: + return turns + + transcript_raw = call_data.get("transcript") + if isinstance(transcript_raw, list): + return [entry for entry in transcript_raw if isinstance(entry, dict)] + + if isinstance(transcript_raw, str) and transcript_raw.strip(): + turns: List[Dict[str, Any]] = [] + for line in transcript_raw.split("\n"): + stripped = line.strip() + if not stripped: + continue + lower = stripped.lower() + if lower.startswith("agent:"): + turns.append({"role": "agent", "content": stripped.split(":", 1)[1].strip()}) + elif lower.startswith("user:"): + turns.append({"role": "user", "content": stripped.split(":", 1)[1].strip()}) + if turns: + return turns + return [] + + +def _turn_window_ms( + entry: Dict[str, Any], + *, + started_at_ms: float, + fallback_start_ms: float, +) -> tuple[float, float]: + words = entry.get("words") + if isinstance(words, list) and words: + first_word = words[0] if isinstance(words[0], dict) else {} + last_word = words[-1] if isinstance(words[-1], dict) else {} + start_sec = _to_float(first_word.get("start")) + end_sec = _to_float(last_word.get("end")) + if start_sec is not None: + end_sec = end_sec if end_sec is not None else start_sec + return started_at_ms + start_sec * 1000.0, started_at_ms + end_sec * 1000.0 + + for key in ("start_time", "timestamp", "start"): + offset = _to_float(entry.get(key)) + if offset is None: + continue + start_ms = offset if offset > 1e10 else started_at_ms + offset * 1000.0 + end_offset = _first_number(entry.get("end_time"), entry.get("end")) + if end_offset is not None: + end_ms = end_offset if end_offset > 1e10 else started_at_ms + end_offset * 1000.0 + else: + end_ms = start_ms + 800.0 + return start_ms, end_ms + + return fallback_start_ms, fallback_start_ms + 800.0 + + +def build_retell_synthetic_trace( + call_data: Dict[str, Any], + *, + provider_call_id: str, +) -> Optional[Dict[str, Any]]: + """Convert Retell call report payload into a synthetic trace tree.""" + if not isinstance(call_data, dict): + return None + + turns = _extract_transcript_turns(call_data) + latency_stats = call_data.get("latency") if isinstance(call_data.get("latency"), dict) else {} + + has_any_signal = bool(turns) or bool(latency_stats) + if not has_any_signal: + return None + + started_at_ms = _to_epoch_ms( + call_data.get("start_timestamp") or call_data.get("startedAt") or call_data.get("started_at") + ) + ended_at_ms = _to_epoch_ms( + call_data.get("end_timestamp") or call_data.get("endedAt") or call_data.get("ended_at") + ) + duration_ms = _to_float(call_data.get("duration_ms")) + duration_seconds = _to_float(call_data.get("duration_seconds")) + + if started_at_ms is None: + started_at_ms = 0.0 + if ended_at_ms is None and duration_ms is not None: + ended_at_ms = started_at_ms + duration_ms + elif ended_at_ms is None and duration_seconds is not None: + ended_at_ms = started_at_ms + duration_seconds * 1000.0 + if ended_at_ms is None: + ended_at_ms = started_at_ms + max(1000.0, len(turns) * 1200.0) + + trace_id = f"retell-{provider_call_id}" + root_span_id = f"retell-root-{provider_call_id}" + root_span = { + "span_id": root_span_id, + "parent_span_id": None, + "name": "conversation", + "start_time": started_at_ms, + "end_time": ended_at_ms, + "duration_ms": max(ended_at_ms - started_at_ms, 0.0), + "attributes": { + "trace.provider": "retell", + "provider.call_id": provider_call_id, + "conversation.id": provider_call_id, + "conversation.type": "provider_synthetic", + "call.status": call_data.get("call_status") or call_data.get("status"), + }, + "status": "1", + } + + spans: List[Dict[str, Any]] = [root_span] + cursor_ms = started_at_ms + turn_span_ids: List[str] = [] + + for idx, entry in enumerate(turns): + role = _normalize_role(entry.get("role")) + text = entry.get("content") or entry.get("text") or "" + msg_start, msg_end = _turn_window_ms(entry, started_at_ms=started_at_ms, fallback_start_ms=cursor_ms) + cursor_ms = max(cursor_ms, msg_end) + + span_id = f"retell-turn-{idx}" + turn_span_ids.append(span_id) + spans.append( + { + "span_id": span_id, + "parent_span_id": root_span_id, + "name": "turn", + "start_time": msg_start, + "end_time": msg_end, + "duration_ms": max(msg_end - msg_start, 0.0), + "attributes": { + "trace.provider": "retell", + "turn.number": idx + 1, + "turn.role": role, + "turn.text_length": len(str(text)), + **({"turn.user_transcript": text} if role == "user" and isinstance(text, str) and text.strip() else {}), + **({"turn.agent_transcript": text} if role == "agent" and isinstance(text, str) and text.strip() else {}), + }, + "status": "1", + } + ) + + layer_candidates = { + "stt": _latency_p50(latency_stats, "asr"), + "llm": _latency_p50(latency_stats, "llm"), + "tts": _latency_p50(latency_stats, "tts"), + } + + for layer, dur in layer_candidates.items(): + if dur is None: + continue + spans.append( + { + "span_id": f"retell-metric-{layer}", + "parent_span_id": root_span_id, + "name": layer, + "start_time": started_at_ms, + "end_time": started_at_ms + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": "retell", + "metric.layer": layer, + "metric.scope": "call_average", + }, + "status": "1", + } + ) + + for idx, parent_span_id in enumerate(turn_span_ids): + turn_span = spans[idx + 1] + base_start = float(turn_span.get("start_time") or started_at_ms + idx * 1000.0) + for layer, dur in layer_candidates.items(): + if dur is None: + continue + spans.append( + { + "span_id": f"retell-turn-{idx}-metric-{layer}-estimated", + "parent_span_id": parent_span_id, + "name": layer, + "start_time": base_start, + "end_time": base_start + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": "retell", + "metric.layer": layer, + "metric.scope": "turn_estimated", + "turn.index": idx, + "metric.estimated": True, + }, + "status": "1", + } + ) + + return { + "trace_id": trace_id, + "root_span_id": root_span_id, + "spans": spans, + "trace_source": "retell_synthetic", + } diff --git a/app/services/observability/trace_archive.py b/app/services/observability/trace_archive.py new file mode 100644 index 00000000..1d0bfa80 --- /dev/null +++ b/app/services/observability/trace_archive.py @@ -0,0 +1,133 @@ +"""Persist provider traces inline and/or in S3 for durable observability.""" + +from __future__ import annotations + +import json +import uuid +from datetime import UTC, datetime +from typing import Any, Dict, Optional +from uuid import UUID + +from loguru import logger + +from app.services.storage.s3_service import s3_service + +INLINE_TRACE_SIZE_LIMIT_BYTES = 256 * 1024 + + +def _json_size_bytes(value: Any) -> int: + try: + return len(json.dumps(value, separators=(",", ":")).encode("utf-8")) + except Exception: + return 0 + + +def build_observability_trace_s3_key( + *, + organization_id: UUID, + call_short_id: str, +) -> str: + return ( + f"{s3_service.prefix}organizations/{organization_id}/observability/" + f"{call_short_id}/traces/{uuid.uuid4()}.json" + ) + + +def persist_provider_trace( + *, + call_data: Dict[str, Any], + provider_platform: str, + organization_id: UUID, + call_short_id: str, + trace_payload: Dict[str, Any], + source: str, + raw_payload: Optional[Dict[str, Any]] = None, + inline_limit_bytes: int = INLINE_TRACE_SIZE_LIMIT_BYTES, +) -> Dict[str, Any]: + """Persist provider trace metadata + normalized trace with optional S3 overflow.""" + if not isinstance(call_data, dict) or not isinstance(trace_payload, dict): + return call_data + + updated = dict(call_data) + trace_id = trace_payload.get("trace_id") or updated.get("trace_id") + normalized_trace = trace_payload + trace_source = trace_payload.get("trace_source") or source + provider_trace: Dict[str, Any] = { + "source": source, + "trace_source": trace_source, + "trace_id": trace_id, + "ingested_at": datetime.now(UTC).isoformat(), + "provider_platform": provider_platform, + "storage": "inline", + "normalized_trace": normalized_trace, + "otlp_traces": None, + "trace_s3_key": None, + } + + if isinstance(raw_payload, dict): + provider_trace["otlp_traces"] = raw_payload + + payload_size = _json_size_bytes(provider_trace) + should_archive = payload_size > max(1, inline_limit_bytes) + if should_archive and s3_service.is_enabled(): + s3_key = build_observability_trace_s3_key( + organization_id=organization_id, + call_short_id=call_short_id, + ) + archive_payload = { + "trace_payload": trace_payload, + "raw_payload": raw_payload, + "source": source, + "trace_source": trace_source, + } + try: + s3_service.upload_file_by_key( + json.dumps(archive_payload).encode("utf-8"), + s3_key, + content_type="application/json", + ) + provider_trace["storage"] = "s3" + provider_trace["trace_s3_key"] = s3_key + provider_trace["otlp_traces"] = None + except Exception as exc: + logger.warning( + "Provider trace S3 archive failed for call_short_id={}: {}", + call_short_id, + exc, + ) + + updated["provider_trace"] = provider_trace + if trace_id: + updated["trace_id"] = trace_id + return updated + + +def load_provider_trace(call_data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Load normalized provider trace from call_data or S3 archive.""" + if not isinstance(call_data, dict): + return None + provider_trace = call_data.get("provider_trace") + if not isinstance(provider_trace, dict): + return None + + normalized = provider_trace.get("normalized_trace") + if isinstance(normalized, dict) and isinstance(normalized.get("spans"), list): + return normalized + + s3_key = provider_trace.get("trace_s3_key") + if not isinstance(s3_key, str) or not s3_key.strip(): + return None + if not s3_service.is_enabled(): + return None + + try: + payload_bytes = s3_service.download_file_by_key(s3_key) + archive_payload = json.loads(payload_bytes.decode("utf-8")) + except Exception as exc: + logger.warning("Failed to load archived provider trace s3_key={}: {}", s3_key, exc) + return None + + trace_payload = archive_payload.get("trace_payload") + if isinstance(trace_payload, dict) and isinstance(trace_payload.get("spans"), list): + return trace_payload + return None diff --git a/app/services/observability/vapi_trace.py b/app/services/observability/vapi_trace.py new file mode 100644 index 00000000..556c7d10 --- /dev/null +++ b/app/services/observability/vapi_trace.py @@ -0,0 +1,323 @@ +"""Build synthetic traces from Vapi call report payloads.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional, Set + + +def _to_epoch_ms(value: Any) -> Optional[float]: + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if stripped.isdigit() or ( + stripped.replace(".", "", 1).isdigit() and stripped.count(".") <= 1 + ): + return _to_epoch_ms(float(stripped)) + try: + return datetime.fromisoformat(stripped.replace("Z", "+00:00")).timestamp() * 1000.0 + except Exception: + return None + if isinstance(value, (int, float)): + numeric = float(value) + if numeric > 1e16: # ns + return numeric / 1_000_000.0 + if numeric > 1e13: # us + return numeric / 1000.0 + if numeric > 1e10: # ms epoch + return numeric + if numeric > 1e3: # likely ms offset + return numeric + if numeric > 0: # likely seconds offset + return numeric * 1000.0 + return None + + +def _to_float(value: Any) -> Optional[float]: + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value.strip()) + except Exception: + return None + return None + + +def _first_number(*values: Any) -> Optional[float]: + for value in values: + parsed = _to_float(value) + if parsed is not None: + return parsed + return None + + +def _extract_messages(call_data: Dict[str, Any]) -> List[Dict[str, Any]]: + messages = call_data.get("messages") + if isinstance(messages, list) and messages: + return [m for m in messages if isinstance(m, dict)] + artifact = call_data.get("artifact") + if isinstance(artifact, dict) and isinstance(artifact.get("messages"), list): + return [m for m in artifact["messages"] if isinstance(m, dict)] + return [] + + +def _extract_latency_stats(call_data: Dict[str, Any]) -> Dict[str, Any]: + analysis = call_data.get("analysis") + if isinstance(analysis, dict): + stats = analysis.get("latency_stats") or analysis.get("latencyStats") + if isinstance(stats, dict): + return stats + artifact = call_data.get("artifact") + if isinstance(artifact, dict): + perf = artifact.get("performanceMetrics") + if isinstance(perf, dict): + return perf + return {} + + +def _normalize_role(raw_role: Any) -> str: + value = str(raw_role or "").lower().strip() + if value in {"assistant", "agent", "bot", "ai"}: + return "agent" + return "user" + + +def build_vapi_synthetic_trace( + call_data: Dict[str, Any], + *, + provider_call_id: str, +) -> Optional[Dict[str, Any]]: + """Convert Vapi call report payload into a synthetic trace tree.""" + if not isinstance(call_data, dict): + return None + + messages = [m for m in _extract_messages(call_data) if m.get("role") != "system"] + latency_stats = _extract_latency_stats(call_data) + turn_latencies = latency_stats.get("turn_latencies") or latency_stats.get("turnLatencies") + + has_any_signal = bool(messages) or isinstance(latency_stats, dict) and len(latency_stats) > 0 + if not has_any_signal: + return None + + started_at_ms = _to_epoch_ms(call_data.get("startedAt") or call_data.get("started_at")) + ended_at_ms = _to_epoch_ms(call_data.get("endedAt") or call_data.get("ended_at")) + duration_seconds = _to_float(call_data.get("duration_seconds")) + + if started_at_ms is None: + started_at_ms = _to_epoch_ms(messages[0].get("time") if messages else None) or 0.0 + if ended_at_ms is None and duration_seconds is not None: + ended_at_ms = started_at_ms + duration_seconds * 1000.0 + if ended_at_ms is None: + ended_at_ms = started_at_ms + max(1000.0, len(messages) * 1200.0) + + trace_id = f"vapi-{provider_call_id}" + root_span_id = f"vapi-root-{provider_call_id}" + root_span = { + "span_id": root_span_id, + "parent_span_id": None, + "name": "conversation", + "start_time": started_at_ms, + "end_time": ended_at_ms, + "duration_ms": max(ended_at_ms - started_at_ms, 0.0), + "attributes": { + "trace.provider": "vapi", + "provider.call_id": provider_call_id, + "conversation.id": provider_call_id, + "conversation.type": "provider_synthetic", + "call.status": call_data.get("status"), + }, + "status": "1", + } + + spans: List[Dict[str, Any]] = [root_span] + cursor_ms = started_at_ms + turn_span_ids: List[str] = [] + + for idx, msg in enumerate(messages): + role = _normalize_role(msg.get("role")) + text = msg.get("message") or msg.get("content") or "" + msg_start = ( + _to_epoch_ms(msg.get("time")) + or (_to_float(msg.get("secondsFromStart")) * 1000.0 + started_at_ms if _to_float(msg.get("secondsFromStart")) is not None else None) + or cursor_ms + ) + msg_end = _to_epoch_ms(msg.get("endTime")) + msg_duration = _to_float(msg.get("duration")) + if msg_end is None and msg_duration is not None: + msg_end = msg_start + msg_duration + if msg_end is None: + msg_end = msg_start + 800.0 + cursor_ms = max(cursor_ms, msg_end) + + span_id = f"vapi-turn-{idx}" + turn_span_ids.append(span_id) + spans.append( + { + "span_id": span_id, + "parent_span_id": root_span_id, + "name": "turn", + "start_time": msg_start, + "end_time": msg_end, + "duration_ms": max(msg_end - msg_start, 0.0), + "attributes": { + "trace.provider": "vapi", + "turn.number": idx + 1, + "turn.role": role, + "turn.text_length": len(str(text)), + **({"turn.user_transcript": text} if role == "user" and isinstance(text, str) and text.strip() else {}), + **({"turn.agent_transcript": text} if role == "agent" and isinstance(text, str) and text.strip() else {}), + }, + "status": "1", + } + ) + + layer_candidates = { + "stt": _first_number( + latency_stats.get("transcriber_latency_avg"), + latency_stats.get("transcriberLatencyAverage"), + latency_stats.get("transcriberLatency"), + latency_stats.get("asr"), + latency_stats.get("asrLatency"), + ), + "llm": _first_number( + latency_stats.get("model_latency_avg"), + latency_stats.get("modelLatencyAverage"), + latency_stats.get("modelLatency"), + latency_stats.get("llm"), + latency_stats.get("llmLatency"), + ), + "tts": _first_number( + latency_stats.get("voice_latency_avg"), + latency_stats.get("voiceLatencyAverage"), + latency_stats.get("voiceLatency"), + latency_stats.get("tts"), + latency_stats.get("ttsLatency"), + ), + "endpointing": _first_number( + latency_stats.get("endpointing_latency_avg"), + latency_stats.get("endpointingLatencyAverage"), + latency_stats.get("endpointingLatency"), + ), + } + + # Always include call-level layer spans so trace stats/icons remain populated + # even if turnLatencies entries are sparse or shaped differently. + for layer, dur in layer_candidates.items(): + if dur is None: + continue + spans.append( + { + "span_id": f"vapi-metric-{layer}", + "parent_span_id": root_span_id, + "name": layer, + "start_time": started_at_ms, + "end_time": started_at_ms + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": "vapi", + "metric.layer": layer, + "metric.scope": "call_average", + }, + "status": "1", + } + ) + + if isinstance(turn_latencies, list): + turns_with_layer_metrics: Set[int] = set() + for idx, entry in enumerate(turn_latencies): + parent_span_id = turn_span_ids[idx] if idx < len(turn_span_ids) else root_span_id + base_start = started_at_ms + idx * 1000.0 + if isinstance(entry, (int, float)): + entries = {"llm": float(entry)} + elif isinstance(entry, dict): + entries = { + "stt": _first_number( + entry.get("transcriber"), + entry.get("asr"), + entry.get("transcriberLatency"), + entry.get("asrLatency"), + ), + "llm": _first_number( + entry.get("model"), + entry.get("llm"), + entry.get("modelLatency"), + entry.get("llmLatency"), + ), + "tts": _first_number( + entry.get("voice"), + entry.get("tts"), + entry.get("voiceLatency"), + entry.get("ttsLatency"), + ), + "endpointing": _first_number( + entry.get("endpointing"), + entry.get("endpointingLatency"), + ), + } + else: + entries = {} + for layer, raw_value in entries.items(): + dur = _to_float(raw_value) + if dur is None: + continue + turns_with_layer_metrics.add(idx) + spans.append( + { + "span_id": f"vapi-turn-{idx}-metric-{layer}", + "parent_span_id": parent_span_id, + "name": layer, + "start_time": base_start, + "end_time": base_start + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": "vapi", + "metric.layer": layer, + "metric.scope": "turn_provider", + "turn.index": idx, + }, + "status": "1", + } + ) + + # Vapi often reports per-turn latencies for only a subset of turns. + # Backfill missing turns from call-level averages so later turns remain inspectable. + for idx in range(len(turn_span_ids)): + if idx in turns_with_layer_metrics: + continue + parent_span_id = turn_span_ids[idx] + turn_span = spans[idx + 1] if idx + 1 < len(spans) else None + base_start = ( + float(turn_span.get("start_time")) if isinstance(turn_span, dict) and isinstance(turn_span.get("start_time"), (int, float)) + else started_at_ms + idx * 1000.0 + ) + for layer, dur in layer_candidates.items(): + if dur is None: + continue + spans.append( + { + "span_id": f"vapi-turn-{idx}-metric-{layer}-estimated", + "parent_span_id": parent_span_id, + "name": layer, + "start_time": base_start, + "end_time": base_start + dur, + "duration_ms": dur, + "attributes": { + "trace.provider": "vapi", + "metric.layer": layer, + "metric.scope": "turn_estimated", + "turn.index": idx, + "metric.estimated": True, + }, + "status": "1", + } + ) + + return { + "trace_id": trace_id, + "root_span_id": root_span_id, + "spans": spans, + "trace_source": "vapi_synthetic", + } + diff --git a/app/services/storage/s3_service.py b/app/services/storage/s3_service.py index b8376a85..bc6d9f46 100644 --- a/app/services/storage/s3_service.py +++ b/app/services/storage/s3_service.py @@ -75,7 +75,8 @@ def _ensure_initialized(self): self.s3_client = boto3.client("s3", **s3_kwargs) - # Test connection by checking if bucket exists (non-blocking) + # Best-effort bucket probe. Some IAM policies allow GetObject but not + # HeadBucket; keep the client so downloads can still succeed. try: self.s3_client.head_bucket(Bucket=self.bucket_name) except ClientError as e: @@ -84,11 +85,11 @@ def _ensure_initialized(self): self._initialization_error = f"S3 bucket '{self.bucket_name}' does not exist" self.s3_client = None elif error_code == "403": - self._initialization_error = f"Access denied to S3 bucket '{self.bucket_name}'. Check credentials." - self.s3_client = None + self._initialization_error = ( + f"HeadBucket denied for '{self.bucket_name}' (GetObject may still work)" + ) else: - self._initialization_error = f"Failed to connect to S3 bucket: {str(e)}" - self.s3_client = None + self._initialization_error = f"S3 bucket probe failed: {str(e)}" except NoCredentialsError: self._initialization_error = "S3 credentials not found. Check your configuration." self.s3_client = None diff --git a/app/services/telephony/call_recording_lifecycle.py b/app/services/telephony/call_recording_lifecycle.py index f5121e36..02dda613 100644 --- a/app/services/telephony/call_recording_lifecycle.py +++ b/app/services/telephony/call_recording_lifecycle.py @@ -473,12 +473,64 @@ def ingest_carrier_recording_url( return s3_key +def register_live_recording_paths( + db: Session, + *, + call_short_id: str, + user_audio_path: str, + bot_audio_path: str, + sample_rate: int, +) -> None: + """Persist shared temp WAV paths so the API can serve partial live audio.""" + row = db.query(CallRecording).filter(CallRecording.call_short_id == call_short_id).first() + if not row: + return + data = _copy_call_data(row) + data["live_user_audio_path"] = user_audio_path + data["live_bot_audio_path"] = bot_audio_path + data["live_recording_sample_rate"] = sample_rate + _save_call_data(db, row, data) + + +def _estimate_turn_duration_sec(content: str) -> float: + word_count = len(content.split()) + return max(1.2, word_count * 0.35) + + +def _upsert_live_speaker_segment( + segments: list, + *, + role: str, + content: str, + start_time_sec: Optional[float], +) -> None: + speaker = "user" if role == "user" else "assistant" + start = max(0.0, start_time_sec) if start_time_sec is not None else None + if start is None: + if segments: + start = float(segments[-1].get("end") or segments[-1].get("end_time") or 0) + else: + start = 0.0 + end = start + _estimate_turn_duration_sec(content) + entry = { + "speaker": speaker, + "text": content, + "start": round(start, 2), + "end": round(end, 2), + } + if segments and segments[-1].get("speaker") == speaker: + segments[-1] = entry + else: + segments.append(entry) + + def append_live_transcript_turn( db: Session, *, call_short_id: str, role: str, content: str, + start_time_sec: Optional[float] = None, ) -> None: if not content.strip(): return @@ -487,28 +539,43 @@ def append_live_transcript_turn( return data = _copy_call_data(row) transcript = list(data.get("live_transcript") or []) + segments = list(data.get("speaker_segments") or []) normalized = content.strip() + turn_entry: Dict[str, Any] = { + "role": role, + "content": normalized, + "timestamp": _now_iso(), + } + if start_time_sec is not None: + turn_entry["start_time"] = round(start_time_sec, 2) + if transcript and transcript[-1].get("role") == role: last_content = transcript[-1].get("content") or "" if normalized == last_content: return if normalized.startswith(last_content): - transcript[-1] = { - "role": role, - "content": normalized, - "timestamp": _now_iso(), - } + prev_start = transcript[-1].get("start_time", start_time_sec) + transcript[-1] = turn_entry + _upsert_live_speaker_segment( + segments, + role=role, + content=normalized, + start_time_sec=start_time_sec if start_time_sec is not None else prev_start, + ) data["live_transcript"] = transcript + data["speaker_segments"] = segments _save_call_data(db, row, data) return - transcript.append( - { - "role": role, - "content": normalized, - "timestamp": _now_iso(), - } + + transcript.append(turn_entry) + _upsert_live_speaker_segment( + segments, + role=role, + content=normalized, + start_time_sec=start_time_sec, ) data["live_transcript"] = transcript + data["speaker_segments"] = segments _save_call_data(db, row, data) @@ -520,6 +587,7 @@ def persist_telephony_call_artifacts( transcript_text: Optional[str] = None, s3_key: Optional[str] = None, duration: Optional[float] = None, + trace_id: Optional[str] = None, ) -> Optional[CallRecording]: """Persist transcript and recording metadata when a telephony call ends.""" row = db.query(CallRecording).filter(CallRecording.call_short_id == call_short_id).first() @@ -564,8 +632,20 @@ def persist_telephony_call_artifacts( data.setdefault("pipeline_recording_s3_key", s3_key) if duration is not None: data["duration_seconds"] = duration + if trace_id: + data["trace_id"] = trace_id + row.trace_id = trace_id if not data.get("ended_at"): data["ended_at"] = _now_iso() + if not data.get("endedAt"): + data["endedAt"] = data["ended_at"] + if not data.get("startedAt") and data.get("started_at"): + data["startedAt"] = data["started_at"] + + row.call_event = "call_ended" + from app.models.database import CallRecordingStatus + + row.status = CallRecordingStatus.UPDATED _save_call_data(db, row, data) # region agent log diff --git a/app/services/telephony/live_recording.py b/app/services/telephony/live_recording.py new file mode 100644 index 00000000..30fd41ec --- /dev/null +++ b/app/services/telephony/live_recording.py @@ -0,0 +1,98 @@ +"""Read in-progress telephony WAV captures and build partial mono playback.""" + +from __future__ import annotations + +import io +import os +import struct +import wave +from typing import Tuple + +import numpy as np + + +def _parse_wav_pcm(raw: bytes) -> Tuple[np.ndarray, int]: + """Parse PCM mono samples from a possibly incomplete WAV file.""" + if len(raw) < 44 or raw[:4] != b"RIFF": + return np.array([], dtype=np.int16), 0 + + sample_rate = 0 + num_channels = 1 + bits_per_sample = 16 + data_start: int | None = None + + pos = 12 + while pos + 8 <= len(raw): + chunk_id = raw[pos : pos + 4] + chunk_size = struct.unpack("= 16: + num_channels = struct.unpack("= 16: + bits_per_sample = struct.unpack(" 1: + samples = samples.reshape(-1, num_channels).mean(axis=1).astype(np.int16) + return samples, sample_rate + + +def read_growing_wav_mono(path: str) -> Tuple[np.ndarray, int]: + """Read whatever PCM has been flushed to a growing WAV capture file.""" + if not path or not os.path.isfile(path): + return np.array([], dtype=np.int16), 0 + try: + with open(path, "rb") as handle: + raw = handle.read() + except OSError: + return np.array([], dtype=np.int16), 0 + return _parse_wav_pcm(raw) + + +def pcm_to_wav_bytes(samples: np.ndarray, sample_rate: int) -> bytes: + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(samples.astype(np.int16).tobytes()) + return buffer.getvalue() + + +def merge_live_tracks_mono(user_path: str, bot_path: str) -> Tuple[bytes, float, int]: + """Merge partial user/bot telephony tracks into a mono WAV for live playback.""" + user, sr_user = read_growing_wav_mono(user_path) + bot, sr_bot = read_growing_wav_mono(bot_path) + + sample_rate = sr_user or sr_bot or 24000 + if len(user) == 0 and len(bot) == 0: + return b"", 0.0, sample_rate + + if len(user) == 0: + mixed = bot + elif len(bot) == 0: + mixed = user + else: + target_len = max(len(user), len(bot)) + user_pad = np.pad(user, (0, target_len - len(user))) if len(user) < target_len else user[:target_len] + bot_pad = np.pad(bot, (0, target_len - len(bot))) if len(bot) < target_len else bot[:target_len] + mixed = ((user_pad.astype(np.int32) + bot_pad.astype(np.int32)) // 2).astype(np.int16) + + duration_sec = len(mixed) / float(sample_rate) if sample_rate > 0 else 0.0 + return pcm_to_wav_bytes(mixed, sample_rate), duration_sec, sample_rate diff --git a/app/services/telephony/platform_outbound_pool.py b/app/services/telephony/platform_outbound_pool.py index deb73d9d..5eb32cbc 100644 --- a/app/services/telephony/platform_outbound_pool.py +++ b/app/services/telephony/platform_outbound_pool.py @@ -184,7 +184,7 @@ def acquire_pool_slot(org_id: UUID) -> bool: if current >= max_concurrent: return False new_count = current + 1 - pipe.setex(key, ttl, json.dumps({"count": new_count})) + pipe.set(key, json.dumps({"count": new_count}), ex=ttl) pipe.execute() return True except redis.RedisError as exc: @@ -208,7 +208,7 @@ def release_pool_slot(org_id: UUID) -> None: if new_count == 0: _get_redis().delete(key) else: - _get_redis().setex(key, ttl, json.dumps({"count": new_count})) + _get_redis().set(key, json.dumps({"count": new_count}), ex=ttl) except redis.RedisError as exc: logger.warning("Redis unavailable for outbound pool release: %s", exc) _purge_expired_in_memory() diff --git a/app/services/telephony/vobiz_session.py b/app/services/telephony/vobiz_session.py index 498816bd..56190b4d 100644 --- a/app/services/telephony/vobiz_session.py +++ b/app/services/telephony/vobiz_session.py @@ -79,7 +79,11 @@ def create_call_session( ttl = max(int(ttl_seconds), 60) key = f"{_SESSION_PREFIX}{call_ref}" try: - _get_redis().setex(key, ttl, json.dumps(payload)) + client = _get_redis() + if hasattr(client, "set"): + client.set(key, json.dumps(payload), ex=ttl) + else: + client.setex(key, ttl, json.dumps(payload)) except redis.RedisError as exc: logger.warning("Redis unavailable for Vobiz session; using in-memory fallback: %s", exc) _purge_expired_in_memory() diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index b13b78b1..571b10e1 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -23,6 +23,8 @@ from app.core.encryption import decrypt_api_key from app.services.voice_providers import get_voice_provider from app.services.storage.s3_service import s3_service +from app.services.observability.elevenlabs_trace import extract_trace_id +from app.services.tracing.efficientai_otel import force_flush_tracing, setup_efficientai_tracing from app.services.testing.test_agent_simulation_prompt import ( build_persona_description_for_bridge, build_test_agent_system_prompt, @@ -333,6 +335,7 @@ async def update_status(new_status: str, event: str = None, error: str = None): status_db.close() try: + setup_efficientai_tracing(service_name="efficientai-test-agent-bridge") logger.info( f"[Bridge WebRTC] Starting WebRTC bridge for evaluator {evaluator_id}, " f"bridging to {provider_platform} call {call_id}" @@ -804,6 +807,7 @@ async def on_call_should_end(): await webrtc_bridge.disconnect() if test_agent: await test_agent.cleanup() + force_flush_tracing() logger.info("[Bridge WebRTC] Bridge cleanup completed") @@ -923,6 +927,21 @@ async def _poll_call_results( call_completed = True logger.info(f"[Bridge Poll] ✅ Call completed: status={call_status}") + if provider_platform == "elevenlabs" and hasattr(provider, "retrieve_conversation_trace"): + try: + trace_payload = provider.retrieve_conversation_trace(call_id) + if isinstance(trace_payload, dict) and isinstance(trace_payload.get("otlp_traces"), dict): + provider_trace_id = extract_trace_id(trace_payload["otlp_traces"]) + call_metrics["provider_trace"] = { + "source": "elevenlabs_get_conversation", + "otlp_traces": trace_payload["otlp_traces"], + "trace_id": provider_trace_id, + } + if provider_trace_id: + call_metrics["trace_id"] = provider_trace_id + except Exception as trace_err: + logger.warning(f"[Bridge Poll] Could not fetch ElevenLabs OTLP trace: {trace_err}") + # === Status: CALL_ENDED === result.status = EvaluatorResultStatus.CALL_ENDED.value result.call_event = "call_ended" diff --git a/app/services/tracing/__init__.py b/app/services/tracing/__init__.py new file mode 100644 index 00000000..f38c6bb1 --- /dev/null +++ b/app/services/tracing/__init__.py @@ -0,0 +1 @@ +"""Tracing service helpers.""" diff --git a/app/services/tracing/efficientai_otel.py b/app/services/tracing/efficientai_otel.py new file mode 100644 index 00000000..37b40efd --- /dev/null +++ b/app/services/tracing/efficientai_otel.py @@ -0,0 +1,164 @@ +"""EfficientAI OpenTelemetry bootstrap helpers.""" + +from __future__ import annotations + +import os +from typing import Dict, Optional + +from loguru import logger + +from app.config import settings + +try: + from opentelemetry import trace + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + + OTEL_AVAILABLE = True +except Exception: # pragma: no cover - optional dependency + OTEL_AVAILABLE = False + + +_INITIALIZED = False + + +def _normalized_sample_rate() -> float: + try: + value = float(settings.OBSERVABILITY_TRACING_SAMPLE_RATE) + except (TypeError, ValueError): + logger.warning( + "Invalid OBSERVABILITY_TRACING_SAMPLE_RATE {!r}; using 1.0", + settings.OBSERVABILITY_TRACING_SAMPLE_RATE, + ) + return 1.0 + if value < 0.0 or value > 1.0: + logger.warning("OBSERVABILITY_TRACING_SAMPLE_RATE out of range {}; clamping to [0,1]", value) + return min(1.0, max(0.0, value)) + return value + + +def _build_headers() -> Dict[str, str]: + api_key = settings.EFFICIENT_AI_API_KEY or os.getenv("EFFICIENT_AI_API_KEY") + agent_id = settings.EFFICIENT_AI_AGENT_ID or os.getenv("EFFICIENT_AI_AGENT_ID") + project_id = settings.EFFICIENT_AI_PROJECT_ID or os.getenv("EFFICIENT_AI_PROJECT_ID") + + headers: Dict[str, str] = {} + if api_key: + headers["x-efficient-ai-api-key"] = api_key + if agent_id: + headers["x-efficient-ai-agent-id"] = agent_id + elif project_id: + headers["x-efficient-ai-project-id"] = project_id + return headers + + +def _get_or_create_provider(service_name: str) -> Optional["TracerProvider"]: + if not OTEL_AVAILABLE: + return None + + provider = trace.get_tracer_provider() + if isinstance(provider, TracerProvider): + return provider + + resource = Resource.create( + { + "service.name": service_name, + "service.instance.id": os.getenv("HOSTNAME", "unknown"), + "deployment.environment": os.getenv("ENVIRONMENT", "development"), + } + ) + sample_rate = _normalized_sample_rate() + provider = TracerProvider( + resource=resource, + sampler=ParentBased(root=TraceIdRatioBased(sample_rate)), + ) + trace.set_tracer_provider(provider) + return provider + + +def setup_efficientai_tracing(service_name: str = "efficientai-voice-agent") -> bool: + """Initialize tracing once per process; safe to call repeatedly.""" + global _INITIALIZED + if _INITIALIZED: + return True + if not OTEL_AVAILABLE: + logger.warning("OpenTelemetry not available; tracing disabled") + return False + if not settings.OBSERVABILITY_TRACING_ENABLED: + return False + + provider = _get_or_create_provider(service_name) + if provider is None: + return False + + exporter_mode = (settings.OBSERVABILITY_TRACING_EXPORTER or "efficientai_http").strip().lower() + endpoint = settings.OTEL_EXPORTER_OTLP_ENDPOINT + headers = _build_headers() + + try: + if exporter_mode == "console": + provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + elif exporter_mode in {"efficientai_http", "tempo_http"}: + if not endpoint: + logger.warning("Tracing enabled but OTLP endpoint is empty") + return False + if "x-efficient-ai-api-key" not in headers and exporter_mode == "efficientai_http": + logger.warning("Tracing enabled but EFFICIENT_AI_API_KEY missing") + return False + exporter = OTLPSpanExporter(endpoint=endpoint, headers=headers) + provider.add_span_processor(BatchSpanProcessor(exporter)) + else: + logger.warning("Unsupported tracing exporter: {}", exporter_mode) + return False + except Exception as exc: + logger.warning("Failed to initialize tracing exporter: {}", exc) + return False + + _INITIALIZED = True + logger.info("Tracing initialized with exporter={}", exporter_mode) + return True + + +def force_flush_tracing(timeout_millis: int = 3000) -> None: + """Flush active span processors; best-effort no-op when unavailable.""" + if not OTEL_AVAILABLE: + return + provider = trace.get_tracer_provider() + if isinstance(provider, TracerProvider): + try: + provider.force_flush(timeout_millis=timeout_millis) + except Exception as exc: # pragma: no cover + logger.debug("Tracing force_flush failed: {}", exc) + + +def log_trace_export_status(trace_id: Optional[str]) -> None: + """Best-effort check that exported spans are queryable in the configured backend.""" + if not trace_id or not settings.OBSERVABILITY_TRACING_ENABLED: + return + backend = (settings.TRACING_QUERY_BACKEND or "cloud").strip().lower() + if backend != "tempo": + return + try: + import httpx + + base = settings.TEMPO_QUERY_URL.rstrip("/") + response = httpx.get(f"{base}/api/traces/{trace_id}", timeout=3.0) + if response.status_code == 404: + logger.warning( + "Trace {} was linked to the call but Tempo has no spans yet. " + "Check OTLP export to {} and Tempo retention.", + trace_id, + settings.OTEL_EXPORTER_OTLP_ENDPOINT, + ) + elif response.is_success: + logger.info("Trace {} confirmed in Tempo", trace_id) + except Exception as exc: + logger.warning( + "Could not verify trace {} in Tempo at {}: {}", + trace_id, + settings.TEMPO_QUERY_URL, + exc, + ) diff --git a/app/services/voice_agent/audio_recorder.py b/app/services/voice_agent/audio_recorder.py index 1599dcb9..28551160 100644 --- a/app/services/voice_agent/audio_recorder.py +++ b/app/services/voice_agent/audio_recorder.py @@ -75,6 +75,11 @@ def _write_audio(self, audio_to_write: bytes, num_channels: int) -> None: num_samples = len(audio_to_write) // (num_channels * 2) self.wave_file.writeframes(audio_to_write) self.total_samples_written += num_samples + if self.alignment_mode == "stream": + try: + self.wave_file._file.flush() + except Exception: + pass async def process_frame(self, frame, direction): await super().process_frame(frame, direction) diff --git a/app/services/voice_agent/bot_fast_api.py b/app/services/voice_agent/bot_fast_api.py index 3bcd92a7..32e83a81 100644 --- a/app/services/voice_agent/bot_fast_api.py +++ b/app/services/voice_agent/bot_fast_api.py @@ -17,6 +17,7 @@ from loguru import logger +from app.services.tracing.efficientai_otel import force_flush_tracing, setup_efficientai_tracing from app.services.voice_agent.audio_recorder import get_audio_recorder_class from app.services.voice_agent.utils.audio_merge import merge_and_upload_audio @@ -98,7 +99,7 @@ def _get_imports(): """ -async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str = None): +async def run_bot(websocket_client, google_api_key: str, system_instruction: str = None, organization_id: str = None, agent_id: str = None, persona_id: str = None, scenario_id: str = None, evaluator_id: str = None, result_id: str = None, model_name: str = None, serializer=None, telephony_mode: bool = False, call_short_id: str = None, silence_hangup_secs: float | None = None, workspace_id: str | None = None, live_observability_emitter=None): """ Run the voice agent bot with the provided Google API key. @@ -118,6 +119,7 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str duration_result = None transcript_text = None conversation_turns = [] + conversation_trace_id = None try: transport_serializer = serializer or imports["ProtobufFrameSerializer"]() @@ -173,6 +175,7 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str system_instruction=instruction, model=formatted_model_name, ) + setattr(llm, "_observability_bundle_type", "s2s") context = imports["LLMContext"]( [ { @@ -215,13 +218,45 @@ async def run_bot(websocket_client, google_api_key: str, system_instruction: str alignment_mode=recorder_alignment, ) + if telephony_mode and call_short_id: + from app.database import SessionLocal + from app.services.telephony.call_recording_lifecycle import register_live_recording_paths + + db = SessionLocal() + try: + register_live_recording_paths( + db, + call_short_id=call_short_id, + user_audio_path=user_audio_path, + bot_audio_path=bot_audio_path, + sample_rate=recorder_sample_rate, + ) + db.commit() + finally: + db.close() + from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor - live_transcript_processor = create_live_transcript_processor(call_short_id) if telephony_mode else None + live_transcript_processor = ( + create_live_transcript_processor( + call_short_id, + call_start_time=start_time, + live_observability_emitter=live_observability_emitter, + ) + if telephony_mode + else None + ) user_transcript_processor = live_transcript_processor agent_transcript_processor = ( - create_live_transcript_processor(call_short_id) if telephony_mode and call_short_id else None + create_live_transcript_processor( + call_short_id, + call_start_time=start_time, + live_observability_emitter=live_observability_emitter, + ) + if telephony_mode and call_short_id + else None ) + task = None pipeline_task_ref: list = [] @@ -238,6 +273,14 @@ async def on_silence_hangup(): on_hangup=on_silence_hangup, ) + setup_efficientai_tracing(service_name="efficientai-voice-agent") + span_attributes = { + "organization_id": organization_id or "", + "agent_id": agent_id or "", + } + if workspace_id: + span_attributes["workspace_id"] = workspace_id + if telephony_mode: pipeline_processors = [ws_transport.input()] if silence_hangup_processor: @@ -273,6 +316,8 @@ async def on_silence_hangup(): audio_in_sample_rate=transport_in_sample_rate, audio_out_sample_rate=transport_out_sample_rate, ), + enable_tracing=True, + additional_span_attributes=span_attributes, ) pipeline_task_ref.append(task) @@ -288,6 +333,20 @@ async def on_client_disconnected(transport, client): else: # RTVI events for efficientai client UI rtvi = imports["RTVIProcessor"](config=imports["RTVIConfig"](config=[])) + playground_transcript_call_id = call_short_id + rtvi_user_transcript_processor = None + rtvi_agent_transcript_processor = None + if playground_transcript_call_id or live_observability_emitter is not None: + rtvi_user_transcript_processor = create_live_transcript_processor( + playground_transcript_call_id, + call_start_time=call_start_time, + live_observability_emitter=live_observability_emitter, + ) + rtvi_agent_transcript_processor = create_live_transcript_processor( + playground_transcript_call_id, + call_start_time=call_start_time, + live_observability_emitter=live_observability_emitter, + ) from app.services.usage.voice_usage_processor import create_llm_usage_recorder @@ -298,23 +357,21 @@ async def on_client_disconnected(transport, client): resource_id=agent_id, resource_type="agent" if agent_id else None, ) - pipeline_steps = [ - ws_transport.input(), - user_recorder, - context_aggregator.user(), - rtvi, - llm, - ] + pipeline_processors = [ws_transport.input(), user_recorder, context_aggregator.user()] + if rtvi_user_transcript_processor: + pipeline_processors.append(rtvi_user_transcript_processor) + pipeline_processors.extend([rtvi, llm]) + if rtvi_agent_transcript_processor: + pipeline_processors.append(rtvi_agent_transcript_processor) if usage_recorder: - pipeline_steps.append(usage_recorder) - pipeline_steps.extend( - [ - bot_recorder, - ws_transport.output(), - context_aggregator.assistant(), - ] - ) - pipeline = imports["Pipeline"](pipeline_steps) + pipeline_processors.append(usage_recorder) + pipeline_processors.extend([ + bot_recorder, + ws_transport.output(), + context_aggregator.assistant(), + ]) + + pipeline = imports["Pipeline"](pipeline_processors) task = imports["PipelineTask"]( pipeline, @@ -323,6 +380,8 @@ async def on_client_disconnected(transport, client): enable_usage_metrics=True, ), observers=[imports["RTVIObserver"](rtvi)], + enable_tracing=True, + additional_span_attributes=span_attributes, ) @rtvi.event_handler("on_client_ready") @@ -350,6 +409,9 @@ async def on_client_disconnected(transport, client): logger.error(f"Error in runner.run(): {run_error}", exc_info=True) raise finally: + if task and getattr(task, "turn_trace_observer", None): + conversation_trace_id = task.turn_trace_observer.get_conversation_trace_id() + force_flush_tracing() # Close recorders explicitly to ensure files are flushed await user_recorder.cleanup() await bot_recorder.cleanup() @@ -407,6 +469,7 @@ async def on_client_disconnected(transport, client): transcript_text=transcript_text, s3_key=s3_key_result, duration=duration_result, + trace_id=conversation_trace_id, ) finally: db.close() @@ -421,6 +484,7 @@ async def on_client_disconnected(transport, client): "scenario_id": scenario_id, "transcription": transcript_text, "speaker_segments": conversation_turns if conversation_turns else None, + "trace_id": conversation_trace_id, "error": str(e) } @@ -432,6 +496,7 @@ async def on_client_disconnected(transport, client): "scenario_id": scenario_id, "transcription": transcript_text, "speaker_segments": conversation_turns if conversation_turns else None, + "trace_id": conversation_trace_id, } if not s3_key_result and not transcript_text: metadata["error"] = "No audio file was uploaded and no transcript captured" diff --git a/app/services/voice_agent/live_transcript_processor.py b/app/services/voice_agent/live_transcript_processor.py index 025da3b9..c88bc6d4 100644 --- a/app/services/voice_agent/live_transcript_processor.py +++ b/app/services/voice_agent/live_transcript_processor.py @@ -2,14 +2,19 @@ from __future__ import annotations +import time from typing import Optional from loguru import logger -def create_live_transcript_processor(call_short_id: Optional[str]): +def create_live_transcript_processor( + call_short_id: Optional[str], + call_start_time: Optional[float] = None, + live_observability_emitter: Optional[Any] = None, +): """Return a FrameProcessor that publishes user/agent transcript turns.""" - if not call_short_id: + if not call_short_id and live_observability_emitter is None: return None imports = None @@ -56,28 +61,46 @@ def __init__(self): self._agent_buffer = "" def _persist_turn(self, role: str, content: str) -> None: - publish_transcript_turn(call_short_id, role, content) - db = SessionLocal() - try: - append_live_transcript_turn( - db, - call_short_id=call_short_id, - role=role, - content=content, - ) - logger.debug("Live transcript saved call={} role={} text={}", call_short_id, role, content[:80]) - # region agent log - from app.utils.debug_agent_log import agent_debug_log - - agent_debug_log( - "live_transcript_processor.py:_persist_turn", - "transcript turn persisted", - {"call_short_id": call_short_id, "role": role, "content_len": len(content)}, - "H2", - ) - # endregion - finally: - db.close() + start_time_sec = None + if call_start_time is not None: + start_time_sec = round(time.time() - call_start_time, 2) + if call_short_id: + publish_transcript_turn(call_short_id, role, content) + db = SessionLocal() + try: + append_live_transcript_turn( + db, + call_short_id=call_short_id, + role=role, + content=content, + start_time_sec=start_time_sec, + ) + logger.debug("Live transcript saved call={} role={} text={}", call_short_id, role, content[:80]) + # region agent log + from app.utils.debug_agent_log import agent_debug_log + + agent_debug_log( + "live_transcript_processor.py:_persist_turn", + "transcript turn persisted", + {"call_short_id": call_short_id, "role": role, "content_len": len(content)}, + "H2", + ) + # endregion + finally: + db.close() + if live_observability_emitter is not None: + try: + live_observability_emitter.emit_turn( + role, + content, + start_time=start_time_sec, + ) + except Exception as exc: + logger.warning( + "Live observability turn emit failed for call {}: {}", + call_short_id or getattr(live_observability_emitter, "provider_call_id", "?"), + exc, + ) async def process_frame(self, frame, direction): await super().process_frame(frame, direction) diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index edebc4d0..246419f2 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -19,6 +19,11 @@ from dotenv import load_dotenv from loguru import logger +from app.services.tracing.efficientai_otel import ( + force_flush_tracing, + log_trace_export_status, + setup_efficientai_tracing, +) from app.services.storage.s3_service import s3_service load_dotenv(override=True) @@ -526,6 +531,7 @@ async def run_voice_bundle_fastapi( telephony_mode: bool = False, call_short_id: str | None = None, silence_hangup_secs: float | None = None, + live_observability_emitter=None, ): """ Run the STT+LLM+TTS voice bundle pipeline over a FastAPI WebSocket. @@ -542,6 +548,7 @@ async def run_voice_bundle_fastapi( duration_result = None transcript_text = None conversation_turns = [] + conversation_trace_id = None # Storage for audio data from the buffer processor recorded_audio_data = {"audio": None, "sample_rate": None, "num_channels": None} @@ -589,6 +596,14 @@ async def run_voice_bundle_fastapi( raise ValueError(f"Missing required API keys for voice bundle: {', '.join(missing)}") try: + setup_efficientai_tracing(service_name="efficientai-voice-bundle") + span_attributes = { + "organization_id": organization_id or "", + "agent_id": agent_id or "", + } + if workspace_id: + span_attributes["workspace_id"] = workspace_id + from app.services.voice_agent.tts_sample_rate import ( resolve_tts_sample_rate_hz, resolve_websocket_audio_in_sample_rate_hz, @@ -697,6 +712,7 @@ async def run_voice_bundle_fastapi( ) else: llm = llm_cfg["factory"](api_key=llm_api_key, model=llm_model, params=llm_params) + setattr(llm, "_observability_bundle_type", getattr(voice_bundle, "bundle_type", None)) # Build context with provided system instruction or a default base_instruction = ( @@ -757,6 +773,22 @@ async def run_voice_bundle_fastapi( recorder_name="BotAudioRecorder", alignment_mode="stream", ) + if call_short_id: + from app.database import SessionLocal + from app.services.telephony.call_recording_lifecycle import register_live_recording_paths + + db = SessionLocal() + try: + register_live_recording_paths( + db, + call_short_id=call_short_id, + user_audio_path=user_audio_path, + bot_audio_path=bot_audio_path, + sample_rate=tts_sample_rate, + ) + db.commit() + finally: + db.close() audio_buffer_input = None audio_buffer_output = None input_audio_chunks = [] @@ -788,6 +820,7 @@ async def on_output_audio_data(buffer, audio, sample_rate, num_channels): recorded_audio_data["num_channels"] = num_channels pipeline_task_ref: list = [] + task = None async def on_silence_hangup(): if pipeline_task_ref: @@ -819,11 +852,15 @@ async def on_silence_hangup(): if silence_hangup_processor: pipeline_processors.append(silence_hangup_processor) pipeline_processors.extend([audio_buffer_input, stt]) - if telephony_mode and call_short_id: + if call_short_id or live_observability_emitter is not None: from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor - user_transcript_processor = create_live_transcript_processor(call_short_id) - agent_transcript_processor = create_live_transcript_processor(call_short_id) + user_transcript_processor = create_live_transcript_processor( + call_short_id, call_start_time=recording_start_time, live_observability_emitter=live_observability_emitter + ) + agent_transcript_processor = create_live_transcript_processor( + call_short_id, call_start_time=recording_start_time, live_observability_emitter=live_observability_emitter + ) if user_transcript_processor: pipeline_processors.append(user_transcript_processor) else: @@ -833,7 +870,7 @@ async def on_silence_hangup(): pipeline_processors.append(context_aggregator.user()) pipeline_processors.append(llm) - if telephony_mode and call_short_id and agent_transcript_processor: + if (call_short_id or live_observability_emitter is not None) and agent_transcript_processor: pipeline_processors.append(agent_transcript_processor) from app.services.usage.voice_usage_processor import create_llm_usage_recorder @@ -868,6 +905,8 @@ async def on_silence_hangup(): task = imports["PipelineTask"]( pipeline, params=pipeline_task_params, + enable_tracing=True, + additional_span_attributes=span_attributes, ) pipeline_task_ref.append(task) @@ -889,12 +928,25 @@ async def on_client_disconnected(transport, client): rtvi_processors = [ws_transport.input()] if silence_hangup_processor: rtvi_processors.append(silence_hangup_processor) + rtvi_processors.extend([audio_buffer_input, stt]) + if call_short_id or live_observability_emitter is not None: + from app.services.voice_agent.live_transcript_processor import create_live_transcript_processor + + rtvi_user_transcript_processor = create_live_transcript_processor( + call_short_id, call_start_time=recording_start_time, live_observability_emitter=live_observability_emitter + ) + rtvi_agent_transcript_processor = create_live_transcript_processor( + call_short_id, call_start_time=recording_start_time, live_observability_emitter=live_observability_emitter + ) + if rtvi_user_transcript_processor: + rtvi_processors.append(rtvi_user_transcript_processor) + else: + rtvi_user_transcript_processor = None + rtvi_agent_transcript_processor = None + rtvi_processors.extend([context_aggregator.user(), rtvi, llm]) + if (call_short_id or live_observability_emitter is not None) and rtvi_agent_transcript_processor: + rtvi_processors.append(rtvi_agent_transcript_processor) rtvi_processors.extend([ - audio_buffer_input, - stt, - context_aggregator.user(), - rtvi, - llm, tts, ]) usage_recorder = create_llm_usage_recorder( @@ -916,6 +968,8 @@ async def on_client_disconnected(transport, client): pipeline, params=pipeline_task_params, observers=[imports["RTVIObserver"](rtvi)], + enable_tracing=True, + additional_span_attributes=span_attributes, ) if silence_hangup_processor: pipeline_task_ref.append(task) @@ -945,6 +999,10 @@ async def on_client_disconnected(transport, client): try: await runner.run(task) finally: + if task and getattr(task, "turn_trace_observer", None): + conversation_trace_id = task.turn_trace_observer.get_conversation_trace_id() + force_flush_tracing() + log_trace_export_status(conversation_trace_id) duration_result = time.time() - call_start_time try: @@ -997,6 +1055,7 @@ async def on_client_disconnected(transport, client): conversation_turns=conversation_turns, transcript_text=transcript_text, duration=duration_result, + trace_id=conversation_trace_id, ) logger.info( "Queued finalize_telephony_recording for call_short_id={} " @@ -1033,6 +1092,7 @@ async def on_client_disconnected(transport, client): transcript_text=transcript_text, s3_key=s3_key_result, duration=duration_result, + trace_id=conversation_trace_id, ) finally: db.close() @@ -1107,10 +1167,29 @@ async def on_client_disconnected(transport, client): transcript_text=transcript_text, s3_key=s3_key_result, duration=duration_result, + trace_id=conversation_trace_id, ) finally: db.close() + if call_short_id and not telephony_mode: + from app.database import SessionLocal + from app.services.telephony.call_recording_lifecycle import persist_telephony_call_artifacts + + db = SessionLocal() + try: + persist_telephony_call_artifacts( + db, + call_short_id=call_short_id, + conversation_turns=conversation_turns, + transcript_text=transcript_text, + s3_key=s3_key_result, + duration=duration_result, + trace_id=conversation_trace_id, + ) + finally: + db.close() + if telephony_mode and call_short_id: # region agent log from app.utils.debug_agent_log import agent_debug_log @@ -1138,6 +1217,7 @@ async def on_client_disconnected(transport, client): "scenario_id": scenario_id, "transcription": transcript_text, "speaker_segments": conversation_turns if conversation_turns else None, + "trace_id": conversation_trace_id, "error": str(e), } @@ -1149,6 +1229,7 @@ async def on_client_disconnected(transport, client): "scenario_id": scenario_id, "transcription": transcript_text, "speaker_segments": conversation_turns if conversation_turns else None, + "trace_id": conversation_trace_id, } if not s3_key_result and not transcript_text: metadata["error"] = "No audio file was uploaded and no transcript captured" diff --git a/app/services/voice_providers/base.py b/app/services/voice_providers/base.py index 76248ade..2b11b831 100644 --- a/app/services/voice_providers/base.py +++ b/app/services/voice_providers/base.py @@ -114,3 +114,15 @@ def test_connection(self) -> bool: """ pass + def list_agents(self, **kwargs) -> Dict[str, Any]: + """Optional provider capability: list external agents for integration pickers.""" + raise NotImplementedError("This provider does not support listing agents") + + def retrieve_provider_trace(self, call_id: str, **kwargs) -> Dict[str, Any]: + """Optional provider capability: fetch provider-native execution trace payload.""" + raise NotImplementedError("This provider does not support provider trace retrieval") + + def list_conversations(self, **kwargs) -> Dict[str, Any]: + """Optional provider capability: list provider-side conversations/calls.""" + raise NotImplementedError("This provider does not support listing conversations") + diff --git a/app/services/voice_providers/elevenlabs.py b/app/services/voice_providers/elevenlabs.py index 362110fc..19830d1e 100644 --- a/app/services/voice_providers/elevenlabs.py +++ b/app/services/voice_providers/elevenlabs.py @@ -18,6 +18,36 @@ def __init__(self, api_key: str): super().__init__(api_key) self.api_url = ELEVENLABS_API_URL + def _request(self, method: str, url: str, **kwargs) -> requests.Response: + """Execute an HTTP request with a no-proxy retry fallback. + + Some local environments inject HTTP(S)_PROXY values that block + ElevenLabs with tunnel 403 responses. We first try the default + request path, then retry once with ``trust_env=False`` to bypass + environment proxy settings. + """ + try: + return requests.request(method, url, **kwargs) + except requests.exceptions.ProxyError as proxy_error: + logger.warning( + "[ElevenLabsProvider] Proxy error for {} {}: {}. " + "Retrying without environment proxies.", + method, + url, + proxy_error, + ) + with requests.Session() as session: + session.trust_env = False + retry_kwargs = dict(kwargs) + # Ensure explicit per-request proxies cannot force the same bad tunnel. + retry_kwargs.pop("proxies", None) + return session.request( + method, + url, + proxies={"http": None, "https": None}, + **retry_kwargs, + ) + def create_web_call( self, agent_id: str, @@ -46,7 +76,7 @@ def create_web_call( logger.info(f"[ElevenLabsProvider] Requesting signed URL for agent_id={agent_id}") - response = requests.get(url, headers=headers, params=params, timeout=30) + response = self._request("GET", url, headers=headers, params=params, timeout=30) if response.status_code == 200: data = response.json() @@ -91,12 +121,182 @@ def get_agent(self, agent_id: str) -> Dict[str, Any]: url = f"{self.api_url}/convai/agents/{agent_id}" headers = {"xi-api-key": self.api_key} - response = requests.get(url, headers=headers, timeout=15) + response = self._request("GET", url, headers=headers, timeout=15) response.raise_for_status() return response.json() except Exception as e: raise ValueError(f"Failed to get ElevenLabs agent: {str(e)}") + def list_agents( + self, + *, + page_size: int = 30, + search: Optional[str] = None, + cursor: Optional[str] = None, + ) -> Dict[str, Any]: + """List ElevenLabs Conversational AI agents.""" + try: + url = f"{self.api_url}/convai/agents" + headers = {"xi-api-key": self.api_key} + params: Dict[str, Any] = {"page_size": max(1, min(page_size, 100))} + if search: + params["search"] = search + if cursor: + params["cursor"] = cursor + + response = self._request("GET", url, headers=headers, params=params, timeout=20) + response.raise_for_status() + payload = response.json() + if isinstance(payload, list): + payload = {"agents": payload} + if not isinstance(payload, dict): + payload = {"agents": []} + + # ElevenLabs response shape has changed across API versions. + # Support common containers so the UI doesn't silently render empty. + items = ( + payload.get("agents") + or payload.get("items") + or payload.get("data") + or payload.get("conversational_ai_agents") + or [] + ) + if isinstance(items, dict): + items = ( + items.get("agents") + or items.get("items") + or items.get("data") + or [] + ) + normalized_agents = [] + for item in items: + if not isinstance(item, dict): + continue + # Some variants nest core fields under `agent`. + agent_obj = item.get("agent") if isinstance(item.get("agent"), dict) else item + agent_id = ( + agent_obj.get("agent_id") + or agent_obj.get("id") + or agent_obj.get("agentId") + ) + if not agent_id: + continue + name = ( + agent_obj.get("name") + or agent_obj.get("agent_name") + or item.get("name") + or str(agent_id) + ) + created_at = ( + agent_obj.get("created_at") + or agent_obj.get("created_at_unix_secs") + or agent_obj.get("created_at_unix_ms") + ) + normalized_agents.append( + { + "id": str(agent_id), + "name": str(name), + "archived": bool( + agent_obj.get("archived", False) + or agent_obj.get("is_archived", False) + ), + "created_at": created_at, + "metadata": item, + } + ) + + return { + "agents": normalized_agents, + "has_more": bool(payload.get("has_more", False)), + "next_cursor": payload.get("next_cursor") or payload.get("cursor"), + } + except Exception as e: + raise ValueError(f"Failed to list ElevenLabs agents: {str(e)}") + + def list_conversations( + self, + *, + agent_id: Optional[str] = None, + cursor: Optional[str] = None, + page_size: int = 100, + call_start_after_unix: Optional[int] = None, + ) -> Dict[str, Any]: + """List ElevenLabs conversations for migration/catalog sync.""" + try: + url = f"{self.api_url}/convai/conversations" + headers = {"xi-api-key": self.api_key} + params: Dict[str, Any] = {"page_size": max(1, min(page_size, 100))} + if agent_id: + params["agent_id"] = agent_id + if cursor: + params["cursor"] = cursor + if call_start_after_unix is not None: + params["call_start_after_unix"] = int(call_start_after_unix) + + response = self._request("GET", url, headers=headers, params=params, timeout=30) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + payload = {} + + items = payload.get("conversations") or payload.get("items") or payload.get("data") or [] + if isinstance(items, dict): + items = items.get("conversations") or items.get("items") or items.get("data") or [] + + normalized: list[Dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + conversation_id = ( + item.get("conversation_id") + or item.get("id") + or item.get("call_id") + ) + if not conversation_id: + continue + normalized.append( + { + "conversation_id": str(conversation_id), + "agent_id": item.get("agent_id"), + "status": item.get("status"), + "start_time_unix_secs": item.get("start_time_unix_secs"), + "call_duration_secs": item.get("call_duration_secs"), + "message_count": item.get("message_count"), + "call_successful": item.get("call_successful"), + "metadata": item, + } + ) + + return { + "conversations": normalized, + "has_more": bool(payload.get("has_more", False)), + "next_cursor": payload.get("next_cursor") or payload.get("cursor"), + } + except Exception as e: + raise ValueError(f"Failed to list ElevenLabs conversations: {str(e)}") + + def retrieve_conversation_trace(self, conversation_id: str) -> Dict[str, Any]: + """Fetch conversation details with OpenTelemetry payload.""" + try: + url = f"{self.api_url}/convai/conversations/{conversation_id}" + headers = {"xi-api-key": self.api_key} + response = self._request( + "GET", + url, + headers=headers, + params={"format": "opentelemetry"}, + timeout=30, + ) + response.raise_for_status() + return response.json() + except Exception as e: + raise ValueError(f"Failed to retrieve ElevenLabs conversation trace: {str(e)}") + + def retrieve_provider_trace(self, call_id: str, **kwargs) -> Dict[str, Any]: + """Provider-agnostic alias used by adapter flows.""" + del kwargs + return self.retrieve_conversation_trace(call_id) + def retrieve_call_metrics(self, call_id: str) -> Dict[str, Any]: """ Retrieve conversation details from ElevenLabs. @@ -114,7 +314,7 @@ def retrieve_call_metrics(self, call_id: str) -> Dict[str, Any]: url = f"{self.api_url}/convai/conversations/{call_id}" headers = {"xi-api-key": self.api_key} - response = requests.get(url, headers=headers, timeout=30) + response = self._request("GET", url, headers=headers, timeout=30) response.raise_for_status() data = response.json() @@ -272,7 +472,7 @@ def update_agent_prompt(self, agent_id: str, system_prompt: str, **kwargs) -> Di }, } logger.info(f"[ElevenLabsProvider] Updating agent prompt: PATCH {url}") - response = requests.patch(url, headers=headers, json=payload, timeout=30) + response = self._request("PATCH", url, headers=headers, json=payload, timeout=30) if not response.ok: try: @@ -295,7 +495,7 @@ def test_connection(self) -> bool: url = f"{self.api_url}/user" headers = {"xi-api-key": self.api_key} - response = requests.get(url, headers=headers, timeout=10) + response = self._request("GET", url, headers=headers, timeout=10) if response.status_code == 200: return True elif response.status_code == 401: diff --git a/app/services/voice_providers/retell.py b/app/services/voice_providers/retell.py index aa394d43..7777c4fc 100644 --- a/app/services/voice_providers/retell.py +++ b/app/services/voice_providers/retell.py @@ -2,7 +2,7 @@ Retell Voice Provider Implementation Handles integration with Retell AI voice agents """ -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Set from retell import Retell from loguru import logger @@ -280,6 +280,78 @@ def retrieve_call_metrics(self, call_id: str) -> Dict[str, Any]: except Exception as e: raise ValueError(f"Failed to retrieve Retell call metrics: {str(e)}") + def list_agents( + self, + *, + page_size: int = 30, + search: Optional[str] = None, + cursor: Optional[str] = None, + ) -> Dict[str, Any]: + """List Retell agents and normalize to external-agents response shape.""" + try: + del cursor # Retell SDK currently handles pagination internally. + response = self.client.agent.list() + + payload: Any + if isinstance(response, dict): + payload = response + elif hasattr(response, "model_dump"): + payload = response.model_dump() + elif hasattr(response, "dict"): + payload = response.dict() + else: + payload = {"items": []} + + items = [] + if isinstance(payload, list): + items = payload + elif isinstance(payload, dict): + items = ( + payload.get("agents") + or payload.get("items") + or payload.get("data") + or payload.get("results") + or [] + ) + if isinstance(items, dict): + items = ( + items.get("agents") + or items.get("items") + or items.get("data") + or items.get("results") + or [] + ) + + normalized = [] + query = (search or "").strip().lower() + for item in items: + if not isinstance(item, dict): + continue + agent_id = item.get("agent_id") or item.get("agentId") or item.get("id") + if not agent_id: + continue + name = item.get("agent_name") or item.get("name") or str(agent_id) + if query and query not in str(name).lower(): + continue + normalized.append( + { + "id": str(agent_id), + "name": str(name), + "archived": bool(item.get("is_archived", False) or item.get("archived", False)), + "created_at": item.get("created_at"), + "metadata": item, + } + ) + + limited = normalized[: max(1, min(page_size, 100))] + return { + "agents": limited, + "has_more": len(normalized) > len(limited), + "next_cursor": None, + } + except Exception as e: + raise ValueError(f"Failed to list Retell agents: {str(e)}") + def extract_agent_prompt(self, agent_id: str) -> Optional[str]: """Extract the system prompt from a Retell agent. @@ -288,72 +360,199 @@ def extract_agent_prompt(self, agent_id: str) -> Optional[str]: - conversation-flow: fetch flow by conversation_flow_id, read global_prompt - custom-llm: no extractable prompt (websocket-based) """ + def _to_dict(obj: Any) -> Dict[str, Any]: + if isinstance(obj, dict): + return obj + if hasattr(obj, "model_dump"): + try: + dumped = obj.model_dump() + if isinstance(dumped, dict): + return dumped + except Exception: + pass + if hasattr(obj, "dict"): + try: + dumped = obj.dict() + if isinstance(dumped, dict): + return dumped + except Exception: + pass + return {} + + def _first_non_empty(*values: Any) -> Optional[str]: + for value in values: + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + def _extract_prompt_like(value: Any, visited: Optional[Set[int]] = None) -> Optional[str]: + """Recursively search common prompt/instruction keys across dict/list payloads.""" + if visited is None: + visited = set() + obj_id = id(value) + if obj_id in visited: + return None + visited.add(obj_id) + + if isinstance(value, str): + cleaned = value.strip() + return cleaned if cleaned else None + + if isinstance(value, dict): + exact_keys = [ + "general_prompt", + "generalPrompt", + "global_prompt", + "globalPrompt", + "system_prompt", + "systemPrompt", + "prompt", + "instructions", + "instruction", + "base_prompt", + "basePrompt", + ] + for key in exact_keys: + hit = value.get(key) + if isinstance(hit, str) and hit.strip(): + return hit.strip() + + # Then recurse into likely nested prompt containers first. + priority_nested_keys = [ + "llm", + "model", + "config", + "settings", + "response_engine", + "responseEngine", + "conversation_flow", + "conversationFlow", + ] + for key in priority_nested_keys: + if key in value: + nested = _extract_prompt_like(value.get(key), visited) + if nested: + return nested + + # Finally search all values but prioritize keys that mention prompt/instruction. + promptish_items = [] + other_items = [] + for key, nested_value in value.items(): + key_lower = str(key).lower() + if any(token in key_lower for token in ("prompt", "instruction", "system")): + promptish_items.append(nested_value) + else: + other_items.append(nested_value) + for nested_value in promptish_items: + nested = _extract_prompt_like(nested_value, visited) + if nested: + return nested + for nested_value in other_items: + if isinstance(nested_value, (dict, list)): + nested = _extract_prompt_like(nested_value, visited) + if nested: + return nested + return None + + if isinstance(value, list): + for item in value: + nested = _extract_prompt_like(item, visited) + if nested: + return nested + return None + + return None + try: agent_response = self.client.agent.retrieve(agent_id=agent_id) + agent_payload = _to_dict(agent_response) response_engine = getattr(agent_response, "response_engine", None) if response_engine is None: - logger.warning("[RetellProvider] No response_engine on agent") - return None + response_engine = agent_payload.get("response_engine") + response_engine_payload = _to_dict(response_engine) + + # Fast path: many Retell agent payloads already include prompt-like fields. + prompt = _first_non_empty( + response_engine_payload.get("general_prompt"), + response_engine_payload.get("system_prompt"), + response_engine_payload.get("prompt"), + agent_payload.get("general_prompt"), + agent_payload.get("system_prompt"), + agent_payload.get("prompt"), + ) + if not prompt: + prompt = _extract_prompt_like(response_engine_payload) or _extract_prompt_like(agent_payload) + if prompt: + return prompt engine_type = getattr(response_engine, "type", None) - if isinstance(response_engine, dict): - engine_type = response_engine.get("type") + if isinstance(response_engine, dict) or response_engine_payload: + engine_type = response_engine_payload.get("type", engine_type) logger.debug(f"[RetellProvider] response_engine type={engine_type}") # --- retell-llm: fetch the LLM and read general_prompt --- llm_id = getattr(response_engine, "llm_id", None) - if isinstance(response_engine, dict): - llm_id = response_engine.get("llm_id", llm_id) + if isinstance(response_engine, dict) or response_engine_payload: + llm_id = response_engine_payload.get("llm_id", llm_id) if llm_id: - logger.debug(f"[RetellProvider] Fetching LLM {llm_id}") - llm_response = self.client.llm.retrieve(llm_id=llm_id) - - prompt = getattr(llm_response, "general_prompt", None) - if isinstance(llm_response, dict): - prompt = llm_response.get("general_prompt", prompt) - if prompt: - return prompt - - if hasattr(llm_response, "model_dump"): - prompt = llm_response.model_dump().get("general_prompt") + try: + logger.debug(f"[RetellProvider] Fetching LLM {llm_id}") + llm_response = self.client.llm.retrieve(llm_id=llm_id) + llm_payload = _to_dict(llm_response) + prompt = _first_non_empty( + getattr(llm_response, "general_prompt", None), + llm_payload.get("general_prompt"), + llm_payload.get("system_prompt"), + llm_payload.get("prompt"), + llm_payload.get("generalPrompt"), + llm_payload.get("systemPrompt"), + ) + if not prompt: + prompt = _extract_prompt_like(llm_payload) if prompt: return prompt - - logger.warning(f"[RetellProvider] LLM {llm_id} returned no general_prompt") - return None + logger.warning(f"[RetellProvider] LLM {llm_id} returned no prompt fields") + except Exception as llm_exc: + logger.warning(f"[RetellProvider] Failed LLM prompt lookup for {llm_id}: {llm_exc}") # --- conversation-flow: fetch the flow and read global_prompt --- flow_id = getattr(response_engine, "conversation_flow_id", None) - if isinstance(response_engine, dict): - flow_id = response_engine.get("conversation_flow_id", flow_id) + if isinstance(response_engine, dict) or response_engine_payload: + flow_id = response_engine_payload.get("conversation_flow_id", flow_id) if flow_id: - logger.debug(f"[RetellProvider] Fetching conversation flow {flow_id}") - flow_response = self.client.conversation_flow.retrieve( - conversation_flow_id=flow_id - ) - - prompt = getattr(flow_response, "global_prompt", None) - if isinstance(flow_response, dict): - prompt = flow_response.get("global_prompt", prompt) - if prompt: - return prompt - - if hasattr(flow_response, "model_dump"): - prompt = flow_response.model_dump().get("global_prompt") + try: + logger.debug(f"[RetellProvider] Fetching conversation flow {flow_id}") + flow_response = self.client.conversation_flow.retrieve( + conversation_flow_id=flow_id + ) + flow_payload = _to_dict(flow_response) + prompt = _first_non_empty( + getattr(flow_response, "global_prompt", None), + flow_payload.get("global_prompt"), + flow_payload.get("system_prompt"), + flow_payload.get("prompt"), + flow_payload.get("globalPrompt"), + flow_payload.get("systemPrompt"), + ) + if not prompt: + prompt = _extract_prompt_like(flow_payload) if prompt: return prompt + logger.warning(f"[RetellProvider] Conversation flow {flow_id} returned no prompt fields") + except Exception as flow_exc: + logger.warning(f"[RetellProvider] Failed flow prompt lookup for {flow_id}: {flow_exc}") - logger.warning(f"[RetellProvider] Conversation flow {flow_id} returned no global_prompt") - return None - - # --- custom-llm or unknown: try system_prompt fallback --- - if isinstance(response_engine, dict): - return response_engine.get("system_prompt") - return getattr(response_engine, "system_prompt", None) + # --- custom-llm or unknown: final fallback --- + return _first_non_empty( + response_engine_payload.get("system_prompt"), + response_engine_payload.get("prompt"), + getattr(response_engine, "system_prompt", None), + getattr(response_engine, "prompt", None), + ) except Exception as e: logger.warning(f"[RetellProvider] Failed to extract agent prompt: {e}") diff --git a/app/services/voice_providers/vapi.py b/app/services/voice_providers/vapi.py index abbc2f2f..dead12ed 100644 --- a/app/services/voice_providers/vapi.py +++ b/app/services/voice_providers/vapi.py @@ -199,6 +199,78 @@ def get_agent(self, agent_id: str) -> Dict[str, Any]: except requests.exceptions.RequestException as e: raise ValueError(f"Failed to get Vapi agent: {str(e)}") + def list_agents( + self, + *, + page_size: int = 30, + search: Optional[str] = None, + cursor: Optional[str] = None, + ) -> Dict[str, Any]: + """List Vapi assistants and normalize to external-agents response shape.""" + try: + url = f"{self.api_url}/assistant" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + params: Dict[str, Any] = {"limit": max(1, min(page_size, 100))} + if search: + params["search"] = search + if cursor: + # Some Vapi API variants expose cursor/pagination tokens. + params["cursor"] = cursor + + response = requests.get(url, headers=headers, params=params, timeout=20) + if not response.ok: + try: + error_body = response.json() + except Exception: + error_body = response.text[:500] + raise ValueError( + f"Vapi API error ({response.status_code}): {error_body}" + ) + + payload = response.json() + if isinstance(payload, list): + items = payload + has_more = False + next_cursor = None + elif isinstance(payload, dict): + items = payload.get("assistants") or payload.get("items") or payload.get("data") or [] + if isinstance(items, dict): + items = items.get("assistants") or items.get("items") or items.get("data") or [] + has_more = bool(payload.get("has_more", False)) + next_cursor = payload.get("next_cursor") or payload.get("cursor") + else: + items = [] + has_more = False + next_cursor = None + + normalized = [] + for item in items: + if not isinstance(item, dict): + continue + assistant_id = item.get("id") or item.get("assistantId") + if not assistant_id: + continue + normalized.append( + { + "id": str(assistant_id), + "name": str(item.get("name") or assistant_id), + "archived": bool(item.get("isArchived", False) or item.get("archived", False)), + "created_at": item.get("createdAt") or item.get("created_at"), + "metadata": item, + } + ) + + return { + "agents": normalized, + "has_more": has_more, + "next_cursor": next_cursor, + } + except requests.exceptions.RequestException as e: + raise ValueError(f"Failed to list Vapi agents: {str(e)}") + def extract_agent_prompt(self, agent_id: str) -> Optional[str]: """Extract the system prompt from a Vapi assistant.""" try: diff --git a/app/workers/celery_app.py b/app/workers/celery_app.py index 5a9c6042..b8447efd 100644 --- a/app/workers/celery_app.py +++ b/app/workers/celery_app.py @@ -21,6 +21,10 @@ run_judge_alignment_task, initiate_vobiz_outbound_call_task, finalize_telephony_recording_task, + sync_elevenlabs_agents_task, + sync_elevenlabs_catalog_task, + sync_elevenlabs_enrich_task, + run_elevenlabs_monitor_bridge_task, ) __all__ = [ @@ -36,4 +40,8 @@ "run_judge_alignment_task", "initiate_vobiz_outbound_call_task", "finalize_telephony_recording_task", + "sync_elevenlabs_agents_task", + "sync_elevenlabs_catalog_task", + "sync_elevenlabs_enrich_task", + "run_elevenlabs_monitor_bridge_task", ] diff --git a/app/workers/concurrency/diarization_dispatch.py b/app/workers/concurrency/diarization_dispatch.py index 90faf35d..927f7d07 100644 --- a/app/workers/concurrency/diarization_dispatch.py +++ b/app/workers/concurrency/diarization_dispatch.py @@ -95,7 +95,10 @@ def store_row_diarization_params_batch( keys.append(_pending_params_key(row_uuid)) ids.append(row_uuid) for key in keys: - pipe.setex(key, _PENDING_PARAMS_TTL_SECONDS, payload) + if hasattr(pipe, "set"): + pipe.set(key, payload, ex=_PENDING_PARAMS_TTL_SECONDS) + else: + pipe.setex(key, _PENDING_PARAMS_TTL_SECONDS, payload) results = pipe.execute() for row_uuid, ok in zip(ids, results): if ok: diff --git a/app/workers/concurrency/fair_dispatch.py b/app/workers/concurrency/fair_dispatch.py index dc37cf19..ee382c35 100644 --- a/app/workers/concurrency/fair_dispatch.py +++ b/app/workers/concurrency/fair_dispatch.py @@ -62,10 +62,10 @@ def store_row_restricted_metrics( return key = f"{_RESTRICTED_ROW_KEY_PREFIX}{eval_row_id}" try: - _get_redis().setex( + _get_redis().set( key, - _RESTRICTED_ROW_TTL_SECONDS, json.dumps(restricted_metric_ids), + ex=_RESTRICTED_ROW_TTL_SECONDS, ) except redis.RedisError as exc: logger.warning( @@ -84,7 +84,7 @@ def store_evaluation_transcribe_overwrite( return key = f"{_TRANSCRIBE_OVERWRITE_KEY_PREFIX}{evaluation_id}" try: - _get_redis().setex(key, _RESTRICTED_ROW_TTL_SECONDS, "1") + _get_redis().set(key, "1", ex=_RESTRICTED_ROW_TTL_SECONDS) except redis.RedisError as exc: logger.warning( "Failed to store transcribe_overwrite for evaluation {}: {}", diff --git a/app/workers/config.py b/app/workers/config.py index c6862c92..15ec0a70 100644 --- a/app/workers/config.py +++ b/app/workers/config.py @@ -73,7 +73,7 @@ log_startup_status(component="celery-worker") # Queues consumed by the dedicated call-import / evaluation worker. -IMPORTS_WORKER_QUEUES = "imports,diarization,eval-control,evaluations" +IMPORTS_WORKER_QUEUES = "imports,diarization,eval-control,evaluations,provider-sync" EVAL_CONTROL_QUEUE = "eval-control" USAGE_WORKER_QUEUE = "usage" PLATFORM_WORKER_QUEUE = "platform" @@ -173,6 +173,10 @@ def _platform_beat_schedule() -> dict: "evaluate_studio_run_item": {"queue": "evaluations"}, "generate_agent_flowchart": {"queue": "celery"}, "map_agent_flowchart_prompt_sections": {"queue": "celery"}, + "sync_elevenlabs_agents": {"queue": "provider-sync"}, + "sync_elevenlabs_catalog": {"queue": "provider-sync"}, + "sync_elevenlabs_enrich": {"queue": "provider-sync"}, + "run_elevenlabs_monitor_bridge": {"queue": "provider-sync"}, "flush_usage_counters": {"queue": USAGE_WORKER_QUEUE}, "recompute_usage_costs": {"queue": USAGE_WORKER_QUEUE}, "evaluate_alerts": {"queue": PLATFORM_WORKER_QUEUE}, diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py index a8542175..3c638c26 100644 --- a/app/workers/tasks/__init__.py +++ b/app/workers/tasks/__init__.py @@ -23,6 +23,7 @@ from . import finalize_telephony_recording from . import evaluate_studio_run_item from . import call_import_bulk_ops +from . import elevenlabs_provider_sync from . import flush_usage_counters from . import recompute_usage_costs from . import prune_oss_usage_history @@ -67,6 +68,10 @@ "materialize_call_import_rows_task", "delete_call_import_task", "materialize_call_import_evaluation_task", + "sync_elevenlabs_agents_task", + "sync_elevenlabs_catalog_task", + "sync_elevenlabs_enrich_task", + "run_elevenlabs_monitor_bridge_task", ] process_evaluation_task = process_evaluation.process_evaluation_task @@ -127,6 +132,12 @@ materialize_call_import_evaluation_task = ( call_import_bulk_ops.materialize_call_import_evaluation_task ) +sync_elevenlabs_agents_task = elevenlabs_provider_sync.sync_elevenlabs_agents_task +sync_elevenlabs_catalog_task = elevenlabs_provider_sync.sync_elevenlabs_catalog_task +sync_elevenlabs_enrich_task = elevenlabs_provider_sync.sync_elevenlabs_enrich_task +run_elevenlabs_monitor_bridge_task = ( + elevenlabs_provider_sync.run_elevenlabs_monitor_bridge_task +) flush_usage_counters_task = flush_usage_counters.flush_usage_counters_task recompute_usage_costs_task = recompute_usage_costs.recompute_usage_costs_task prune_oss_usage_history_task = prune_oss_usage_history.prune_oss_usage_history_task diff --git a/app/workers/tasks/elevenlabs_provider_sync.py b/app/workers/tasks/elevenlabs_provider_sync.py new file mode 100644 index 00000000..c0d6b0b7 --- /dev/null +++ b/app/workers/tasks/elevenlabs_provider_sync.py @@ -0,0 +1,466 @@ +"""Celery tasks for ElevenLabs provider migration sync.""" + +from __future__ import annotations + +import asyncio +import threading +import time +from datetime import UTC, datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +from loguru import logger + +from app.core.encryption import decrypt_api_key +from app.config import settings +from app.database import SessionLocal +from app.models.database import ( + Agent, + CallRecording, + CallRecordingSource, + Integration, + ProviderSyncJob, + ProviderSyncJobError, +) +from app.models.enums import CallMediumEnum, CallTypeEnum, IntegrationPlatform +from app.services.observability.call_ingest import upsert_call_recording +from app.services.observability.elevenlabs_monitor_bridge import ElevenLabsMonitorBridge +from app.services.voice_providers import get_voice_provider +from app.services.voice_providers.prompt_sync import sync_provider_prompt +from app.workers.config import celery_app + +_MONITOR_SEMAPHORE = threading.BoundedSemaphore( + max(1, int(settings.ELEVENLABS_MONITOR_MAX_CONCURRENCY)) +) + + +def _job_by_id(db, job_id: str) -> ProviderSyncJob: + job = db.query(ProviderSyncJob).filter(ProviderSyncJob.id == UUID(job_id)).first() + if not job: + raise ValueError(f"Provider sync job not found: {job_id}") + return job + + +def _mark_job(db, job: ProviderSyncJob, *, status: Optional[str] = None, phase: Optional[str] = None) -> None: + if status: + job.status = status + if phase: + job.phase = phase + if status == "running" and not job.started_at: + job.started_at = datetime.now(UTC) + if status in {"completed", "failed", "cancelled"}: + job.completed_at = datetime.now(UTC) + db.commit() + + +def _is_job_cancelled(db, job_id: str) -> bool: + row = ( + db.query(ProviderSyncJob.status) + .filter(ProviderSyncJob.id == UUID(job_id)) + .first() + ) + return bool(row and row[0] == "cancelled") + + +def _record_error( + db, + *, + job: ProviderSyncJob, + phase: str, + error_message: str, + provider_call_id: Optional[str] = None, + provider_agent_id: Optional[str] = None, + payload: Optional[Dict[str, Any]] = None, +) -> None: + row = ProviderSyncJobError( + job_id=job.id, + phase=phase, + provider_call_id=provider_call_id, + provider_agent_id=provider_agent_id, + error_message=error_message[:4000], + payload=payload, + ) + db.add(row) + job.errors_count = int(job.errors_count or 0) + 1 + job.last_error = error_message[:4000] + db.commit() + + +def _provider_from_job(db, job: ProviderSyncJob): + integration = _integration_for_job(db, job) + platform = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + provider_class = get_voice_provider(platform) + return provider_class(api_key=decrypt_api_key(integration.api_key)) + + +def _integration_for_job(db, job: ProviderSyncJob) -> Integration: + integration = ( + db.query(Integration) + .filter( + Integration.id == job.integration_id, + Integration.organization_id == job.organization_id, + Integration.is_active == True, + ) + .first() + ) + if not integration: + raise ValueError("Integration not found or inactive for provider sync job") + platform = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + if platform != IntegrationPlatform.ELEVENLABS.value: + raise ValueError(f"Unsupported provider for sync job: {platform}") + return integration + + +def _trim_insights_only_payload(call_data: Dict[str, Any]) -> Dict[str, Any]: + raw = call_data.get("raw_data") if isinstance(call_data.get("raw_data"), dict) else {} + trimmed_raw = {} + for key in ("metadata", "analysis", "status", "agent_id", "conversation_id", "has_audio", "transcript"): + if key in raw: + trimmed_raw[key] = raw[key] + payload = { + **call_data, + "raw_data": trimmed_raw, + "insights_only": True, + "audio_storage": "elevenlabs", + } + return payload + + +def _status_to_event(status_name: str) -> str: + lowered = (status_name or "").strip().lower() + if lowered in {"done", "ended", "completed", "failed"}: + return "call_ended" + if lowered in {"initiated", "in-progress", "processing"}: + return "call_in_progress" + return "call_in_progress" + + +def _throttle(last_request_at: float, *, rps: float) -> float: + safe_rps = max(float(rps or 1.0), 0.1) + min_interval = 1.0 / safe_rps + now = time.monotonic() + elapsed = now - last_request_at if last_request_at > 0 else min_interval + if elapsed < min_interval: + time.sleep(min_interval - elapsed) + now = time.monotonic() + return now + + +@celery_app.task(name="sync_elevenlabs_agents", queue="provider-sync") +def sync_elevenlabs_agents_task(job_id: str) -> Dict[str, Any]: + db = SessionLocal() + try: + job = _job_by_id(db, job_id) + if job.status == "cancelled": + return {"status": "cancelled"} + _mark_job(db, job, status="running", phase="agents") + provider = _provider_from_job(db, job) + integration = _integration_for_job(db, job) + last_req = 0.0 + + cursor = None + total = 0 + while True: + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + last_req = _throttle(last_req, rps=settings.ELEVENLABS_SYNC_MAX_RPS) + payload = provider.list_agents(page_size=100, cursor=cursor) + for item in payload.get("agents", []): + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + provider_agent_id = str(item.get("id") or "").strip() + if not provider_agent_id: + continue + name = str(item.get("name") or provider_agent_id).strip() + agent = ( + db.query(Agent) + .filter( + Agent.organization_id == job.organization_id, + Agent.workspace_id == job.workspace_id, + Agent.voice_ai_integration_id == job.integration_id, + Agent.voice_ai_agent_id == provider_agent_id, + ) + .first() + ) + if not agent: + agent = Agent( + organization_id=job.organization_id, + workspace_id=job.workspace_id, + name=name, + language="english", + call_type=CallTypeEnum.OUTBOUND.value, + call_medium=CallMediumEnum.PHONE_CALL.value, + voice_ai_integration_id=job.integration_id, + voice_ai_agent_id=provider_agent_id, + description="Imported from ElevenLabs provider sync", + ) + db.add(agent) + else: + agent.name = name + agent.voice_ai_integration_id = job.integration_id + agent.voice_ai_agent_id = provider_agent_id + try: + sync_provider_prompt(agent=agent, integration=integration, db=db) + except Exception as prompt_exc: + logger.warning( + "Provider prompt sync failed for provider_agent_id={}: {}", + provider_agent_id, + prompt_exc, + ) + total += 1 + db.flush() + job.agents_synced = total + db.commit() + + if not payload.get("has_more"): + break + cursor = payload.get("next_cursor") + if not cursor: + break + + _mark_job(db, job, status="completed", phase="complete") + return {"status": "ok", "agents_synced": total} + except Exception as exc: + logger.exception("sync_elevenlabs_agents_task failed job_id={}", job_id) + try: + job = _job_by_id(db, job_id) + _record_error(db, job=job, phase="agents", error_message=str(exc)) + _mark_job(db, job, status="failed", phase="failed") + except Exception: + pass + raise + finally: + db.close() + + +@celery_app.task(name="sync_elevenlabs_catalog", queue="provider-sync") +def sync_elevenlabs_catalog_task(job_id: str) -> Dict[str, Any]: + db = SessionLocal() + try: + job = _job_by_id(db, job_id) + if job.status == "cancelled": + return {"status": "cancelled"} + _mark_job(db, job, status="running", phase="catalog") + provider = _provider_from_job(db, job) + last_req = 0.0 + config = job.config if isinstance(job.config, dict) else {} + since_unix = config.get("since_unix") + allowed_agents = config.get("agent_ids") + if isinstance(allowed_agents, list) and allowed_agents: + provider_agent_ids = [str(a) for a in allowed_agents if str(a).strip()] + else: + provider_agent_ids = [ + row[0] + for row in ( + db.query(Agent.voice_ai_agent_id) + .filter( + Agent.organization_id == job.organization_id, + Agent.workspace_id == job.workspace_id, + Agent.voice_ai_integration_id == job.integration_id, + Agent.voice_ai_agent_id.isnot(None), + ) + .all() + ) + if row and row[0] + ] + + total = int(job.conversations_cataloged or 0) + cursor_state = job.cursor_state if isinstance(job.cursor_state, dict) else {} + for provider_agent_id in provider_agent_ids: + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + cursor = cursor_state.get(provider_agent_id) + while True: + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + last_req = _throttle(last_req, rps=settings.ELEVENLABS_SYNC_MAX_RPS) + payload = provider.list_conversations( + agent_id=provider_agent_id, + cursor=cursor, + page_size=100, + call_start_after_unix=since_unix, + ) + conversations = payload.get("conversations") or [] + for item in conversations: + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + conversation_id = str(item.get("conversation_id") or "").strip() + if not conversation_id: + continue + linked_agent = ( + db.query(Agent) + .filter( + Agent.organization_id == job.organization_id, + Agent.voice_ai_integration_id == job.integration_id, + Agent.voice_ai_agent_id == provider_agent_id, + ) + .first() + ) + call_data_payload = { + "conversation_id": conversation_id, + "agent_id": provider_agent_id, + "provider_platform": IntegrationPlatform.ELEVENLABS.value, + "status": item.get("status"), + "call_status": item.get("status"), + "duration_seconds": item.get("call_duration_secs"), + "start_time_unix_secs": item.get("start_time_unix_secs"), + "message_count": item.get("message_count"), + "call_successful": item.get("call_successful"), + "insights_only": True, + "audio_storage": "elevenlabs", + "_sync_source": "elevenlabs_catalog", + "_sync_job_id": str(job.id), + } + upsert_call_recording( + db=db, + organization_id=job.organization_id, + workspace_id=linked_agent.workspace_id if linked_agent and linked_agent.workspace_id else job.workspace_id, + provider_platform=IntegrationPlatform.ELEVENLABS.value, + provider_call_id=conversation_id, + call_data_payload=call_data_payload, + explicit_agent_id=linked_agent.id if linked_agent else None, + call_event=_status_to_event(str(item.get("status") or "")), + source=CallRecordingSource.WEBHOOK, + ) + total += 1 + job.conversations_cataloged = total + cursor = payload.get("next_cursor") + cursor_state[provider_agent_id] = cursor + job.cursor_state = cursor_state + db.commit() + if not payload.get("has_more") or not cursor: + break + + _mark_job(db, job, status="running", phase="enrich") + return {"status": "ok", "conversations_cataloged": total} + except Exception as exc: + logger.exception("sync_elevenlabs_catalog_task failed job_id={}", job_id) + try: + job = _job_by_id(db, job_id) + _record_error(db, job=job, phase="catalog", error_message=str(exc)) + _mark_job(db, job, status="failed", phase="failed") + except Exception: + pass + raise + finally: + db.close() + + +@celery_app.task(name="sync_elevenlabs_enrich", queue="provider-sync") +def sync_elevenlabs_enrich_task(job_id: str) -> Dict[str, Any]: + db = SessionLocal() + try: + job = _job_by_id(db, job_id) + if job.status == "cancelled": + return {"status": "cancelled"} + _mark_job(db, job, status="running", phase="enrich") + provider = _provider_from_job(db, job) + last_req = 0.0 + config = job.config if isinstance(job.config, dict) else {} + since_unix = config.get("since_unix") + if since_unix: + since_dt = datetime.fromtimestamp(int(since_unix), tz=UTC) + rows = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == job.organization_id, + CallRecording.workspace_id == job.workspace_id, + CallRecording.provider_platform == IntegrationPlatform.ELEVENLABS.value, + CallRecording.updated_at >= since_dt, + ) + .all() + ) + else: + rows = ( + db.query(CallRecording) + .filter( + CallRecording.organization_id == job.organization_id, + CallRecording.workspace_id == job.workspace_id, + CallRecording.provider_platform == IntegrationPlatform.ELEVENLABS.value, + ) + .all() + ) + + total = int(job.conversations_enriched or 0) + for row in rows: + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + call_data = row.call_data if isinstance(row.call_data, dict) else {} + if call_data.get("transcript") and call_data.get("analysis"): + continue + provider_call_id = row.provider_call_id or call_data.get("conversation_id") + if not provider_call_id: + continue + try: + last_req = _throttle(last_req, rps=settings.ELEVENLABS_SYNC_MAX_RPS) + refreshed = provider.retrieve_call_metrics(str(provider_call_id)) + if not isinstance(refreshed, dict): + continue + merged = {**call_data, **refreshed} + merged = _trim_insights_only_payload(merged) + merged["_sync_source"] = "elevenlabs_enrich" + merged["_sync_job_id"] = str(job.id) + row.call_data = merged + total += 1 + job.conversations_enriched = total + db.commit() + except Exception as exc: + _record_error( + db, + job=job, + phase="enrich", + error_message=str(exc), + provider_call_id=str(provider_call_id), + ) + db.rollback() + if _is_job_cancelled(db, job_id): + return {"status": "cancelled"} + _mark_job(db, job, status="completed", phase="complete") + return {"status": "ok", "conversations_enriched": total} + except Exception as exc: + logger.exception("sync_elevenlabs_enrich_task failed job_id={}", job_id) + try: + job = _job_by_id(db, job_id) + _record_error(db, job=job, phase="enrich", error_message=str(exc)) + _mark_job(db, job, status="failed", phase="failed") + except Exception: + pass + raise + finally: + db.close() + + +@celery_app.task(name="run_elevenlabs_monitor_bridge", queue="provider-sync") +def run_elevenlabs_monitor_bridge_task( + *, + conversation_id: str, + elevenlabs_api_key: str, + efficientai_api_key: str, + workspace_id: Optional[str] = None, + efficientai_base_url: str = "http://localhost:8000", + provider_platform: str = "elevenlabs", +) -> Dict[str, Any]: + acquired = _MONITOR_SEMAPHORE.acquire(timeout=5) + if not acquired: + raise RuntimeError("ElevenLabs monitor concurrency limit reached") + try: + bridge = ElevenLabsMonitorBridge( + conversation_id=conversation_id, + elevenlabs_api_key=elevenlabs_api_key, + efficientai_api_key=efficientai_api_key, + workspace_id=workspace_id, + efficientai_base_url=efficientai_base_url, + provider_platform=provider_platform, + ) + asyncio.run(bridge.run()) + return {"status": "ok", "conversation_id": conversation_id} + finally: + _MONITOR_SEMAPHORE.release() diff --git a/app/workers/tasks/finalize_telephony_recording.py b/app/workers/tasks/finalize_telephony_recording.py index d90d7f3b..eb86900e 100644 --- a/app/workers/tasks/finalize_telephony_recording.py +++ b/app/workers/tasks/finalize_telephony_recording.py @@ -26,6 +26,7 @@ def finalize_telephony_recording_task( conversation_turns: Optional[List[Dict[str, Any]]] = None, transcript_text: Optional[str] = None, duration: Optional[float] = None, + trace_id: Optional[str] = None, ) -> dict: """Merge dual-track WAVs, upload to S3, persist CallRecording, queue evaluator.""" try: @@ -48,6 +49,7 @@ def finalize_telephony_recording_task( transcript_text=transcript_text, s3_key=s3_key, duration=effective_duration, + trace_id=trace_id, ) finally: db.close() diff --git a/config.docker.yml b/config.docker.yml index 3d169129..9060f06c 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -76,7 +76,20 @@ api: # storage: "filesystem" (default, local Docker volume) or "s3" (durable, for production) # multi_tenant: false (default, single tenant) or true (per-org log isolation) observability: - enabled: false + enabled: true + live: + ingest_enabled: true + aggregates_enabled: true + dashboard_enabled: true + slo_alerts_enabled: true + tracing: + enabled: true + exporter: tempo_http + endpoint: "http://tempo:4318/v1/traces" + sample_rate: 1.0 + query_backend: tempo + tempo_query_url: "http://tempo:3200" + # trace_query_url: "https://api.efficientai.ai/observability/v1/traces" loki: enabled: false url: "http://loki:3100" @@ -135,4 +148,4 @@ judge_alignment: # Enterprise License (JWT signed with RS256). Unlocks gated features like # oidc_sso, mfa_enforce, audit_export, voice_playground, gepa_optimization. # license: -# key: "eyJhbGciOi..." \ No newline at end of file +# key: "eyJhbGciOi..." diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index 35b84190..58b390d6 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -124,6 +124,19 @@ services: depends_on: - prometheus - loki + - tempo + + tempo: + image: grafana/tempo:2.6.1 + container_name: efficientai_tempo + command: + - "-config.file=/etc/tempo/tempo.yml" + volumes: + - ./observability/tempo/tempo.yml:/etc/tempo/tempo.yml:ro + - tempo_data:/var/tempo + ports: + - "3200:3200" + - "4318:4318" postgres-exporter: image: prometheuscommunity/postgres-exporter:v0.16.0 @@ -162,3 +175,4 @@ volumes: prometheus_data: grafana_data: loki_data: + tempo_data: diff --git a/docker-compose.yml b/docker-compose.yml index 1d6f0d0b..4b7a465d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,12 +70,18 @@ services: # AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=https;AccountName=... # AZURE_CONTAINER_NAME: your-container SERVICE_MODE: api + # Live observability ingest (Pipecat / external runtime → /observability/live/events) + OBSERVABILITY_LIVE_INGEST_ENABLED: ${OBSERVABILITY_LIVE_INGEST_ENABLED:-true} + OBSERVABILITY_LIVE_AGGREGATES_ENABLED: ${OBSERVABILITY_LIVE_AGGREGATES_ENABLED:-true} + OBSERVABILITY_LIVE_DASHBOARD_ENABLED: ${OBSERVABILITY_LIVE_DASHBOARD_ENABLED:-true} # Browser voice-agent WS when split (optional). Vobiz uses vobiz.webhook_base_url on telephony. # MEDIA_WS_BASE_URL: ${MEDIA_WS_BASE_URL:-ws://localhost:8001} volumes: - ./uploads:/app/uploads - ./.data:/app/.data - ./config.docker.yml:/app/config.yml:ro + # Mount local frontend build so UI changes appear without rebuilding the API image + - ./frontend/dist:/app/frontend/dist:ro # Optional: mount GCP service account for GCS auth # - ./secrets/gcp-sa.json:/app/secrets/gcp-sa.json:ro ports: @@ -186,7 +192,7 @@ services: sh -c "celery -A app.workers.celery_app worker -Q platform --pool threads --concurrency 2 --loglevel=info & exec celery -A app.workers.celery_app beat --loglevel=info" - # Dedicated worker for call-import + evaluation queues. Celery drains + # Dedicated worker for call-import + evaluation + provider-sync queues. Celery drains # ``imports`` (recording fetch) before ``diarization`` (manual diarise), # then ``eval-control`` (cancel/retry/materialize), then ``evaluations`` # (fair dispatch + LLM scoring). @@ -224,7 +230,7 @@ services: # - ./secrets/gcp-sa.json:/app/secrets/gcp-sa.json:ro # Thread pool: keep concurrency near Redis inflight caps + headroom when sharding # (each task may hold catalog + shard connections for tens of seconds). - command: eai worker --config /app/config.yml --loglevel info --queues imports,diarization,eval-control,evaluations --pool threads --concurrency 12 + command: eai worker --config /app/config.yml --loglevel info --queues imports,diarization,eval-control,evaluations,provider-sync --pool threads --concurrency 12 # Low-priority usage pricing: Redis flush + cost recompute/backfill. worker-usage: diff --git a/docs/telemetry/README.md b/docs/telemetry/README.md new file mode 100644 index 00000000..851b7c70 --- /dev/null +++ b/docs/telemetry/README.md @@ -0,0 +1,48 @@ +# Telemetry Docs (Product Observability) + +This folder defines the Product Observability contract for EfficientAI live calls. +It is the source of truth for: + +- what we trace +- which span names and attributes are stable +- PII and data-handling boundaries +- local developer setup for validating live STT/LLM/TTS traces +- scale architecture for high-volume traffic + +## Documents + +- `taxonomy.md`: Product observability vs platform observability ownership. +- `span-contract.md`: Required span names and attributes for EfficientAI UI. +- `pii-policy.md`: Transcript and sensitive data policy for trace attributes. +- `local-dev-live-calls.md`: Local V1 flow and environment setup. +- `scale-architecture.md`: Media plane, collector, sampling, and quotas. +- `live-event-contract.md`: Platform-neutral live event envelope and ingest semantics. + +## V1 scope guardrails + +V1 focuses on live calls first: + +- In scope: `conversation`, `turn`, `stt`, `llm`, `tts`, `trace_id` linking. +- Out of scope: `s2s`, `tool_call`, external provider webhooks, platform ops. + +Phase 2 extends to external webhooks and additional span types. + +## Implementation status + +- [x] Live tracing bootstrap (`efficientai_otel`) and `trace_id` call linkage +- [x] Trace waterfall UI and trace fetch API (`cloud` + `tempo`) +- [x] Calls summary cards with latency/volume and trace/eval rates +- [x] Header-auth observe endpoint (`POST /api/v1/observability/observe`) +- [x] Webhook ingest routes for flat, Retell, ElevenLabs, and Vapi payloads +- [x] Optional transcript attribute suppression via `OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS` +- [x] Agent-level auto-eval trigger on `call_ended` webhook ingest +- [x] `tool_call`/`s2s` span naming alignment and TraceTree colors +- [x] Quota/sampling config hooks for scale preparation docs +- [x] Provider trace persistence (`call_data.provider_trace`) with stored-first trace resolution +- [x] S3 trace archival fallback for large provider trace payloads (`provider_trace.trace_s3_key`) +- [x] Live event contract and rollout flags for staged live-call tracking +- [x] Idempotent incremental live event ingest (`POST /api/v1/observability/live/events`) +- [x] Rolling live latency percentile APIs (`GET /api/v1/observability/live/metrics/latency`) +- [x] Agent-scoped live latency percentile API (`GET /api/v1/observability/live/agents/{agent_id}/latency`) +- [x] Level 3 trace-correlation fallback (ingest ACK issues trace id when external trace id is missing) +- [x] Live SLO breach recording + evaluator automation hooks (flag-gated) diff --git a/docs/telemetry/custom-sdk-observe-guide.md b/docs/telemetry/custom-sdk-observe-guide.md new file mode 100644 index 00000000..717cbb61 --- /dev/null +++ b/docs/telemetry/custom-sdk-observe-guide.md @@ -0,0 +1,44 @@ +# Custom SDK Observe Guide (Phase 2) + +Use this when ingesting external calls while preserving trace linkage. + +## Endpoint + +POST `https://api.efficientai.ai/api/v1/observability/observe` + +For incremental live runtime tracking (LiveKit/Pipecat/external): + +POST `https://api.efficientai.ai/api/v1/observability/live/events` + +Include auth: + +- `x-efficient-ai-api-key` +- `x-efficient-ai-agent-id` (or project ID) + +## Minimum payload + +```json +{ + "id": "provider-call-id", + "provider_platform": "retell", + "startedAt": "2026-08-07T08:20:00Z", + "endedAt": "2026-08-07T08:21:02Z", + "messages": [{"role": "assistant", "content": "hello"}], + "trace_id": "0af7651916cd43dd8448eb211c80319c" +} +``` + +## Mapping rules + +- `id` -> provider call identifier +- `provider_platform` -> provider namespace +- `trace_id` -> call-to-trace join key +- `messages` -> transcript/evaluation input + +Live event envelope fields are documented in `docs/telemetry/live-event-contract.md`. + +## Validation checklist + +- `trace_id` present for every completed call +- all timestamps in ISO-8601 UTC +- no credential or secret fields included in payload/body diff --git a/docs/telemetry/elevenlabs-integration.md b/docs/telemetry/elevenlabs-integration.md new file mode 100644 index 00000000..ffdeb110 --- /dev/null +++ b/docs/telemetry/elevenlabs-integration.md @@ -0,0 +1,109 @@ +# ElevenLabs Integration and Trace Validation + +This guide describes the ElevenLabs-first integration path for: + +1. Loading provider agents into EfficientAI dropdowns. +2. Capturing ElevenLabs OTLP traces through webhook and on-demand APIs. +3. Validating end-to-end turn capture in observability. + +## Required ElevenLabs scopes + +- `CONVAI_READ` for: + - `GET /v1/convai/agents` + - `GET /v1/convai/conversations/{conversation_id}?format=opentelemetry` +- `CONVAI_WRITE` if creating signed web calls from EfficientAI. + +## Provider agent catalog + +EfficientAI backend endpoint: + +- `GET /api/v1/integrations/{integration_id}/external-agents` + +For ElevenLabs integrations, this proxies: + +- `GET https://api.elevenlabs.io/v1/convai/agents` + +and returns normalized rows: + +```json +{ + "agents": [ + { + "id": "agent_...", + "name": "Customer Support Agent", + "archived": false + } + ], + "has_more": false, + "next_cursor": null +} +``` + +## Trace ingestion surfaces + +### 1) Post-call webhook (preferred durable path) + +Webhook endpoint: + +- `POST /api/v1/observability/calls/webhook/elevenlabs/{api_key}` + +Expected OTLP webhook payload type: + +- `post_call_transcription_otel` + +EfficientAI stores: + +- `call_data.provider_trace.source = elevenlabs_post_call_webhook` +- `call_data.provider_trace.trace_source = elevenlabs` +- `call_data.provider_trace.normalized_trace = { trace_id, root_span_id, spans }` +- `call_data.provider_trace.otlp_traces = { resourceSpans: ... }` (inline when small) +- `call_data.provider_trace.trace_s3_key` when raw OTLP exceeds inline threshold + +### 2) On-demand fallback fetch + +When opening call detail trace and no stored provider trace exists, EfficientAI can call: + +- `GET /v1/convai/conversations/{conversation_id}?format=opentelemetry` + +This path requires resolving the ElevenLabs integration linked to the call agent. +Fetched OTLP payloads are written back into `call_data.provider_trace` so subsequent loads do not require another provider API fetch. + +## Span namespace rules + +EfficientAI does not remap provider span names to internal names. + +- Keep ElevenLabs names unchanged: + - `elevenlabs.conversation` + - `elevenlabs.recv.user_transcript` + - `elevenlabs.recv.agent_response` + - `elevenlabs.tool.*` +- Use `attributes.trace.provider = elevenlabs` on normalized spans. + +This prevents collisions with EfficientAI-native names such as `turn`, `stt`, `llm`, and `tts`. + +## End-to-end validation checklist + +1. Save ElevenLabs integration in EfficientAI. +2. Confirm external agent dropdown populates from provider. +3. Link an EfficientAI agent to one ElevenLabs provider agent. +4. Configure ElevenLabs post-call webhook with transcript format `opentelemetry`. +5. Complete a test call with at least: + - 2 user turns + - 2 agent turns +6. Open observability call detail and verify trace view: + - source badge shows `elevenlabs` + - root span `elevenlabs.conversation` + - user turn spans `elevenlabs.recv.user_transcript` + - agent turn spans `elevenlabs.recv.agent_response` + - tool spans nested under agent response (if tools are used) + +## Regression fixtures + +Keep redacted fixtures under: + +- `tests/fixtures/elevenlabs/agents_list.json` +- `tests/fixtures/elevenlabs/conv.json` +- `tests/fixtures/elevenlabs/conv_otel.json` +- `tests/fixtures/elevenlabs/post_call_transcription_otel.json` + +These fixtures back normalization and webhook tests. diff --git a/docs/telemetry/elevenlabs-local-ngrok.md b/docs/telemetry/elevenlabs-local-ngrok.md new file mode 100644 index 00000000..18f22e48 --- /dev/null +++ b/docs/telemetry/elevenlabs-local-ngrok.md @@ -0,0 +1,77 @@ +# ElevenLabs Local ngrok Validation + +This guide validates that ElevenLabs post-call webhooks reach local EfficientAI and create Observability rows. + +## Scope + +- Tunnel target: `http://localhost:8000` (API) +- Webhook route: + `POST /api/v1/observability/calls/webhook/elevenlabs/{api_key}` +- Event type: `post_call_transcription_otel` +- Transcript format: `opentelemetry` + +Do not use port `8001` for this path. Port `8001` is the telephony/media edge. + +## Prerequisites + +1. EfficientAI API stack running (venv path used by this repo): + +```bash +source /Users/aadharsinghbhadauria/Desktop/efficientai/.venv/bin/activate +cd /Users/aadharsinghbhadauria/Desktop/efficientai/efficientAI +python -m app.cli start-all --config config.yml --no-reload --no-build-frontend +``` + +2. Active ElevenLabs integration in EfficientAI with `CONVAI_READ` scope. +3. At least one EfficientAI agent linked to an ElevenLabs provider agent (`voice_ai_agent_id`). + +## Start ngrok + +```bash +ngrok http 8000 +``` + +Copy the generated HTTPS URL, for example: + +`https://.ngrok-free.app` + +## Configure ElevenLabs webhook + +In ElevenLabs workspace/agent webhook settings: + +- URL: + `https://.ngrok-free.app/api/v1/observability/calls/webhook/elevenlabs/{api_key}` +- Events: `transcript` +- Transcript format: `opentelemetry` +- Avoid enabling audio webhook for migration validation (large base64 payload, not required for insights-only import). + +## Run validation call + +1. Start and finish a real call on the linked ElevenLabs agent. +2. Check ngrok request inspector: + - Request path includes `/webhook/elevenlabs/{api_key}` + - Response status is `201` +3. Open EfficientAI: + - Observability -> Calls + - Verify a new ElevenLabs row exists + - Open call detail and verify transcript/insights + +## Expected backend behavior + +The webhook handler in `app/api/v1/routes/observability.py`: + +- Accepts `post_call_transcription_otel` payload +- Resolves linked internal agent from provider `agent_id` +- Persists call shell + transcript +- Normalizes provider OTLP trace for the trace tab + +## Troubleshooting + +- `404 integration/agent`: + Ensure the internal agent is linked to the ElevenLabs provider agent ID. +- `502 failed to list/fetch provider data`: + Verify ElevenLabs key scope and integration key correctness. +- No row in Observability: + Confirm route is on API `:8000` and the webhook URL includes the API key segment. +- No trace rendered: + Confirm ElevenLabs sends `post_call_transcription_otel` and not plain JSON transcript. diff --git a/docs/telemetry/live-event-contract.md b/docs/telemetry/live-event-contract.md new file mode 100644 index 00000000..925d0ae9 --- /dev/null +++ b/docs/telemetry/live-event-contract.md @@ -0,0 +1,133 @@ +# Live Event Contract (Incremental Ingest) + +This document defines the platform-neutral envelope accepted by live observability ingest. + +## Endpoint + +- `POST /api/v1/observability/live/events` + +## Envelope + +```json +{ + "event_id": "evt_01J6H6FD4NBRS3P78N84TFM2QV", + "call_id": "call_abc123", + "event_type": "turn.assistant", + "seq": 12, + "event_ts": "2026-08-12T13:01:22.145Z", + "platform": "livekit", + "agent_ref": "agent_42", + "payload": { + "content": "Sure, I can help with that.", + "latency": { + "llm_ms": 420, + "tts_ms": 210 + } + }, + "trace_id": "0af7651916cd43dd8448eb211c80319c" +} +``` + +## Required fields + +- `event_id`: globally unique idempotency key. +- `call_id`: provider/external call identifier. +- `event_type`: semantic event name (`call.started`, `turn.user`, `turn.assistant`, `call.ended`, etc). +- `seq`: monotonically increasing sequence per call. +- `event_ts`: event timestamp in ISO-8601 UTC. +- `platform`: source runtime/platform (`livekit`, `pipecat`, `external`). +- `payload`: event body with provider-specific details. + +## Optional fields + +- `agent_ref`: external agent identifier. +- `trace_id`: external trace id for direct Level 3 correlation. + +## Delivery and ordering semantics + +- Delivery is **at-least-once**. +- Idempotency key is `(organization_id, event_id)`. +- Duplicate events return success with `duplicate=true` and do not mutate state. +- Events with sequence older than watermark by more than + `OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ` are rejected as stale. +- Accepted timestamp drift window is controlled by + `OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS`. +- If `trace_id` is missing, EfficientAI issues one in ACK so clients can reuse it. + +## Merge semantics + +- `call.started`: creates/upserts call shell and stamps start metadata. +- `turn.*`: appends live transcript turns and updates in-progress state. +- `call.ended` / `call.failed`: stamps terminal state and final metadata. +- Existing `call_data` keys are preserved; only live-observability fields are patched. + +## Transcript (required for UI + synthetic trace) + +Post one event per finalized utterance: + +```json +{ + "event_type": "turn.user", + "payload": { "content": "Hello?" } +} +``` + +```json +{ + "event_type": "turn.assistant", + "payload": { + "content": "Hi, how can I help?", + "latency": { "stt_ms": 120, "llm_ms": 380, "tts_ms": 210 } + } +} +``` + +Without `turn.user` / `turn.assistant` events, EfficientAI only stores an empty call shell +(`call.started` + `call.ended`) — no transcript tab and no synthetic trace. + +## Recording (optional, on `call.ended`) + +```json +{ + "event_type": "call.ended", + "payload": { + "endedAt": "2026-08-18T09:00:00.000Z", + "recording_url": "https://your-cdn.example/recording.wav", + "duration_seconds": 42.5 + } +} +``` + +EfficientAI archives `recording_url` to object storage when the call is terminal. + +## Trace (choose one) + +**Level 2 — synthetic (recommended for Pipecat/LiveKit):** include turn latencies on +`turn.assistant` / `turn.user` events; EfficientAI builds STT/LLM/TTS spans on `call.ended`. + +**Level 3 — native OTLP:** pass `trace_id` on every event and include exported spans on +`call.ended`: + +```json +{ + "event_type": "call.ended", + "trace_id": "0af7651916cd43dd8448eb211c80319c", + "payload": { + "otlp_traces": { "...": "OTLP JSON resourceSpans blob" }, + "trace_source": "pipecat_native" + } +} +``` + +Or inline normalized trace: + +```json +{ + "payload": { + "provider_trace": { + "trace_source": "livekit_native", + "normalized_trace": { "trace_id": "...", "spans": [] } + } + } +} +``` diff --git a/docs/telemetry/local-dev-live-calls.md b/docs/telemetry/local-dev-live-calls.md new file mode 100644 index 00000000..5af89328 --- /dev/null +++ b/docs/telemetry/local-dev-live-calls.md @@ -0,0 +1,67 @@ +# Local Dev: Live Calls (V1) + +Goal: validate STT/LLM/TTS live flow and trace naming locally before external webhook expansion. + +## Prerequisites + +- Docker + Docker Compose +- BYOK provider credentials for your voice bundle providers +- EfficientAI OTLP credentials: + - `EFFICIENT_AI_API_KEY` + - `EFFICIENT_AI_AGENT_ID` (or `EFFICIENT_AI_PROJECT_ID`) + +## Core environment + +Set: + +- `OBSERVABILITY_ENABLED=true` +- `OBSERVABILITY_TRACING_ENABLED=true` +- `OBSERVABILITY_TRACING_EXPORTER=efficientai_http` +- `OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-http.efficientai.ai/v1/traces` + +Optional local debug: + +- `OBSERVABILITY_TRACING_EXPORTER=console` + +## Run path + +1. Start local stack (`api`, `db`, `redis`, `frontend`) with compose. +2. Open playground / voice agent websocket flow. +3. Start one live call through `run_voice_bundle_fastapi`. +4. Confirm call end stores metadata in `CallRecording`. +5. Open call detail and verify trace fetch / waterfall rendering. + +## Troubleshooting trace fetch + +### Tempo is running but UI shows “Trace not found” + +These are different problems: + +| Check | Command / signal | +|-------|------------------| +| Tempo process up | `curl -sf http://localhost:3200/ready` → 200 | +| Spans actually stored | `curl "http://localhost:3200/api/search?limit=5"` → `traces` non-empty | +| Trace for this call | `curl "http://localhost:3200/api/traces/{trace_id}"` → 200 with batches | + +A **`trace_id` on the call record only means the pipeline created a local OTel trace.** Spans must still be exported to Tempo during the call (`exporter: tempo_http`, endpoint `http://localhost:4318/v1/traces`) from the **media** process (`:8001` when using `start-all`). + +Common causes of 404 on trace query: + +- Tempo was **paused/stopped** during the call (export failed; ID still saved on call). +- Trace **expired** — default retention is **24h** (`observability/tempo/tempo.yml`). +- Tempo volume was **recreated** (empty store; old `trace_id` links remain in Postgres). +- API/media not using tracing config (`observability.tracing.enabled: false`). + +After fixing export, run a **new** test call and verify the trace exists in Tempo before opening Observability. + +### API returned 502 for traces (legacy) + +Older builds mapped Tempo 404 → HTTP 502. Current API returns **404** with an explicit “not found in trace store” message when Tempo is reachable but empty. + +## Validation checklist + +- call emits `conversation -> turn -> stt/llm/tts` +- attributes map to span contract +- call has a persisted `trace_id` +- trace endpoint resolves the same `trace_id` +- summary endpoint reports call counts and durations diff --git a/docs/telemetry/pii-policy.md b/docs/telemetry/pii-policy.md new file mode 100644 index 00000000..4b195301 --- /dev/null +++ b/docs/telemetry/pii-policy.md @@ -0,0 +1,32 @@ +# PII Policy for Traces + +This policy applies to Product Observability traces and call metadata. + +## Never store + +- API keys +- auth tokens +- raw credential headers +- secrets from provider integrations + +## Transcript handling + +- `stt.transcript` can contain user speech and may include PII. +- Default stance: allow transcript attributes for controlled environments. +- For strict environments, disable transcript attribute export and keep only timing/provider metadata. + +## Recommended controls + +- Add configuration flags to disable transcript payload attributes. +- Truncate long text attributes in UI and API responses. +- Restrict trace query access to workspace-scoped users. + +## Logging boundaries + +- Do not mirror full transcript text into generic service logs by default. +- Keep detailed transcript payloads in controlled call records or approved stores. + +## Compliance posture + +- Treat call transcript data as sensitive operational data. +- Preserve tenant isolation using `organization_id` and workspace-level access checks. diff --git a/docs/telemetry/platform-ops-later.md b/docs/telemetry/platform-ops-later.md new file mode 100644 index 00000000..9a2a96ec --- /dev/null +++ b/docs/telemetry/platform-ops-later.md @@ -0,0 +1,21 @@ +# Platform Ops Backlog (Later Rung) + +This is intentionally outside Product Observability V1 delivery. + +## Deferred scope + +- Prometheus metrics hardening for API/media/worker +- Celery trace propagation and queue saturation telemetry +- KEDA/elastic worker autoscaling policies +- PostHog product analytics integration + +## Guardrails + +- keep product trace shipping independent of platform dashboard rollout +- avoid coupling customer call latency to internal metrics pipelines + +## Entry criteria + +- V1 call traces are stable in production +- phase 2 webhooks and trace linkage are complete +- load test validates collector and media split behavior diff --git a/docs/telemetry/provider-webhook-map.md b/docs/telemetry/provider-webhook-map.md new file mode 100644 index 00000000..d4a1c99b --- /dev/null +++ b/docs/telemetry/provider-webhook-map.md @@ -0,0 +1,73 @@ +# Provider Webhook Mapping (Phase 2) + +## Supported endpoints + +- `POST /api/v1/observability/calls/webhook/{api_key}` (flat payload) +- `POST /api/v1/observability/calls/webhook/retell/{api_key}` +- `POST /api/v1/observability/calls/webhook/elevenlabs/{api_key}` +- `POST /api/v1/observability/calls/webhook/vapi/{api_key}` + +## Required normalized fields + +- `provider_call_id` +- `provider_platform` +- `event` +- optional `trace_id` + +## Retell + +- accepts native shape: `{ "event": "...", "call": { ... } }` +- `call.call_id` or `call.id` maps to provider call ID +- terminal Retell events are normalized to `call_ended` +- if terminal payload is incomplete (missing transcript/analysis/cost), EfficientAI performs a one-shot provider pull refresh before returning + +## ElevenLabs + +- accepts provider payload, then normalizes platform to `elevenlabs` +- if payload is flat with `id`/`call_id`, route wraps to internal `call` shape +- supports OpenTelemetry webhook type `post_call_transcription_otel`: + - expected shape: `{ "type": "post_call_transcription_otel", "data": { "conversation_id", "agent_id", "otlp_traces" } }` + - persists `call_data.provider_trace` with: + - `source` / `trace_source` + - `trace_id` + - `storage` (`inline` or `s3`) + - `normalized_trace` (UI-ready trace payload) + - optional `otlp_traces` (inline raw OTLP when small) + - optional `trace_s3_key` (raw archive for large payloads) + - stores trace source as `elevenlabs_post_call_webhook` + - keeps provider span names (e.g. `elevenlabs.recv.user_transcript`) to avoid taxonomy collisions + +### ElevenLabs webhook setup checklist + +1. In ElevenLabs workspace settings, configure a post-call webhook URL: + `POST /api/v1/observability/calls/webhook/elevenlabs/{api_key}` +2. Enable `events: ["transcript"]`. +3. Set transcript format to `opentelemetry`. +4. Ensure your ElevenLabs API key has `CONVAI_READ` scope for fallback GET trace fetches. + +## Vapi + +- accepts provider payload, then normalizes platform to `vapi` +- same fallback wrapping behavior as ElevenLabs path +- terminal statuses (`ended`, `completed`, `end-of-call-report`, `done`, `failed`) are normalized to `call_ended` +- if terminal payload is incomplete (missing transcript/analysis/cost sections), EfficientAI performs a one-shot provider pull refresh before returning + +## Trace linking + +Every provider payload should include: + +- `trace_id` at top-level, or +- `trace_id` inside provider call payload + +The backend persists `CallRecording.trace_id` when supplied. + +For hosted providers, EfficientAI also persists provider traces in `call_data.provider_trace` +on terminal webhook ingest, refresh, or first trace fetch. + +## Live runtime events (LiveKit / Pipecat / external) + +- `POST /api/v1/observability/live/events` accepts the incremental envelope described in + `docs/telemetry/live-event-contract.md`. +- Delivery is at-least-once, idempotent by `(organization_id, event_id)`. +- Events update `call_data` incrementally (no full overwrite) and keep existing provider data intact. +- Missing external `trace_id` values are backfilled by EfficientAI in the ingest ACK. diff --git a/docs/telemetry/retell-integration.md b/docs/telemetry/retell-integration.md new file mode 100644 index 00000000..95ef5e41 --- /dev/null +++ b/docs/telemetry/retell-integration.md @@ -0,0 +1,79 @@ +# Retell Integration and Call Observability + +This guide captures the Retell Level 1 integration flow: + +1. Load provider agents into EfficientAI integration pickers. +2. Ingest Retell call payloads via webhook. +3. Reconcile incomplete webhook payloads with on-demand provider refresh. + +## Required Retell scope + +- API key with access to: + - `agent.list` and `agent.retrieve` + - `call.retrieve` + +## Provider agent catalog + +EfficientAI endpoint: + +- `GET /api/v1/integrations/{integration_id}/external-agents` + +For Retell integrations this normalizes provider agents to: + +```json +{ + "agents": [ + { "id": "agent_...", "name": "Support Agent", "archived": false } + ], + "has_more": false, + "next_cursor": null +} +``` + +## Webhook ingest + +Retell webhook endpoint: + +- `POST /api/v1/observability/calls/webhook/retell/{api_key}` + +Expected native shape: + +```json +{ + "event": "call_ended", + "call": { "call_id": "...", "...": "..." } +} +``` + +Behavior: + +- Terminal Retell events are normalized to `call_event=call_ended` +- If a terminal payload is incomplete (missing transcript/call analysis/call cost), EfficientAI performs one pull fallback via `call.retrieve` and upserts richer `call_data` + +## Manual refresh endpoint + +To force a provider re-pull from observability detail: + +- `POST /api/v1/observability/calls/{call_short_id}/refresh` + +This resolves the linked integration and refreshes `call_data` using the provider call ID. + +## Level 1 dashboard expectations + +Retell call detail should include: + +- Call summary and sentiment (`call_analysis.*`) +- Cost and product costs (`call_cost.*`) +- Latency buckets (`latency.e2e/asr/llm/tts`) +- Transcript turns (`transcript_object` / transcript text) +- Recording links (`recording_url`, `recording_multi_channel_url`) +- Archived playback copies are stored in S3 as `call_data.recording_s3_key` when S3 is configured + +Trace note: + +- Level 1 uses provider call report payloads. +- Level 2 synthetic provider traces are available for Retell when `transcript_object` and/or `latency` are present in stored `call_data` (`trace_source=retell_synthetic`). +- Retell synthetic traces are persisted to `call_data.provider_trace` on terminal webhook ingest, sparse-enrich fallback, and manual refresh. +- `GET /calls/{call_short_id}/trace` serves stored provider traces first and only rebuilds when no persisted trace exists. +- Recording playback is served from S3 (`recording_s3_key`) after provider audio is archived on ingest, refresh, or first `/audio` request. +- Level 3 EfficientAI-native OTEL traces are separate scope. diff --git a/docs/telemetry/scale-architecture.md b/docs/telemetry/scale-architecture.md new file mode 100644 index 00000000..c973ddd0 --- /dev/null +++ b/docs/telemetry/scale-architecture.md @@ -0,0 +1,47 @@ +# Scale Architecture One-Pager (~10M calls/day) + +## Baseline model + +- Media plane handles live WebSockets and in-process span creation. +- API plane handles CRUD, call metadata, and trace query proxy. +- Worker plane handles async batch tasks (imports/evals), not real-time media. +- OTel Collector fleet handles telemetry ingestion buffering, retries, and fan-out. + +## Flow + +```text +Client/WebRTC/Telephony + -> media replicas (SERVICE_MODE=media) + -> spans (non-blocking, batch processor) + -> OTel Collector fleet + -> EfficientAI trace store (and optional Tempo) + +call-end metadata -> API/DB (single write per call) +``` + +## Operational rules + +- No per-span Postgres writes. +- Use `BatchSpanProcessor`; never block media thread on exporter. +- Apply sampling by tier/org (for example 10-20% default, 100% premium). +- Prefer dropping spans over dropping calls under pressure. + +## Capacity strategy + +- Scale media replicas on concurrent sessions. +- Scale collectors on span ingress throughput. +- Keep API isolated from media spikes. + +## Quotas and fairness + +- Per-org call and trace budget controls. +- Backpressure and shed-load behavior should be explicit and observable. +- Local/OSS config hooks: + - `OBSERVABILITY_TRACING_SAMPLE_RATE` for baseline sampling. + - `OBSERVABILITY_TRACE_QUOTA_PER_ORG_PER_DAY` to emit quota warnings without blocking calls. + +## Suggested tier defaults + +- Default orgs: sample 10-20% of traces. +- Premium orgs: sample 100% of traces. +- Under sustained pressure, drop spans before impacting live call handling. diff --git a/docs/telemetry/span-contract.md b/docs/telemetry/span-contract.md new file mode 100644 index 00000000..e66a169d --- /dev/null +++ b/docs/telemetry/span-contract.md @@ -0,0 +1,66 @@ +# Span Contract (V1) + +This contract is optimized for EfficientAI call-trace UI rendering. + +## Required hierarchy + +```text +conversation +└── turn + ├── stt + ├── llm + └── tts +``` + +`turn` spans are optional in the frontend rendering path, but are strongly preferred. + +## Stable span names + +- `conversation` (root) +- `turn` +- `stt` +- `llm` +- `tts` + +Phase 2: + +- `s2s` +- `tool_call` + +## Required attributes + +### `conversation` + +- `conversation.id` +- `organization_id` +- `workspace_id` +- `agent_id` + +### `turn` + +- `turn.number` + +### `stt` + +- `stt.provider` +- `stt.transcript` (subject to transcript policy) +- `gen_ai.request.model` (when available) + +### `llm` + +- `gen_ai.system` +- `gen_ai.request.model` +- `gen_ai.usage.input_tokens` +- `gen_ai.usage.output_tokens` + +### `tts` + +- `tts.provider` +- `tts.characters` +- `gen_ai.request.model` (when available) + +## Notes + +- Do not add spans for internal non-service transforms. +- Keep names stable; UI styling and rollups depend on this. +- If an existing attribute key already exists under a different legacy name, dual-write both keys during migration windows. diff --git a/docs/telemetry/taxonomy.md b/docs/telemetry/taxonomy.md new file mode 100644 index 00000000..68348dc8 --- /dev/null +++ b/docs/telemetry/taxonomy.md @@ -0,0 +1,69 @@ +# Taxonomy: Product vs Platform Observability + +EfficientAI has two observability layers that share telemetry primitives but solve different problems. + +## Product Observability (this epic) + +Primary user: customer teams and agent builders. + +Questions answered: + +- Which part of a call was slow (STT, LLM, TTS)? +- Why did this specific call fail? +- What was the trace for this call ID? + +Primary entities: + +- `CallRecording` +- `trace_id` +- call detail UI +- trace waterfall UI + +Signals: + +- traces and call metadata +- call-level aggregate stats + +### Provider trace namespaces + +To avoid span naming conflicts between EfficientAI-native traces and provider traces: + +- EfficientAI voice-bundle traces retain canonical names: + - `conversation`, `turn`, `stt`, `llm`, `tts`, `tool_call` +- Provider traces keep provider-native names: + - ElevenLabs examples: `elevenlabs.conversation`, `elevenlabs.recv.user_transcript`, `elevenlabs.recv.agent_response` +- Provider-derived synthetic rows (if added) must stay namespaced: + - `elevenlabs.metric.asr`, `elevenlabs.metric.llm`, `elevenlabs.metric.tts` + +Every normalized provider span should include `attributes.trace.provider` to make source-aware UI rendering deterministic. + +## Platform Observability (later epic) + +Primary user: platform and SRE operators. + +Questions answered: + +- Are API/media/worker services healthy? +- Is queue depth growing? +- Are collectors dropping spans? + +Primary entities: + +- service-level metrics and logs +- infra dashboards and alerts + +Signals: + +- Prometheus metrics +- Loki logs +- collector/process health + +## Shared correlation spine + +Both layers should preserve these keys end-to-end: + +- `organization_id` +- `workspace_id` +- `agent_id` +- `trace_id` +- `provider_call_id` diff --git a/docs/telemetry/vapi-integration.md b/docs/telemetry/vapi-integration.md new file mode 100644 index 00000000..78818da9 --- /dev/null +++ b/docs/telemetry/vapi-integration.md @@ -0,0 +1,90 @@ +# Vapi Integration and Call Observability + +This document describes the Vapi Level 1 integration path for: + +1. Loading provider agents into EfficientAI dropdowns. +2. Capturing Vapi call payloads through webhook and on-demand refresh. +3. Validating call metrics, transcript, and dashboard coverage in observability. + +## Required Vapi keys + +- Private API key: + - Server-side call metrics retrieval (`GET /call/{id}`) + - Assistant listing (`GET /assistant`) +- Public key: + - Required by Vapi web-call creation (`POST /call/web`) + +## Provider agent catalog + +EfficientAI backend endpoint: + +- `GET /api/v1/integrations/{integration_id}/external-agents` + +For Vapi integrations, this proxies: + +- `GET https://api.vapi.ai/assistant` + +and returns normalized rows: + +```json +{ + "agents": [ + { + "id": "assist_...", + "name": "Support Assistant", + "archived": false + } + ], + "has_more": false, + "next_cursor": null +} +``` + +## Webhook ingest + +Vapi webhook endpoint: + +- `POST /api/v1/observability/calls/webhook/vapi/{api_key}` + +Expected behavior: + +- Ingest provider payload and normalize platform to `vapi` +- For terminal events (`ended`, `completed`, `end-of-call-report`, `done`, `failed`), normalize `call_event` to `call_ended` +- If terminal payload is incomplete (missing transcript/analysis/cost sections), EfficientAI performs one provider pull fallback (`GET /call/{id}`) and upserts the richer payload + +Recommended Vapi dashboard setup: + +1. Configure your server URL to the endpoint above. +2. Ensure end-of-call payloads are enabled. +3. Keep provider call IDs and assistant IDs included in webhook payloads. + +## Manual refresh endpoint + +When webhook payloads are delayed or partial, use: + +- `POST /api/v1/observability/calls/{call_short_id}/refresh` + +Refresh flow: + +1. Resolve call recording and linked internal agent. +2. Resolve voice integration and decrypt private key. +3. Pull latest provider payload from Vapi (`GET /call/{provider_call_id}`). +4. Overwrite `call_data` with the refreshed provider payload. + +## What should appear in call detail (Level 1) + +From the Vapi provider payload: + +- Summary and success evaluation (`analysis.*`) +- Cost and token usage (`cost`, `costBreakdown.*`) +- Latency and interruption metrics (`analysis.latencyStats`, `artifact.performanceMetrics`) +- Transcript turns (`messages`, `artifact.messages`, fallback transcript text) +- Recording references (`recordingUrl`, `stereoRecordingUrl`, artifact recording URLs) + +Trace note: + +- Level 1 focuses on provider call reports. +- Level 2 synthetic traces (`trace_source=vapi_synthetic`) are persisted on webhook terminal ingest, sparse enrich fallback, and manual refresh. +- Trace stats include endpointing where Vapi exposes it (`metric.layer=endpointing`). +- Stored provider traces are served first; rebuilds happen only when persisted traces are missing. +- EfficientAI-native OTEL traces remain Level 3 scope. diff --git a/env.example b/env.example index 40b2f1a5..78f3ca8b 100644 --- a/env.example +++ b/env.example @@ -69,6 +69,31 @@ AUTH_LOCAL_ALLOW_SIGNUP=true # Enterprise license JWT (RS256). Obtain from the EfficientAI team. # EFFICIENTAI_LICENSE=eyJhbGciOi... +# Product observability tracing (OTLP HTTP -> EfficientAI) +OBSERVABILITY_ENABLED=false +OBSERVABILITY_TRACING_ENABLED=false +OBSERVABILITY_TRACING_EXPORTER=efficientai_http +OBSERVABILITY_TRACING_SAMPLE_RATE=1.0 +OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS=true +OBSERVABILITY_TRACE_QUOTA_PER_ORG_PER_DAY= +OBSERVABILITY_LIVE_INGEST_ENABLED=false +OBSERVABILITY_LIVE_AGGREGATES_ENABLED=false +OBSERVABILITY_LIVE_DASHBOARD_ENABLED=false +OBSERVABILITY_LIVE_EVENT_IDEMPOTENCY_TTL_SECONDS=86400 +OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ=5 +OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS=300 +OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED=false +OBSERVABILITY_LIVE_SLO_AUTOMATION_ENABLED=false +OBSERVABILITY_LIVE_SLO_P90_LLM_MS=1800 +OBSERVABILITY_LIVE_SLO_MIN_SAMPLE_COUNT=20 +OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-http.efficientai.ai/v1/traces +TRACING_QUERY_BACKEND=cloud +EFFICIENT_AI_API_KEY= +EFFICIENT_AI_AGENT_ID= +# EFFICIENT_AI_PROJECT_ID= +# EFFICIENT_AI_TRACE_QUERY_URL=https://api.efficientai.ai/observability/v1/traces +# TEMPO_QUERY_URL=http://tempo:3200 + # ----------------------------------------------------------------------------- # Usage cost flush (Celery Beat + worker-usage queue) # Platform schedules: Beat enqueues flush/alerts/FX/prune; worker-usage runs flush tasks. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b62b6e72..17e1fd4a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -63,6 +63,7 @@ import EvaluationsList from './pages/evaluators/results/EvaluationsList' import Observability from './pages/observability/Observability' import ObservabilityCalls from './pages/observability/ObservabilityCalls' import ObservabilityCallDetail from './pages/observability/ObservabilityCallDetail' +import RouteErrorBoundary from './components/RouteErrorBoundary' // Alerting import Alerts from './pages/alerting/Alerts' @@ -195,7 +196,14 @@ function App() { } /> } /> } /> - } /> + + + + )} + /> } /> } /> } /> diff --git a/frontend/src/components/RouteErrorBoundary.tsx b/frontend/src/components/RouteErrorBoundary.tsx new file mode 100644 index 00000000..54fa23df --- /dev/null +++ b/frontend/src/components/RouteErrorBoundary.tsx @@ -0,0 +1,61 @@ +import { Component, type ErrorInfo, type ReactNode } from 'react' + +interface RouteErrorBoundaryProps { + children: ReactNode +} + +interface RouteErrorBoundaryState { + hasError: boolean + errorMessage: string +} + +export default class RouteErrorBoundary extends Component< + RouteErrorBoundaryProps, + RouteErrorBoundaryState +> { + constructor(props: RouteErrorBoundaryProps) { + super(props) + this.state = { hasError: false, errorMessage: '' } + } + + static getDerivedStateFromError(error: unknown): RouteErrorBoundaryState { + return { + hasError: true, + errorMessage: error instanceof Error ? error.message : 'Unknown rendering error', + } + } + + componentDidCatch(error: unknown, errorInfo: ErrorInfo) { + console.error('Route render error:', error, errorInfo) + } + + private handleReload = () => { + window.location.reload() + } + + render() { + if (!this.state.hasError) { + return this.props.children + } + + return ( +
+
+

This page hit a rendering error

+

+ We captured the error and prevented a blank screen. Please reload once. +

+

{this.state.errorMessage}

+ +
+
+ ) + } +} + diff --git a/frontend/src/components/observability/DualTrackWaveformPlayer.tsx b/frontend/src/components/observability/DualTrackWaveformPlayer.tsx new file mode 100644 index 00000000..51650140 --- /dev/null +++ b/frontend/src/components/observability/DualTrackWaveformPlayer.tsx @@ -0,0 +1,559 @@ +import { + forwardRef, + useCallback, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, + type ReactNode, +} from 'react' +import { Bot, Loader, Pause, Play, User } from 'lucide-react' +import { + formatPlaybackTime, + loadAudioPeaks, + peaksForTimeRange, + buildSyntheticPeaks, + type AudioPeakData, +} from '../../lib/audioWaveform' +import { + findSegmentIndexAtTime, + type WaveformSegment, + type WaveformSpeaker, +} from './waveformSegments' + +const SPEAKER_COLORS: Record< + WaveformSpeaker, + { stroke: string; fill: string; activeFill: string } +> = { + agent: { + stroke: '#16a34a', + fill: 'rgba(34, 197, 94, 0.55)', + activeFill: 'rgba(34, 197, 94, 0.85)', + }, + user: { + stroke: '#2563eb', + fill: 'rgba(59, 130, 246, 0.55)', + activeFill: 'rgba(59, 130, 246, 0.85)', + }, +} + +const SPEED_OPTIONS = [0.75, 1, 1.25, 1.5, 2] + +function drawSegmentWaveform( + ctx: CanvasRenderingContext2D, + x: number, + width: number, + height: number, + segmentPeaks: Float32Array, + colors: { stroke: string; fill: string }, + active: boolean, +) { + if (width < 2) return + const midY = height / 2 + const radius = 6 + const left = x + const right = x + width + const top = 4 + const bottom = height - 4 + const innerH = bottom - top + + ctx.beginPath() + ctx.moveTo(left + radius, top) + ctx.lineTo(right - radius, top) + ctx.quadraticCurveTo(right, top, right, top + radius) + ctx.lineTo(right, bottom - radius) + ctx.quadraticCurveTo(right, bottom, right - radius, bottom) + ctx.lineTo(left + radius, bottom) + ctx.quadraticCurveTo(left, bottom, left, bottom - radius) + ctx.lineTo(left, top + radius) + ctx.quadraticCurveTo(left, top, left + radius, top) + ctx.closePath() + ctx.fillStyle = active ? colors.fill.replace('0.55', '0.85') : colors.fill + ctx.fill() + + if (segmentPeaks.length === 0) return + + const barWidth = Math.max(1, width / segmentPeaks.length) + ctx.fillStyle = active ? '#ffffff' : colors.stroke + ctx.globalAlpha = active ? 0.95 : 0.75 + + for (let i = 0; i < segmentPeaks.length; i += 1) { + const amp = Math.max(0.08, segmentPeaks[i]) + const barH = amp * innerH * 0.9 + const bx = left + i * barWidth + barWidth * 0.15 + const bw = Math.max(1, barWidth * 0.7) + ctx.fillRect(bx, midY - barH / 2, bw, barH) + } + ctx.globalAlpha = 1 +} + +function TrackCanvas({ + speaker, + label, + icon, + segments, + peakData, + durationSec, + currentTimeSec, + activeSegmentIndex, + onSeek, + onSegmentClick, +}: { + speaker: WaveformSpeaker + label: string + icon: ReactNode + segments: WaveformSegment[] + peakData: AudioPeakData + durationSec: number + currentTimeSec: number + activeSegmentIndex: number | null + onSeek: (timeSec: number) => void + onSegmentClick: (segmentIndex: number, startSec: number) => void +}) { + const canvasRef = useRef(null) + const containerRef = useRef(null) + const speakerSegments = useMemo( + () => segments.filter((seg) => seg.speaker === speaker), + [segments, speaker], + ) + + const globalIndexBySpeakerIndex = useMemo(() => { + return speakerSegments.map((seg) => segments.indexOf(seg)) + }, [segments, speakerSegments]) + + const redraw = useCallback(() => { + const canvas = canvasRef.current + const container = containerRef.current + if (!canvas || !container || durationSec <= 0) return + + const width = container.clientWidth + const height = 52 + const dpr = window.devicePixelRatio || 1 + canvas.width = width * dpr + canvas.height = height * dpr + canvas.style.width = `${width}px` + canvas.style.height = `${height}px` + + const ctx = canvas.getContext('2d') + if (!ctx) return + ctx.setTransform(dpr, 0, 0, dpr, 0, 0) + ctx.clearRect(0, 0, width, height) + + ctx.strokeStyle = '#e5e7eb' + ctx.lineWidth = 1 + ctx.beginPath() + ctx.moveTo(0, height / 2) + ctx.lineTo(width, height / 2) + ctx.stroke() + + const colors = SPEAKER_COLORS[speaker] + speakerSegments.forEach((seg, speakerIdx) => { + const globalIdx = globalIndexBySpeakerIndex[speakerIdx] + const x = (seg.startSec / durationSec) * width + const segWidth = Math.max(4, ((seg.endSec - seg.startSec) / durationSec) * width) + const segPeaks = peaksForTimeRange( + peakData.peaks, + peakData.durationSec, + seg.startSec, + seg.endSec, + Math.max(12, Math.floor(segWidth / 3)), + ) + drawSegmentWaveform( + ctx, + x, + segWidth, + height, + segPeaks, + colors, + activeSegmentIndex === globalIdx, + ) + }) + + const playheadX = (currentTimeSec / durationSec) * width + ctx.strokeStyle = '#ef4444' + ctx.lineWidth = 2 + ctx.beginPath() + ctx.moveTo(playheadX, 0) + ctx.lineTo(playheadX, height) + ctx.stroke() + }, [ + activeSegmentIndex, + currentTimeSec, + durationSec, + globalIndexBySpeakerIndex, + peakData, + speaker, + speakerSegments, + ]) + + useEffect(() => { + redraw() + }, [redraw]) + + useEffect(() => { + const container = containerRef.current + if (!container) return + const observer = new ResizeObserver(() => redraw()) + observer.observe(container) + return () => observer.disconnect() + }, [redraw]) + + const handleClick = (event: React.MouseEvent) => { + const rect = event.currentTarget.getBoundingClientRect() + const ratio = Math.min(1, Math.max(0, (event.clientX - rect.left) / rect.width)) + const timeSec = ratio * durationSec + + const clickedSpeakerIdx = speakerSegments.findIndex( + (seg) => timeSec >= seg.startSec && timeSec <= seg.endSec, + ) + if (clickedSpeakerIdx >= 0) { + const globalIdx = globalIndexBySpeakerIndex[clickedSpeakerIdx] + onSegmentClick(globalIdx, speakerSegments[clickedSpeakerIdx].startSec) + } else { + onSeek(timeSec) + } + } + + return ( +
+
+ {icon} + {label} +
+
+ +
+
+ ) +} + +export interface DualTrackWaveformPlayerHandle { + seek: (timeSec: number) => void + play: () => void + pause: () => void +} + +export interface DualTrackWaveformPlayerProps { + audioUrl: string + segments: WaveformSegment[] + agentLabel?: string + userLabel?: string + activeSegmentIndex?: number | null + fallbackDurationSec?: number | null + liveMode?: boolean + liveDurationSec?: number | null + onTimeUpdate?: (timeSec: number) => void + onSegmentActive?: (segmentIndex: number | null) => void + onSegmentClick?: (segmentIndex: number, startSec: number) => void +} + +const DualTrackWaveformPlayer = forwardRef( + function DualTrackWaveformPlayer( + { + audioUrl, + segments, + agentLabel = 'Agent', + userLabel = 'Customer', + activeSegmentIndex = null, + fallbackDurationSec = null, + liveMode = false, + liveDurationSec = null, + onTimeUpdate, + onSegmentActive, + onSegmentClick, + }, + ref, + ) { + const audioRef = useRef(null) + const [peakData, setPeakData] = useState(null) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [isPlaying, setIsPlaying] = useState(false) + const [audioReady, setAudioReady] = useState(false) + const [playbackError, setPlaybackError] = useState(null) + const [currentTimeSec, setCurrentTimeSec] = useState(0) + const [mediaDurationSec, setMediaDurationSec] = useState(0) + const [playbackRate, setPlaybackRate] = useState(1) + + const durationSec = + mediaDurationSec > 0 + ? mediaDurationSec + : liveDurationSec && liveDurationSec > 0 + ? liveDurationSec + : (peakData?.durationSec ?? 0) + + const estimateSegmentDuration = useCallback(() => { + const lastSeg = segments[segments.length - 1] + return ( + fallbackDurationSec ?? + liveDurationSec ?? + lastSeg?.endSec ?? + segments.reduce((max, s) => Math.max(max, s.endSec), 0) ?? + 60 + ) + }, [fallbackDurationSec, liveDurationSec, segments]) + + useEffect(() => { + let cancelled = false + + if (liveMode) { + const duration = Math.max(estimateSegmentDuration(), 1) + setPeakData(buildSyntheticPeaks(duration)) + setLoading(false) + setError(null) + return () => { + cancelled = true + } + } + + setLoading(true) + setError(null) + loadAudioPeaks(audioUrl) + .then((data) => { + if (!cancelled) { + setPeakData(data) + setLoading(false) + } + }) + .catch(() => { + if (!cancelled) { + setPeakData(buildSyntheticPeaks(Math.max(estimateSegmentDuration(), 1))) + setError(null) + setLoading(false) + } + }) + return () => { + cancelled = true + } + }, [audioUrl, estimateSegmentDuration, liveMode]) + + const playAudio = useCallback(async () => { + const audio = audioRef.current + if (!audio) return + setPlaybackError(null) + try { + await audio.play() + } catch { + setPlaybackError('Could not start playback. The recording may still be loading.') + } + }, []) + + useEffect(() => { + const audio = audioRef.current + if (!audio || !audioUrl) return + + if (liveMode) { + const wasPlaying = !audio.paused + const prevTime = audio.currentTime + if (audio.src !== audioUrl) { + audio.src = audioUrl + } + const onReady = () => { + if (Number.isFinite(audio.duration) && audio.duration > 0) { + setMediaDurationSec(audio.duration) + } else if (liveDurationSec && liveDurationSec > 0) { + setMediaDurationSec(liveDurationSec) + } + setAudioReady(true) + setPlaybackError(null) + if (Number.isFinite(prevTime) && prevTime > 0) { + const cap = audio.duration > 0 ? audio.duration : prevTime + audio.currentTime = Math.min(prevTime, cap) + setCurrentTimeSec(audio.currentTime) + } + if (wasPlaying) void playAudio() + } + audio.addEventListener('loadedmetadata', onReady, { once: true }) + audio.load() + return () => audio.removeEventListener('loadedmetadata', onReady) + } + + setAudioReady(false) + setIsPlaying(false) + setPlaybackError(null) + setCurrentTimeSec(0) + setMediaDurationSec(0) + + if (audio.src !== audioUrl) { + audio.src = audioUrl + } + audio.load() + }, [audioUrl, liveDurationSec, liveMode, playAudio]) + + const seek = useCallback( + (timeSec: number) => { + const audio = audioRef.current + if (!audio || durationSec <= 0) return + audio.currentTime = Math.min(durationSec, Math.max(0, timeSec)) + setCurrentTimeSec(audio.currentTime) + onTimeUpdate?.(audio.currentTime) + const idx = findSegmentIndexAtTime(segments, audio.currentTime) + onSegmentActive?.(idx) + }, + [durationSec, onSegmentActive, onTimeUpdate, segments], + ) + + useImperativeHandle( + ref, + () => ({ + seek, + play: () => { + void playAudio() + }, + pause: () => audioRef.current?.pause(), + }), + [playAudio, seek], + ) + + useEffect(() => { + const audio = audioRef.current + if (!audio) return + audio.playbackRate = playbackRate + }, [playbackRate]) + + const togglePlay = () => { + const audio = audioRef.current + if (!audio || !audioReady) return + if (audio.paused) void playAudio() + else audio.pause() + } + + const handleTimeUpdate = () => { + const audio = audioRef.current + if (!audio) return + setCurrentTimeSec(audio.currentTime) + onTimeUpdate?.(audio.currentTime) + const idx = findSegmentIndexAtTime(segments, audio.currentTime) + onSegmentActive?.(idx) + } + + const handleLoadedMetadata = () => { + const audio = audioRef.current + if (!audio) return + if (Number.isFinite(audio.duration) && audio.duration > 0) { + setMediaDurationSec(audio.duration) + } + setAudioReady(true) + setPlaybackError(null) + } + + const handleSegmentClick = (segmentIndex: number, startSec: number) => { + seek(startSec) + onSegmentClick?.(segmentIndex, startSec) + void playAudio() + } + + if (loading) { + return ( +
+ + Loading waveform… +
+ ) + } + + if (error || !peakData) { + return ( +
+ Could not render waveform. Using basic audio controls. +
+ ) + } + + return ( +
+
+ ) + }, +) + +export default DualTrackWaveformPlayer diff --git a/frontend/src/components/observability/TraceRecordingPanel.tsx b/frontend/src/components/observability/TraceRecordingPanel.tsx new file mode 100644 index 00000000..d2bfbf48 --- /dev/null +++ b/frontend/src/components/observability/TraceRecordingPanel.tsx @@ -0,0 +1,271 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { Download, GitBranch, Loader, AlertCircle } from 'lucide-react' +import type { ObservabilityCallTrace, ObservabilityTraceSpan } from '../../types/api' +import TraceTree from './TraceTree' +import DualTrackWaveformPlayer, { + type DualTrackWaveformPlayerHandle, +} from './DualTrackWaveformPlayer' +import { + buildSpanTree, + findNearestTurnSpan, + flattenSpanTree, + getTraceRootStartMs, + spanOffsetSec, +} from './traceDisplay' +import type { WaveformSegment } from './waveformSegments' +import { useObservabilityCallAudioBlob } from '../../hooks/useObservabilityCallAudioBlob' +import { useObservabilityLiveAudio } from '../../hooks/useObservabilityLiveAudio' + +export interface TraceRecordingPanelProps { + callShortId: string + traceId: string | null + callTrace: ObservabilityCallTrace | undefined + traceLoading: boolean + traceError: boolean + traceErrorDisplay: { title: string; body: string; hint?: string } + playbackUrl: string | null + audioLoading: boolean + isLiveCall?: boolean + callStartMs: number | null + waveformSegments: WaveformSegment[] + agentLabel: string + hasStorageRecording: boolean + fallbackDurationSec?: number | null + onRefreshTrace: () => void + selectedSpanId?: string | null + onSelectedSpanIdChange?: (spanId: string | null) => void +} + +export default function TraceRecordingPanel({ + callShortId, + traceId, + callTrace, + traceLoading, + traceError, + traceErrorDisplay, + playbackUrl, + audioLoading, + isLiveCall = false, + callStartMs, + waveformSegments, + agentLabel, + hasStorageRecording, + fallbackDurationSec, + onRefreshTrace, + selectedSpanId: externalSelectedSpanId, + onSelectedSpanIdChange, +}: TraceRecordingPanelProps) { + const waveformRef = useRef(null) + const [audioCurrentTimeSec, setAudioCurrentTimeSec] = useState(0) + const [internalSelectedSpanId, setInternalSelectedSpanId] = useState(null) + const selectedSpanId = externalSelectedSpanId ?? internalSelectedSpanId + + const { data: blobAudioUrl, isLoading: blobLoading } = useObservabilityCallAudioBlob( + callShortId, + hasStorageRecording && !isLiveCall, + ) + + const { + data: liveAudio, + isLoading: liveAudioLoading, + isFetching: liveAudioFetching, + isError: liveAudioUnavailable, + } = useObservabilityLiveAudio(callShortId, isLiveCall) + + const waveformAudioUrl = isLiveCall ? liveAudio?.blobUrl : blobAudioUrl || playbackUrl + const liveDurationSec = isLiveCall ? liveAudio?.durationSec ?? null : null + + const allSpans = useMemo(() => { + if (!callTrace) return [] + return flattenSpanTree(buildSpanTree(callTrace)) + }, [callTrace]) + + const rootStartMs = useMemo(() => getTraceRootStartMs(allSpans), [allSpans]) + + const setSelectedSpanId = useCallback( + (spanId: string | null) => { + if (onSelectedSpanIdChange) onSelectedSpanIdChange(spanId) + else setInternalSelectedSpanId(spanId) + }, + [onSelectedSpanIdChange], + ) + + const seekAudio = useCallback((sec: number) => { + waveformRef.current?.seek(sec) + }, []) + + const handleSelectSpan = useCallback( + (span: ObservabilityTraceSpan) => { + if (!span.span_id) return + setSelectedSpanId(span.span_id) + if (span.name === 'turn' || span.name === 'conversation') { + const offset = spanOffsetSec(span, callStartMs, rootStartMs) + if (offset !== null) seekAudio(offset) + } + }, + [callStartMs, rootStartMs, seekAudio, setSelectedSpanId], + ) + + const handleAudioTimeUpdate = useCallback( + (timeSec: number) => { + setAudioCurrentTimeSec(timeSec) + if (!callTrace || allSpans.length === 0) return + const nearest = findNearestTurnSpan(timeSec, callStartMs, rootStartMs, allSpans) + if (nearest?.span_id && nearest.span_id !== selectedSpanId) { + setSelectedSpanId(nearest.span_id) + } + }, + [allSpans, callStartMs, callTrace, rootStartMs, selectedSpanId, setSelectedSpanId], + ) + + useEffect(() => { + if (!externalSelectedSpanId) return + const span = allSpans.find((s) => s.span_id === externalSelectedSpanId) + if (!span?.span_id) return + if (internalSelectedSpanId !== externalSelectedSpanId && onSelectedSpanIdChange === undefined) { + setInternalSelectedSpanId(externalSelectedSpanId) + } + if (waveformAudioUrl && (span.name === 'turn' || span.name === 'conversation')) { + const offset = spanOffsetSec(span, callStartMs, rootStartMs) + if (offset !== null) seekAudio(offset) + } + }, [ + externalSelectedSpanId, + allSpans, + callStartMs, + rootStartMs, + waveformAudioUrl, + seekAudio, + onSelectedSpanIdChange, + internalSelectedSpanId, + ]) + + const liveAudioBlobUrl = liveAudio?.blobUrl ?? null + const effectiveTraceId = traceId || callTrace?.trace_id || null + + const recordingLoading = isLiveCall + ? liveAudioLoading && !liveAudioBlobUrl + : audioLoading || (hasStorageRecording && blobLoading && !blobAudioUrl) + + return ( +
+ {(waveformAudioUrl || recordingLoading || (isLiveCall && !liveAudioUnavailable)) && ( +
+ {recordingLoading ? ( +
+ + {isLiveCall ? 'Waiting for live audio…' : 'Loading recording…'} +
+ ) : waveformAudioUrl ? ( + <> + + {isLiveCall && liveAudioFetching && ( +

Refreshing live audio…

+ )} + {!isLiveCall && playbackUrl && ( + + )} + {callTrace && callTrace.spans.length > 0 && ( +

+ {isLiveCall + ? 'Live audio and trace update during the call. Scrub the waveform or click turns to seek.' + : 'Scrub the waveform to jump between turns, or click a turn in the trace tree to seek audio.'} +

+ )} + + ) : isLiveCall ? ( +
+ Live audio will appear once the call captures enough audio. +
+ ) : null} +
+ )} + +
+
+

+ + Execution Trace +

+ {effectiveTraceId && ( + + )} +
+ + {traceLoading ? ( +
+ + Loading trace spans… +
+ ) : traceError ? ( +
+
+ +
+

{traceErrorDisplay.title}

+

+ Trace ID {effectiveTraceId ?? 'N/A'}. {traceErrorDisplay.body} +

+ {traceErrorDisplay.hint && ( +

{traceErrorDisplay.hint}

+ )} +
+
+
+ ) : callTrace && callTrace.spans.length > 0 ? ( + + ) : !effectiveTraceId ? ( +
+ +

No trace linked to this call

+

+ Internal EfficientAI voice-bundle calls record a trace automatically when tracing is enabled. +

+
+ ) : ( +
+

No spans found for this trace

+

+ Trace ID {effectiveTraceId ?? 'N/A'} was saved, but the trace store returned no spans. +

+
+ )} +
+
+ ) +} diff --git a/frontend/src/components/observability/TraceTree.tsx b/frontend/src/components/observability/TraceTree.tsx new file mode 100644 index 00000000..c86871b7 --- /dev/null +++ b/frontend/src/components/observability/TraceTree.tsx @@ -0,0 +1,541 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { + ChevronDown, + ChevronRight, + Layers, + MessageSquare, + Mic, + Sparkles, + Volume2, + Wrench, + Zap, + ArrowLeftRight, + Circle, + ChevronsDownUp, + ChevronsUpDown, +} from 'lucide-react' +import type { ObservabilityCallTrace, ObservabilityTraceSpan } from '../../types/api' +import { + buildSpanTree, + computeTraceStats, + enrichSpanAttributes, + flattenSpanTree, + formatAttributeValue, + formatDuration, + formatRelativeOffset, + formatSpanTimestamp, + getAllCollapsibleSpanIds, + getDefaultCollapsedSpanIds, + getServiceName, + getSpanDisplayName, + getSpanKind, + getSpanSummaryLines, + getStatusLabel, + getTraceRootStartMs, + getTurnPreview, + isElevenLabsTurnSpan, + partitionAttributes, + spanOffsetSec, + truncateSpanId, + type SpanTreeNode, +} from './traceDisplay' + +const SPAN_ICON_CLASS: Record = { + conversation: 'text-slate-500', + turn: 'text-violet-600', + stt: 'text-cyan-600', + llm: 'text-violet-500', + tts: 'text-emerald-600', + s2s: 'text-fuchsia-600', + tool_call: 'text-amber-600', + endpointing: 'text-gray-500', +} + +function SpanTypeIcon({ name, className = 'h-3.5 w-3.5' }: { name: string; className?: string }) { + const color = + SPAN_ICON_CLASS[name] || + (name.startsWith('elevenlabs.recv.') ? 'text-violet-600' : undefined) || + (name.startsWith('elevenlabs.tool.') ? 'text-amber-600' : undefined) || + (name.startsWith('elevenlabs.metric.') ? 'text-cyan-600' : undefined) || + 'text-gray-400' + const props = { className: `${className} ${color} shrink-0` } + if (name.startsWith('elevenlabs.metric.')) { + return + } + if (name.startsWith('elevenlabs.tool.')) { + return + } + if (name === 'elevenlabs.recv.user_transcript') { + return + } + if (name === 'elevenlabs.recv.agent_response') { + return + } + if (name === 'elevenlabs.conversation') { + return + } + switch (name) { + case 'conversation': + return + case 'turn': + return + case 'stt': + return + case 'llm': + return + case 'tts': + return + case 's2s': + return + case 'tool_call': + return + case 'endpointing': + return + default: + return + } +} + +function InfoRow({ label, value }: { label: string; value: string }) { + return ( +
+ {label} + {value} +
+ ) +} + +function AttributeTable({ entries }: { entries: Array<[string, unknown]> }) { + if (entries.length === 0) { + return

No attributes recorded.

+ } + return ( +
+ {entries.map(([key, value]) => ( +
+ {key} + + {formatAttributeValue(value)} + +
+ ))} +
+ ) +} + +function SpanDetailPanel({ + span, + allSpans, + rootStartMs, + callStartMs, +}: { + span: ObservabilityTraceSpan + allSpans: ObservabilityTraceSpan[] + rootStartMs: number + callStartMs: number | null +}) { + const enriched = useMemo(() => enrichSpanAttributes(span, allSpans), [span, allSpans]) + const summaryLines = useMemo(() => getSpanSummaryLines(span, allSpans), [span, allSpans]) + const { highlighted, rest } = useMemo(() => partitionAttributes(enriched), [enriched]) + const [showAllAttrs, setShowAllAttrs] = useState(false) + const relativeOffset = span.start_time + ? formatRelativeOffset(span.start_time - rootStartMs) + : '—' + + return ( +
+
+
+
+
+ +

{getSpanDisplayName(span)}

+
+

{truncateSpanId(span.span_id)}

+

+ {relativeOffset} · {formatSpanTimestamp(span.start_time)} +

+
+
+

Latency

+

+ {formatDuration(span.duration_ms, 'seconds')} +

+
+
+
+ +
+ {summaryLines.length > 0 && ( +
+

Summary

+
+ {summaryLines.map((line) => ( +
+

+ {line.label} +

+

{line.value}

+
+ ))} +
+
+ )} + +
+

+ Basic Information +

+
+ + + + + + + + {callStartMs && span.start_time && ( + + )} + + +
+
+ +
+

+ Span Attributes +

+ + {rest.length > 0 && ( +
+ + {showAllAttrs && ( +
+ +
+ )} +
+ )} +
+
+
+ ) +} + +function SpanTreeRow({ + node, + depth, + selectedSpanId, + collapsedIds, + rootStartMs, + allSpans, + onSelect, + onToggle, +}: { + node: SpanTreeNode + depth: number + selectedSpanId: string | null + collapsedIds: Set + rootStartMs: number + allSpans: ObservabilityTraceSpan[] + onSelect: (spanId: string) => void + onToggle: (spanId: string) => void +}) { + if (!node.span_id) return null + + const spanId = node.span_id + const hasChildren = node.children.length > 0 + const isCollapsed = collapsedIds.has(spanId) + const isSelected = selectedSpanId === spanId + const durationLabel = + node.name === 'conversation' || node.name === 'turn' || isElevenLabsTurnSpan(node) + ? formatDuration(node.duration_ms, 'seconds') + : formatDuration(node.duration_ms, 'auto') + const relativeLabel = node.start_time + ? formatRelativeOffset(node.start_time - rootStartMs) + : null + const preview = node.name === 'turn' || isElevenLabsTurnSpan(node) ? getTurnPreview(node, allSpans) : null + + return ( + <> + + {hasChildren && + !isCollapsed && + node.children.map((child) => ( + + ))} + + ) +} + +export interface TraceTreeProps { + trace: ObservabilityCallTrace + embedded?: boolean + callStartMs?: number | null + audioCurrentTimeSec?: number | null + selectedSpanId?: string | null + onSelectSpan?: (span: ObservabilityTraceSpan) => void + showTimeline?: boolean + syncAudioToSelection?: boolean +} + +export default function TraceTree({ + trace, + embedded = false, + callStartMs = null, + audioCurrentTimeSec = null, + selectedSpanId: controlledSelectedSpanId, + onSelectSpan, + showTimeline: showTimelineProp = true, + syncAudioToSelection = true, +}: TraceTreeProps) { + const tree = useMemo(() => buildSpanTree(trace), [trace]) + const allSpans = useMemo(() => flattenSpanTree(tree), [tree]) + const stats = useMemo(() => computeTraceStats(allSpans), [allSpans]) + const rootStartMs = useMemo(() => getTraceRootStartMs(allSpans), [allSpans]) + + const [internalSelectedSpanId, setInternalSelectedSpanId] = useState(null) + const selectedSpanId = controlledSelectedSpanId ?? internalSelectedSpanId + + const [collapsedIds, setCollapsedIds] = useState>(() => getDefaultCollapsedSpanIds(tree)) + + useEffect(() => { + setCollapsedIds(getDefaultCollapsedSpanIds(tree)) + }, [trace.trace_id]) + + useEffect(() => { + if (controlledSelectedSpanId !== undefined) return + if (!internalSelectedSpanId && allSpans.length > 0) { + const firstTurn = allSpans.find((s) => (s.name === 'turn' || isElevenLabsTurnSpan(s)) && s.span_id) + const defaultId = firstTurn?.span_id ?? allSpans.find((s) => s.span_id)?.span_id ?? null + if (defaultId) setInternalSelectedSpanId(defaultId) + } + }, [allSpans, internalSelectedSpanId, controlledSelectedSpanId]) + + const setSelectedSpanId = useCallback( + (spanId: string) => { + if (controlledSelectedSpanId === undefined) { + setInternalSelectedSpanId(spanId) + } + const span = allSpans.find((s) => s.span_id === spanId) + if (span && onSelectSpan) onSelectSpan(span) + }, + [allSpans, controlledSelectedSpanId, onSelectSpan], + ) + + const selected = allSpans.find((s) => s.span_id === selectedSpanId) || null + + const toggleCollapsed = (spanId: string) => { + setCollapsedIds((prev) => { + const next = new Set(prev) + if (next.has(spanId)) next.delete(spanId) + else next.add(spanId) + return next + }) + } + + const expandAll = () => setCollapsedIds(new Set()) + const collapseAll = () => setCollapsedIds(new Set(getAllCollapsibleSpanIds(tree))) + + const pct = (part: number) => Math.round((part / stats.totalMs) * 100) + + const playheadOffsetSec = audioCurrentTimeSec ?? null + const playheadPct = + playheadOffsetSec !== null && stats.totalMs > 0 + ? Math.min(100, Math.max(0, ((playheadOffsetSec * 1000) / stats.totalMs) * 100)) + : null + + return ( +
+
+ {!embedded && Execution Trace} + {trace.trace_id} + {trace.trace_source && ( + + {trace.trace_source} + + )} + + {formatDuration(stats.totalMs, 'seconds')} total + + LLM {pct(stats.llmMs)}% + STT {pct(stats.sttMs)}% + TTS {pct(stats.ttsMs)}% +
+ + +
+
+ + {showTimelineProp && allSpans.length > 0 && ( +
+
+

Timeline

+ {playheadOffsetSec !== null && syncAudioToSelection && ( +

+ Playhead {formatRelativeOffset(playheadOffsetSec * 1000, 'long')} +

+ )} +
+
+ {allSpans + .filter((span) => span.span_id && (span.name === 'turn' || isElevenLabsTurnSpan(span))) + .map((span) => { + const startOffset = ((span.start_time || rootStartMs) - rootStartMs) / stats.totalMs + const width = Math.max((span.duration_ms || 0) / stats.totalMs, 0.008) + const isActive = span.span_id === selectedSpanId + return ( +
+
+ +0:00 + {formatRelativeOffset(stats.totalMs)} +
+
+ )} + +
+ + +
+ {!selected ? ( +
+ Select a span to inspect details. +
+ ) : ( + + )} +
+
+
+ ) +} diff --git a/frontend/src/components/observability/traceDisplay.ts b/frontend/src/components/observability/traceDisplay.ts new file mode 100644 index 00000000..18b5f5eb --- /dev/null +++ b/frontend/src/components/observability/traceDisplay.ts @@ -0,0 +1,495 @@ +import type { ObservabilityCallTrace, ObservabilityTraceSpan } from '../../types/api' + +export type SpanTreeNode = ObservabilityTraceSpan & { + children: SpanTreeNode[] +} + +const SPAN_LABELS: Record = { + conversation: 'Call', + turn: 'User turn', + stt: 'STT', + llm: 'LLM inference', + tts: 'TTS', + s2s: 'Speech-to-speech', + tool_call: 'Tool call', + endpointing: 'Endpointing', +} + +const ELEVENLABS_SPAN_LABELS: Record = { + 'elevenlabs.conversation': 'ElevenLabs conversation', + 'elevenlabs.recv.user_transcript': 'User transcript', + 'elevenlabs.recv.agent_response': 'Agent response', +} + +const HIGHLIGHT_ATTR_KEYS = [ + 'turn.number', + 'turn.duration_seconds', + 'turn.was_interrupted', + 'turn.user_transcript', + 'turn.agent_transcript', + 'conversation.id', + 'conversation.type', + 'stt.transcript', + 'transcript', + 'gen_ai.system', + 'gen_ai.request.model', + 'gen_ai.response.model', + 'llm.model', + 'tts.voice', + 'function.name', +] + +export function buildSpanTree(trace: ObservabilityCallTrace): SpanTreeNode[] { + const spans = trace.spans || [] + const byId = new Map() + const children = new Map() + + spans.forEach((span) => { + if (!span.span_id) return + byId.set(span.span_id, { ...span, children: [] }) + }) + + spans.forEach((span) => { + if (!span.span_id) return + const parent = span.parent_span_id || '__root__' + const list = children.get(parent) || [] + const node = byId.get(span.span_id) + if (node) list.push(node) + children.set(parent, list) + }) + + const roots = + children.get('__root__') || + spans + .filter((span) => span.span_id && (!span.parent_span_id || !byId.has(span.parent_span_id))) + .map((span) => byId.get(span.span_id!)!) + .filter(Boolean) + + const attachChildren = (node: SpanTreeNode) => { + if (!node.span_id) { + node.children = [] + return + } + const kids = children.get(node.span_id) || [] + node.children = [...kids].sort((a, b) => (a.start_time || 0) - (b.start_time || 0)) + node.children.forEach(attachChildren) + } + + const sortedRoots = [...roots].sort((a, b) => (a.start_time || 0) - (b.start_time || 0)) + sortedRoots.forEach(attachChildren) + return sortedRoots +} + +export function flattenSpanTree(nodes: SpanTreeNode[]): ObservabilityTraceSpan[] { + const out: ObservabilityTraceSpan[] = [] + const walk = (list: SpanTreeNode[]) => { + list.forEach((node) => { + out.push(node) + walk(node.children) + }) + } + walk(nodes) + return out +} + +export function getSpanDisplayName(span: ObservabilityTraceSpan): string { + const estimatedSuffix = span.attributes?.['metric.estimated'] === true ? ' (estimated)' : '' + if (isElevenLabsSpan(span)) { + if (span.name in ELEVENLABS_SPAN_LABELS) return `${ELEVENLABS_SPAN_LABELS[span.name]}${estimatedSuffix}` + if (span.name.startsWith('elevenlabs.tool.')) { + const toolName = span.name.replace('elevenlabs.tool.', '') + return `Tool call (${toolName})${estimatedSuffix}` + } + if (span.name.startsWith('elevenlabs.metric.')) { + const metric = span.name.replace('elevenlabs.metric.', '').toUpperCase() + return `${metric} metrics${estimatedSuffix}` + } + return `${span.name}${estimatedSuffix}` + } + const base = SPAN_LABELS[span.name] || span.name + if (span.name === 'turn') { + const role = String(span.attributes?.['turn.role'] || '').toLowerCase() + const rolePrefix = role === 'agent' ? 'Agent turn' : 'User turn' + const turnNumber = span.attributes?.['turn.number'] + if (turnNumber !== undefined && turnNumber !== null) { + return `${rolePrefix} ${turnNumber}` + } + return rolePrefix + } + return `${base}${estimatedSuffix}` +} + +export function formatDuration(ms: number | null | undefined, style: 'auto' | 'ms' | 'seconds' = 'auto'): string { + const value = ms ?? 0 + if (style === 'ms') return `${Math.round(value)} ms` + if (style === 'seconds' || value >= 1000) { + const seconds = value / 1000 + if (seconds >= 10) return `${seconds.toFixed(1)}s` + return `${seconds.toFixed(2)}s` + } + return `${Math.round(value)} ms` +} + +export function truncateSpanId(spanId: string | undefined | null, head = 6, tail = 4): string { + if (!spanId) return '—' + if (spanId.length <= head + tail + 1) return spanId + return `${spanId.slice(0, head)}…${spanId.slice(-tail)}` +} + +export function formatSpanTimestamp(startTimeMs: number | null | undefined): string { + if (!startTimeMs) return '—' + return new Date(startTimeMs).toLocaleString(undefined, { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + second: '2-digit', + }) +} + +export function getServiceName(span: ObservabilityTraceSpan): string { + const attrs = span.attributes || {} + const candidates = [ + attrs['service.name'], + attrs['service_name'], + attrs['gen_ai.system'], + ] + for (const value of candidates) { + if (typeof value === 'string' && value.trim()) return value + } + if (isElevenLabsSpan(span)) return 'elevenlabs' + return 'efficientai' +} + +export function getSpanKind(span: ObservabilityTraceSpan): string { + const kind = span.attributes?.['span.kind'] ?? span.attributes?.['span_kind'] + if (typeof kind === 'string' && kind.trim()) return kind + return 'Internal' +} + +export function getStatusLabel(status: string | null | undefined): string { + if (!status) return 'Unset' + const normalized = String(status).toLowerCase() + if (normalized.includes('ok') || normalized === '1') return 'OK' + if (normalized.includes('error') || normalized === '2') return 'Error' + return String(status) +} + +function collectDescendants(spanId: string, allSpans: ObservabilityTraceSpan[]): ObservabilityTraceSpan[] { + const children = allSpans.filter((s) => s.parent_span_id === spanId) + return children.flatMap((child) => { + if (!child.span_id) return [child] + return [child, ...collectDescendants(child.span_id, allSpans)] + }) +} + +export function enrichSpanAttributes( + span: ObservabilityTraceSpan, + allSpans: ObservabilityTraceSpan[], +): Record { + const attrs = { ...(span.attributes || {}) } + + if (span.name === 'turn' && span.span_id) { + const descendants = collectDescendants(span.span_id, allSpans) + if (!attrs['turn.user_transcript']) { + const stt = descendants.find((s) => s.name === 'stt') + const transcript = + stt?.attributes?.['stt.transcript'] ?? stt?.attributes?.['transcript'] + if (typeof transcript === 'string' && transcript.trim()) { + attrs['turn.user_transcript'] = transcript + } + } + if (!attrs['turn.agent_transcript']) { + const llm = descendants.find((s) => s.name === 'llm') + const response = + llm?.attributes?.['gen_ai.response.text'] ?? + llm?.attributes?.['llm.response'] ?? + llm?.attributes?.['output'] + if (typeof response === 'string' && response.trim()) { + attrs['turn.agent_transcript'] = response + } + } + } + + return attrs +} + +export function partitionAttributes(attrs: Record): { + highlighted: Array<[string, unknown]> + rest: Array<[string, unknown]> +} { + const entries = Object.entries(attrs) + const highlighted: Array<[string, unknown]> = [] + const rest: Array<[string, unknown]> = [] + const seen = new Set() + + for (const key of HIGHLIGHT_ATTR_KEYS) { + if (key in attrs) { + highlighted.push([key, attrs[key]]) + seen.add(key) + } + } + + entries + .filter(([key]) => !seen.has(key)) + .sort(([a], [b]) => a.localeCompare(b)) + .forEach(([key, value]) => rest.push([key, value])) + + return { highlighted, rest } +} + +export function computeTraceStats(spans: ObservabilityTraceSpan[]) { + const isEstimatedMetric = (span: ObservabilityTraceSpan) => + span.attributes?.['metric.estimated'] === true + const isTurnScopedMetric = (span: ObservabilityTraceSpan) => { + const scope = span.attributes?.['metric.scope'] + return typeof scope === 'string' && scope.startsWith('turn_') + } + + const sumByMatcher = (matcher: (span: ObservabilityTraceSpan) => boolean) => + spans + .filter((s) => !isEstimatedMetric(s) && !isTurnScopedMetric(s) && matcher(s)) + .reduce((sum, s) => sum + (s.duration_ms || 0), 0) + + const llmMs = sumByMatcher( + (s) => + s.name === 'llm' || + s.name === 'elevenlabs.metric.llm' || + s.name === 'retell-metric-llm' || + s.name === 'vapi-metric-llm' || + s.attributes?.['metric.layer'] === 'llm', + ) + const sttMs = sumByMatcher( + (s) => + s.name === 'stt' || + s.name === 'elevenlabs.metric.asr' || + s.name === 'retell-metric-stt' || + s.name === 'vapi-metric-stt' || + s.attributes?.['metric.layer'] === 'stt', + ) + const ttsMs = sumByMatcher( + (s) => + s.name === 'tts' || + s.name === 'elevenlabs.metric.tts' || + s.name === 'retell-metric-tts' || + s.name === 'vapi-metric-tts' || + s.attributes?.['metric.layer'] === 'tts', + ) + const totalMs = Math.max( + ...spans.map((s) => s.duration_ms || 0), + llmMs + sttMs + ttsMs, + 1, + ) + + const root = spans.find((s) => s.name === 'conversation') || spans[0] + const traceDurationMs = root?.duration_ms ?? totalMs + + return { llmMs, sttMs, ttsMs, totalMs: traceDurationMs || totalMs } +} + +export function formatAttributeValue(value: unknown): string { + if (value === null || value === undefined) return '—' + if (typeof value === 'string') return value + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + try { + return JSON.stringify(value) + } catch { + return String(value) + } +} + +/** Offset from trace/call start, e.g. "+0:12" or "+1:05.3" */ +export function formatRelativeOffset( + offsetMs: number | null | undefined, + style: 'short' | 'long' = 'short', +): string { + if (offsetMs === null || offsetMs === undefined || Number.isNaN(offsetMs)) return '—' + const totalSec = Math.max(0, offsetMs / 1000) + const mins = Math.floor(totalSec / 60) + const secs = totalSec % 60 + if (style === 'long') { + return mins > 0 ? `+${mins}:${secs.toFixed(1).padStart(4, '0')}` : `+${secs.toFixed(1)}s` + } + if (mins > 0) { + return `+${mins}:${Math.floor(secs).toString().padStart(2, '0')}` + } + return `+${secs.toFixed(secs >= 10 ? 0 : 1)}s` +} + +export function getTraceRootStartMs(spans: ObservabilityTraceSpan[]): number { + const values = spans.map((s) => s.start_time || 0).filter((v) => v > 0) + return values.length ? Math.min(...values) : 0 +} + +const PIPELINE_SPAN_NAMES = new Set(['stt', 'llm', 'tts', 's2s', 'tool_call', 'endpointing']) + +export function getDefaultCollapsedSpanIds(nodes: SpanTreeNode[]): Set { + const collapsed = new Set() + const walk = (list: SpanTreeNode[]) => { + list.forEach((node) => { + if ( + node.span_id && + (PIPELINE_SPAN_NAMES.has(node.name) || node.name.startsWith('elevenlabs.tool.')) + ) { + collapsed.add(node.span_id) + } + walk(node.children) + }) + } + walk(nodes) + return collapsed +} + +export function getAllCollapsibleSpanIds(nodes: SpanTreeNode[]): string[] { + const ids: string[] = [] + const walk = (list: SpanTreeNode[]) => { + list.forEach((node) => { + if (node.span_id && node.children.length > 0) ids.push(node.span_id) + walk(node.children) + }) + } + walk(nodes) + return ids +} + +export function getTurnPreview(span: ObservabilityTraceSpan, allSpans: ObservabilityTraceSpan[]): string | null { + if (isElevenLabsTurnSpan(span)) { + const text = + span.attributes?.['elevenlabs.user.text'] ?? + span.attributes?.['elevenlabs.agent.text'] ?? + span.attributes?.['text'] + if (typeof text === 'string' && text.trim()) { + const clipped = text.trim() + return clipped.length > 72 ? `${clipped.slice(0, 72)}…` : clipped + } + } + const enriched = enrichSpanAttributes(span, allSpans) + const user = enriched['turn.user_transcript'] + if (typeof user === 'string' && user.trim()) { + return user.trim().length > 72 ? `${user.trim().slice(0, 72)}…` : user.trim() + } + const agent = enriched['turn.agent_transcript'] + if (typeof agent === 'string' && agent.trim()) { + return agent.trim().length > 72 ? `${agent.trim().slice(0, 72)}…` : agent.trim() + } + return null +} + +export interface SpanSummaryLine { + label: string + value: string +} + +export function getSpanSummaryLines( + span: ObservabilityTraceSpan, + allSpans: ObservabilityTraceSpan[], +): SpanSummaryLine[] { + const attrs = enrichSpanAttributes(span, allSpans) + const lines: SpanSummaryLine[] = [] + + if (isElevenLabsTurnSpan(span)) { + const label = span.name.includes('user') ? 'User said' : 'Agent replied' + const text = + attrs['elevenlabs.user.text'] ?? + attrs['elevenlabs.agent.text'] ?? + attrs['text'] + if (typeof text === 'string' && text.trim()) lines.push({ label, value: text.trim() }) + } else if (span.name === 'turn') { + const user = attrs['turn.user_transcript'] + const agent = attrs['turn.agent_transcript'] + if (typeof user === 'string' && user.trim()) lines.push({ label: 'User said', value: user.trim() }) + if (typeof agent === 'string' && agent.trim()) lines.push({ label: 'Agent replied', value: agent.trim() }) + if (attrs['turn.was_interrupted']) lines.push({ label: 'Interrupted', value: 'Yes' }) + } else if (span.name === 'stt') { + const transcript = attrs['stt.transcript'] ?? attrs['transcript'] + if (typeof transcript === 'string' && transcript.trim()) { + lines.push({ label: 'Transcript', value: transcript.trim() }) + } + } else if (span.name === 'llm') { + const model = attrs['gen_ai.request.model'] ?? attrs['gen_ai.response.model'] ?? attrs['llm.model'] + if (typeof model === 'string') lines.push({ label: 'Model', value: model }) + const inputTokens = attrs['gen_ai.usage.input_tokens'] + const outputTokens = attrs['gen_ai.usage.output_tokens'] + if (inputTokens !== undefined || outputTokens !== undefined) { + lines.push({ + label: 'Tokens', + value: `${inputTokens ?? '?'} in / ${outputTokens ?? '?'} out`, + }) + } + const response = + attrs['gen_ai.response.text'] ?? attrs['llm.response'] ?? attrs['output'] + if (typeof response === 'string' && response.trim()) { + const preview = response.trim().length > 200 ? `${response.trim().slice(0, 200)}…` : response.trim() + lines.push({ label: 'Response', value: preview }) + } + } else if (span.name === 'tts') { + const voice = attrs['tts.voice'] + if (typeof voice === 'string') lines.push({ label: 'Voice', value: voice }) + } else if (span.name === 'tool_call') { + const fn = attrs['function.name'] + if (typeof fn === 'string') lines.push({ label: 'Function', value: fn }) + } + + return lines +} + +export function findTurnSpanForTranscriptIndex( + turnIndex: number, + allSpans: ObservabilityTraceSpan[], +): ObservabilityTraceSpan | null { + const turnSpans = allSpans + .filter((s) => (s.name === 'turn' || isElevenLabsTurnSpan(s)) && s.span_id) + .sort((a, b) => (a.start_time || 0) - (b.start_time || 0)) + if (turnSpans.length === 0) return null + const idx = Math.min(turnIndex, turnSpans.length - 1) + return turnSpans[idx] ?? null +} + +export function findNearestTurnSpan( + audioOffsetSec: number, + callStartMs: number | null, + rootStartMs: number, + allSpans: ObservabilityTraceSpan[], +): ObservabilityTraceSpan | null { + const turnSpans = allSpans + .filter((s) => (s.name === 'turn' || isElevenLabsTurnSpan(s)) && s.span_id && s.start_time) + .sort((a, b) => (a.start_time || 0) - (b.start_time || 0)) + if (turnSpans.length === 0) return null + + const anchorMs = callStartMs && callStartMs > 0 ? callStartMs : rootStartMs + const targetMs = anchorMs + audioOffsetSec * 1000 + + let nearest = turnSpans[0] + let nearestDist = Math.abs((nearest.start_time || 0) - targetMs) + for (const turn of turnSpans) { + const dist = Math.abs((turn.start_time || 0) - targetMs) + if (dist < nearestDist) { + nearest = turn + nearestDist = dist + } + } + return nearest +} + +export function isElevenLabsSpan(span: ObservabilityTraceSpan): boolean { + const provider = span.attributes?.['trace.provider'] + return provider === 'elevenlabs' || span.name.startsWith('elevenlabs.') +} + +export function isElevenLabsTurnSpan(span: ObservabilityTraceSpan): boolean { + return ( + span.name === 'elevenlabs.recv.user_transcript' || + span.name === 'elevenlabs.recv.agent_response' + ) +} + +export function spanOffsetSec( + span: ObservabilityTraceSpan, + callStartMs: number | null, + rootStartMs: number, +): number | null { + if (!span.start_time) return null + const anchorMs = callStartMs && callStartMs > 0 ? callStartMs : rootStartMs + return Math.max(0, (span.start_time - anchorMs) / 1000) +} diff --git a/frontend/src/components/observability/waveformSegments.ts b/frontend/src/components/observability/waveformSegments.ts new file mode 100644 index 00000000..97ac97fc --- /dev/null +++ b/frontend/src/components/observability/waveformSegments.ts @@ -0,0 +1,121 @@ +import type { ObservabilityCallData } from '../../types/api' + +export type WaveformSpeaker = 'agent' | 'user' + +export interface WaveformSegment { + speaker: WaveformSpeaker + startSec: number + endSec: number + text?: string +} + +interface RawSegment { + speaker?: string + role?: string + text?: string + content?: string + start?: number + end?: number + start_time?: number + end_time?: number +} + +function normalizeSpeaker(raw: string | undefined, role?: string): WaveformSpeaker { + const value = (raw || role || '').toLowerCase() + if ( + value === 'user' || + value === 'caller' || + value === 'customer' || + value === 'speaker 1' || + value === 'speaker_1' + ) { + return 'user' + } + return 'agent' +} + +function toSeconds(value: number | undefined): number | null { + if (value === undefined || value === null || Number.isNaN(value)) return null + if (value > 1e10) return value / 1000 + return value +} + +export function buildWaveformSegments( + callData: ObservabilityCallData | null | undefined, + transcriptTurns: Array<{ role: 'user' | 'agent'; content: string; start_time?: number }>, + _fallbackDurationSec?: number | null, +): WaveformSegment[] { + const raw = callData?.speaker_segments + if (Array.isArray(raw) && raw.length > 0) { + return (raw as RawSegment[]) + .map((seg, index) => { + const startSec = toSeconds(seg.start ?? seg.start_time) ?? index * 2 + const endSec = toSeconds(seg.end ?? seg.end_time) ?? startSec + 1.5 + return { + speaker: normalizeSpeaker(seg.speaker, seg.role), + startSec: Math.max(0, startSec), + endSec: Math.max(startSec + 0.2, endSec), + text: seg.text || seg.content, + } + }) + .sort((a, b) => a.startSec - b.startSec) + } + + let elapsed = 0 + return transcriptTurns.map((turn) => { + const startSec = + turn.start_time !== undefined ? (toSeconds(turn.start_time) ?? elapsed) : elapsed + const wordCount = turn.content.split(/\s+/).filter(Boolean).length + const duration = Math.max(1.2, wordCount * 0.35) + const endSec = startSec + duration + elapsed = endSec + return { + speaker: turn.role === 'user' ? 'user' : 'agent', + startSec, + endSec, + text: turn.content, + } + }) +} + +export function findSegmentIndexAtTime(segments: WaveformSegment[], timeSec: number): number | null { + const idx = segments.findIndex((seg) => timeSec >= seg.startSec && timeSec <= seg.endSec + 0.05) + if (idx >= 0) return idx + let nearest = 0 + let nearestDist = Infinity + segments.forEach((seg, i) => { + const dist = Math.min(Math.abs(timeSec - seg.startSec), Math.abs(timeSec - seg.endSec)) + if (dist < nearestDist) { + nearest = i + nearestDist = dist + } + }) + return segments.length > 0 ? nearest : null +} + +export function mapTranscriptIndexToSegmentIndex( + transcriptTurns: Array<{ role: 'user' | 'agent'; content: string; start_time?: number }>, + segments: WaveformSegment[], + transcriptIndex: number, +): number | null { + if (segments.length === 0 || transcriptIndex < 0) return null + const turn = transcriptTurns[transcriptIndex] + if (!turn) return null + const turnStart = + turn.start_time !== undefined ? (toSeconds(turn.start_time) ?? null) : null + if (turnStart !== null) { + const match = segments.findIndex( + (seg) => + seg.speaker === (turn.role === 'user' ? 'user' : 'agent') && + Math.abs(seg.startSec - turnStart) < 1.5, + ) + if (match >= 0) return match + } + const sameSpeakerSegments = segments + .map((seg, i) => ({ seg, i })) + .filter(({ seg }) => seg.speaker === (turn.role === 'user' ? 'user' : 'agent')) + const speakerTurnIndex = transcriptTurns + .slice(0, transcriptIndex + 1) + .filter((t) => t.role === turn.role).length - 1 + return sameSpeakerSegments[speakerTurnIndex]?.i ?? null +} diff --git a/frontend/src/hooks/useObservabilityCallAudioBlob.ts b/frontend/src/hooks/useObservabilityCallAudioBlob.ts new file mode 100644 index 00000000..513a3978 --- /dev/null +++ b/frontend/src/hooks/useObservabilityCallAudioBlob.ts @@ -0,0 +1,39 @@ +import { useEffect, useRef } from 'react' +import { useQuery } from '@tanstack/react-query' + +import { apiClient } from '../lib/api' + +/** Same-origin blob URL for waveform decode (avoids presigned S3 CORS on fetch). */ +export function useObservabilityCallAudioBlob(callShortId: string | undefined, enabled: boolean) { + const blobUrlRef = useRef(null) + + const query = useQuery({ + queryKey: ['observability-call-audio-blob', callShortId], + queryFn: () => apiClient.getObservabilityCallAudioUrl(callShortId!), + enabled: !!callShortId && enabled, + staleTime: 5 * 60 * 1000, + gcTime: 10 * 60 * 1000, + }) + + useEffect(() => { + const nextUrl = query.data + if (!nextUrl) return + const prevUrl = blobUrlRef.current + if (prevUrl && prevUrl !== nextUrl && prevUrl.startsWith('blob:')) { + URL.revokeObjectURL(prevUrl) + } + blobUrlRef.current = nextUrl + }, [query.data]) + + useEffect(() => { + return () => { + const url = blobUrlRef.current + if (url?.startsWith('blob:')) { + URL.revokeObjectURL(url) + } + blobUrlRef.current = null + } + }, [callShortId]) + + return query +} diff --git a/frontend/src/hooks/useObservabilityLiveAudio.ts b/frontend/src/hooks/useObservabilityLiveAudio.ts new file mode 100644 index 00000000..8392f063 --- /dev/null +++ b/frontend/src/hooks/useObservabilityLiveAudio.ts @@ -0,0 +1,53 @@ +import { useEffect, useRef } from 'react' +import { useQuery } from '@tanstack/react-query' + +import { apiClient } from '../lib/api' + +export interface LiveAudioSnapshot { + blobUrl: string + durationSec: number +} + +/** Poll partial live call audio (merged mono WAV) while the call is in progress. */ +export function useObservabilityLiveAudio(callShortId: string | undefined, enabled: boolean) { + const blobUrlRef = useRef(null) + + const query = useQuery({ + queryKey: ['observability-live-audio', callShortId], + queryFn: async (): Promise => { + const { blob, durationSec } = await apiClient.getObservabilityLiveAudioBlob(callShortId!) + const blobUrl = URL.createObjectURL(blob) + return { blobUrl, durationSec } + }, + enabled: !!callShortId && enabled, + refetchInterval: enabled ? 3000 : false, + staleTime: 0, + retry: (failureCount, error) => { + if (failureCount >= 2) return false + const status = (error as { response?: { status?: number } })?.response?.status + return status !== 404 + }, + }) + + useEffect(() => { + const nextUrl = query.data?.blobUrl + if (!nextUrl) return + const prevUrl = blobUrlRef.current + if (prevUrl && prevUrl !== nextUrl && prevUrl.startsWith('blob:')) { + URL.revokeObjectURL(prevUrl) + } + blobUrlRef.current = nextUrl + }, [query.data?.blobUrl]) + + useEffect(() => { + return () => { + const url = blobUrlRef.current + if (url?.startsWith('blob:')) { + URL.revokeObjectURL(url) + } + blobUrlRef.current = null + } + }, [callShortId]) + + return query +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index c432705b..95bff2be 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -31,6 +31,7 @@ import type { Role, Integration, IntegrationCreate, + ExternalProviderAgentListResponse, S3ConnectionTestResponse, S3ListFilesResponse, S3BrowseResponse, @@ -40,6 +41,9 @@ import type { CallImportListResponse, CallImportRow, ObservabilityCall, + ObservabilityCallTrace, + ObservabilityCallsSummary, + ObservabilityLiveLatencyResponse, CallImportSchema, CallImportSchemaCreate, CallImportSchemaListResponse, @@ -1624,6 +1628,17 @@ class ApiClient { return response.data } + async listIntegrationExternalAgents( + integrationId: string, + params?: { search?: string; cursor?: string; page_size?: number }, + ): Promise { + const response = await this.client.get( + `/api/v1/integrations/${integrationId}/external-agents`, + { params }, + ) + return response.data + } + async previewIntegrationAgentPrompt( integrationId: string, voiceAiAgentId: string, @@ -3981,6 +3996,28 @@ class ApiClient { return response.data } + async getObservabilityCallsSummary(): Promise { + const response = await this.client.get('/api/v1/observability/calls/summary') + return response.data + } + + async getObservabilityLiveLatencyMetrics(platform?: string): Promise { + const response = await this.client.get('/api/v1/observability/live/metrics/latency', { + params: platform ? { platform } : undefined, + }) + return response.data + } + + async getObservabilityAgentLiveLatencyMetrics( + agentId: string, + platform?: string, + ): Promise { + const response = await this.client.get(`/api/v1/observability/live/agents/${agentId}/latency`, { + params: platform ? { platform } : undefined, + }) + return response.data + } + private buildAuthenticatedApiUrl(path: string): string { const normalizedPath = path.startsWith('/') ? path : `/${path}` const configuredBase = (this.client.defaults.baseURL || '').replace(/\/$/, '') @@ -4007,6 +4044,16 @@ class ApiClient { return response.data } + async getObservabilityCallTrace(callShortId: string): Promise { + const response = await this.client.get(`/api/v1/observability/calls/${callShortId}/trace`) + return response.data + } + + async refreshObservabilityCall(callShortId: string): Promise { + const response = await this.client.post(`/api/v1/observability/calls/${callShortId}/refresh`) + return response.data + } + getObservabilityCallLiveEventsUrl(callShortId: string): string { return this.buildAuthenticatedApiUrl( `/api/v1/observability/calls/${callShortId}/live-events`, @@ -4021,6 +4068,18 @@ class ApiClient { return URL.createObjectURL(response.data) } + async getObservabilityLiveAudioBlob( + callShortId: string, + ): Promise<{ blob: Blob; durationSec: number }> { + const response = await this.client.get( + `/api/v1/observability/calls/${callShortId}/live-audio`, + { responseType: 'blob' }, + ) + const rawDuration = response.headers['x-audio-duration-sec'] + const durationSec = rawDuration ? parseFloat(String(rawDuration)) : 0 + return { blob: response.data, durationSec } + } + async deleteObservabilityCall(callShortId: string): Promise<{ message: string }> { const response = await this.client.delete(`/api/v1/observability/calls/${callShortId}`) return response.data diff --git a/frontend/src/lib/audioWaveform.ts b/frontend/src/lib/audioWaveform.ts new file mode 100644 index 00000000..9fa6b14b --- /dev/null +++ b/frontend/src/lib/audioWaveform.ts @@ -0,0 +1,99 @@ +/** Decode audio from URL and produce normalized peak buckets for waveform rendering. */ + +export interface AudioPeakData { + peaks: Float32Array + durationSec: number +} + +let sharedAudioContext: AudioContext | null = null + +function getAudioContext(): AudioContext { + if (!sharedAudioContext) { + sharedAudioContext = new AudioContext() + } + return sharedAudioContext +} + +export async function loadAudioPeaks(audioUrl: string, bucketCount = 1600): Promise { + const response = await fetch(audioUrl) + if (!response.ok) { + throw new Error(`Failed to load audio (${response.status})`) + } + const buffer = await response.arrayBuffer() + return decodePeaksFromArrayBuffer(buffer, bucketCount) +} + +export async function decodePeaksFromArrayBuffer( + buffer: ArrayBuffer, + bucketCount = 1600, +): Promise { + const ctx = getAudioContext() + const audioBuffer = await ctx.decodeAudioData(buffer.slice(0)) + + const channel = audioBuffer.numberOfChannels > 0 ? audioBuffer.getChannelData(0) : new Float32Array() + const durationSec = audioBuffer.duration + const samplesPerBucket = Math.max(1, Math.floor(channel.length / bucketCount)) + const peaks = new Float32Array(bucketCount) + + for (let i = 0; i < bucketCount; i += 1) { + const start = i * samplesPerBucket + const end = Math.min(channel.length, start + samplesPerBucket) + let max = 0 + for (let j = start; j < end; j += 1) { + const v = Math.abs(channel[j]) + if (v > max) max = v + } + peaks[i] = max + } + + return { peaks, durationSec } +} + +/** Fallback when decode fails — flat peaks so segment bars still render. */ +export function buildSyntheticPeaks(durationSec: number, bucketCount = 1600): AudioPeakData { + const peaks = new Float32Array(bucketCount) + for (let i = 0; i < bucketCount; i += 1) { + peaks[i] = 0.15 + Math.random() * 0.35 + } + return { peaks, durationSec: Math.max(durationSec, 1) } +} + +export function peaksForTimeRange( + peaks: Float32Array, + totalDurationSec: number, + startSec: number, + endSec: number, + targetBuckets = 48, +): Float32Array { + if (totalDurationSec <= 0 || peaks.length === 0) return new Float32Array(0) + const startBucket = Math.floor((startSec / totalDurationSec) * peaks.length) + const endBucket = Math.max( + startBucket + 1, + Math.ceil((endSec / totalDurationSec) * peaks.length), + ) + const slice = peaks.subarray( + Math.max(0, startBucket), + Math.min(peaks.length, endBucket), + ) + if (slice.length <= targetBuckets) return slice + + const out = new Float32Array(targetBuckets) + const ratio = slice.length / targetBuckets + for (let i = 0; i < targetBuckets; i += 1) { + const from = Math.floor(i * ratio) + const to = Math.min(slice.length, Math.floor((i + 1) * ratio) + 1) + let max = 0 + for (let j = from; j < to; j += 1) { + if (slice[j] > max) max = slice[j] + } + out[i] = max + } + return out +} + +export function formatPlaybackTime(seconds: number): string { + const safe = Math.max(0, seconds) + const mins = Math.floor(safe / 60) + const secs = Math.floor(safe % 60) + return `${mins}:${secs.toString().padStart(2, '0')}` +} diff --git a/frontend/src/pages/agents/components/AgentEditForm.tsx b/frontend/src/pages/agents/components/AgentEditForm.tsx index d734ff8b..65772e61 100644 --- a/frontend/src/pages/agents/components/AgentEditForm.tsx +++ b/frontend/src/pages/agents/components/AgentEditForm.tsx @@ -1,13 +1,20 @@ import { useState, useEffect } from 'react' import { useNavigate } from 'react-router-dom' import { useQuery, useMutation } from '@tanstack/react-query' +import type { AxiosError } from 'axios' import { Sparkles, Loader2, Bot, Eye, Code, Trash2, Save, PhoneOutgoing, PhoneIncoming } from 'lucide-react' import ParamSlider from './ParamSlider' import { OverviewSection, formatSilenceHangupLabel } from './AgentOverviewLayout' import ReactMarkdown from 'react-markdown' import Button from '../../../components/Button' import { apiClient } from '../../../lib/api' -import { VoiceBundle, Integration, AIProvider, IntegrationPlatform } from '../../../types/api' +import { + VoiceBundle, + Integration, + AIProvider, + IntegrationPlatform, + ExternalProviderAgent, +} from '../../../types/api' import { getIntegrationPlatformLabel, getIntegrationPlatformLogo, getTelephonyProviderLabel } from '../../../config/providers' import { useOrgTelephony } from '../../../hooks/useOrgTelephony' import { TelephonyProvider } from '../../../types/api' @@ -82,6 +89,7 @@ export default function AgentEditForm({ const [aiModel, setAiModel] = useState('') const [phoneNumberInputMode, setPhoneNumberInputMode] = useState<'provider' | 'custom'>('provider') const [testAgentSubTab, setTestAgentSubTab] = useState('prompt') + const [manualPlatformAgentId, setManualPlatformAgentId] = useState(false) const { data: aiProviders = [] } = useQuery({ queryKey: ['ai-providers'], @@ -207,6 +215,31 @@ export default function AgentEditForm({ const selectedVoiceIntegration = voiceAgentIntegrations.find( (integration) => integration.id === formData.voice_ai_integration_id, ) + const externalAgentPlatform = selectedVoiceIntegration?.platform as IntegrationPlatform | undefined + const externalAgentPlatformLabel = externalAgentPlatform + ? getIntegrationPlatformLabel(externalAgentPlatform) + : 'provider' + + const shouldLoadExternalAgents = + [IntegrationPlatform.ELEVENLABS, IntegrationPlatform.VAPI, IntegrationPlatform.RETELL].includes( + selectedVoiceIntegration?.platform as IntegrationPlatform, + ) && + !!selectedVoiceIntegration?.id + + const { + data: externalAgentPayload, + isLoading: externalAgentLoading, + isError: externalAgentError, + error: externalAgentQueryError, + } = useQuery({ + queryKey: ['integration-external-agents', selectedVoiceIntegration?.id], + queryFn: () => apiClient.listIntegrationExternalAgents(selectedVoiceIntegration!.id), + enabled: shouldLoadExternalAgents, + }) + const externalAgents: ExternalProviderAgent[] = externalAgentPayload?.agents || [] + const externalAgentErrorMessage = + (externalAgentQueryError as AxiosError<{ detail?: string }>)?.response?.data?.detail || + `Could not load ${externalAgentPlatformLabel} agents.` const linkedVoiceBundle = formData.voice_bundle_id ? voiceBundles.find((vb) => vb.id === formData.voice_bundle_id) @@ -830,7 +863,14 @@ export default function AgentEditForm({
onChange({ ...formData, voice_ai_agent_id: e.target.value })} - className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500 focus:border-transparent bg-white" - placeholder="Enter agent ID from Retell/Vapi/ElevenLabs/Smallest" - /> + {[IntegrationPlatform.ELEVENLABS, IntegrationPlatform.VAPI, IntegrationPlatform.RETELL].includes( + selectedVoiceIntegration?.platform as IntegrationPlatform, + ) ? ( +
+ {!manualPlatformAgentId && ( + + )} + {externalAgentError && ( +

+ {externalAgentErrorMessage} You can enter the agent ID manually. +

+ )} + {!externalAgentLoading && !externalAgentError && externalAgents.length === 0 && ( +

+ No agents were returned by {externalAgentPlatformLabel} for this API key. Verify the key and + that agents exist in your {externalAgentPlatformLabel} workspace. +

+ )} + {(manualPlatformAgentId || externalAgentError || externalAgents.length === 0) && ( + onChange({ ...formData, voice_ai_agent_id: e.target.value })} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500 focus:border-transparent bg-white font-mono" + placeholder={`Enter ${externalAgentPlatformLabel} agent ID`} + /> + )} + {!manualPlatformAgentId && externalAgents.length > 0 && ( + + )} +
+ ) : ( + onChange({ ...formData, voice_ai_agent_id: e.target.value })} + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500 focus:border-transparent bg-white" + placeholder="Enter agent ID from Retell/Vapi/ElevenLabs/Smallest" + /> + )}
)} diff --git a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx index 1afd845d..7210c245 100644 --- a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx +++ b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx @@ -56,10 +56,11 @@ export default function AgentTalkSidebar({ const vapiClientRef = useRef(null) const elevenLabsConversationRef = useRef(null) const smallestClientRef = useRef(null) + const currentCallShortIdRef = useRef(null) + const elevenLabsConversationIdStoredRef = useRef(false) const userInitiatedDisconnectRef = useRef(false) const wasOpenRef = useRef(false) const userSpeakingTimeoutRef = useRef | null>(null) - const callShortIdRef = useRef(null) const pulseUserSpeaking = (durationMs = 1200) => { setActiveSpeaker('user') @@ -143,6 +144,33 @@ export default function AgentTalkSidebar({ userInitiatedDisconnectRef.current = false } + const persistProviderCallId = async (providerCallId?: string | null) => { + const callShortId = currentCallShortIdRef.current + if (!callShortId || !providerCallId) return false + try { + await apiClient.updateCallRecording(callShortId, providerCallId) + return true + } catch (err) { + console.error('Failed to update call recording provider ID', err) + return false + } + } + + const refreshCurrentCallRecording = (delayMs = 0) => { + const callShortId = currentCallShortIdRef.current + if (!callShortId) return + const run = () => { + apiClient.refreshCallRecording(callShortId).catch((err) => { + console.error('Failed to refresh call recording', err) + }) + } + if (delayMs > 0) { + window.setTimeout(run, delayMs) + return + } + run() + } + const handleConnectVoiceAI = async () => { if (!canTalkVoiceAI) return setIsConnecting(true) @@ -199,6 +227,9 @@ export default function AgentTalkSidebar({ setActiveSpeaker(null) }) const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + if (webCall.call_short_id) { + currentCallShortIdRef.current = webCall.call_short_id + } await client.startCall({ accessToken: webCall.access_token!, callId: webCall.call_id, @@ -209,13 +240,7 @@ export default function AgentTalkSidebar({ client.on('call-start', async (call: any) => { setIsConnected(true) setIsConnecting(false) - if (callShortIdRef.current && call?.id) { - try { - await apiClient.updateCallRecording(callShortIdRef.current, call.id) - } catch (err) { - console.error('Failed to update Vapi call recording', err) - } - } + await persistProviderCallId(call?.id) }) client.on('speech-start', () => setActiveSpeaker('agent')) client.on('speech-end', () => setActiveSpeaker((prev) => (prev === 'agent' ? null : prev))) @@ -232,36 +257,66 @@ export default function AgentTalkSidebar({ } } }) - client.on('call-end', async () => { + client.on('call-end', async (call: any) => { + await persistProviderCallId(call?.id) + refreshCurrentCallRecording() setIsConnected(false) setIsConnecting(false) setActiveSpeaker(null) - if (callShortIdRef.current) { - apiClient.refreshCallRecording(callShortIdRef.current).catch((err) => { - console.error('Failed to refresh Vapi call recording', err) - }) - } }) const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) - callShortIdRef.current = webCall.call_short_id ?? null - const vapiCall = await client.start(agent.voice_ai_agent_id!) - if (callShortIdRef.current && vapiCall?.id) { - try { - await apiClient.updateCallRecording(callShortIdRef.current, vapiCall.id) - } catch (err) { - console.error('Failed to update Vapi call recording from start()', err) - } + if (webCall.call_short_id) { + currentCallShortIdRef.current = webCall.call_short_id } + const vapiCall = await client.start(agent.voice_ai_agent_id!) + await persistProviderCallId(vapiCall?.id) } else if (isElevenLabs) { const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + if (webCall.call_short_id) { + currentCallShortIdRef.current = webCall.call_short_id + } + elevenLabsConversationIdStoredRef.current = false if (!webCall.signed_url) throw new Error('No signed URL') + const tryPersistElevenLabsConversationId = async ( + conversation: { getId?: () => string | undefined } | null | undefined, + maxAttempts = 10, + intervalMs = 1000, + ) => { + if (!conversation?.getId) return false + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + const conversationId = conversation.getId() + if (conversationId) { + const persisted = await persistProviderCallId(conversationId) + if (persisted) { + elevenLabsConversationIdStoredRef.current = true + return true + } + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + return false + } const conversation = await Conversation.startSession({ signedUrl: webCall.signed_url, onConnect: () => { setIsConnected(true) setIsConnecting(false) + if (!elevenLabsConversationIdStoredRef.current) { + void tryPersistElevenLabsConversationId( + elevenLabsConversationRef.current, + 20, + 500, + ) + } }, - onDisconnect: () => { + onDisconnect: async () => { + if (!elevenLabsConversationIdStoredRef.current) { + await tryPersistElevenLabsConversationId(conversation, 3, 500) + } + if (elevenLabsConversationIdStoredRef.current) { + // ElevenLabs may still be finalizing when disconnect fires. + refreshCurrentCallRecording(5000) + } setIsConnected(false) setIsConnecting(false) setActiveSpeaker(null) @@ -295,9 +350,13 @@ export default function AgentTalkSidebar({ }, }) elevenLabsConversationRef.current = conversation + void tryPersistElevenLabsConversationId(conversation, 20, 500) } else if (isSmallest) { const { AtomsClient } = await import('atoms-client-sdk') const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + if (webCall.call_short_id) { + currentCallShortIdRef.current = webCall.call_short_id + } if (!webCall.access_token || !webCall.host) throw new Error('Missing Smallest credentials') const client = new AtomsClient() smallestClientRef.current = client @@ -306,6 +365,7 @@ export default function AgentTalkSidebar({ setIsConnecting(false) }) client.on('session_ended', () => { + refreshCurrentCallRecording(3000) setIsConnected(false) setIsConnecting(false) setActiveSpeaker(null) diff --git a/frontend/src/pages/agents/components/create/PlatformConnectStep.tsx b/frontend/src/pages/agents/components/create/PlatformConnectStep.tsx index 36a0af82..2ee53e9f 100644 --- a/frontend/src/pages/agents/components/create/PlatformConnectStep.tsx +++ b/frontend/src/pages/agents/components/create/PlatformConnectStep.tsx @@ -1,5 +1,9 @@ import { Link } from 'react-router-dom' -import { Integration, IntegrationPlatform } from '../../../../types/api' +import { useMemo, useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import type { AxiosError } from 'axios' +import { apiClient } from '../../../../lib/api' +import { ExternalProviderAgent, Integration, IntegrationPlatform } from '../../../../types/api' import { getIntegrationPlatformLabel, getIntegrationPlatformLogo } from '../../../../config/providers' const PLATFORM_OPTIONS: IntegrationPlatform[] = [ @@ -32,11 +36,47 @@ export default function PlatformConnectStep({ onIntegrationChange, onAgentIdChange, }: PlatformConnectStepProps) { + const [manualAgentIdEnabled, setManualAgentIdEnabled] = useState(false) const activeIntegrations = integrations.filter((integration) => integration.is_active) const integrationsForPlatform = (platform: IntegrationPlatform) => activeIntegrations.filter((integration) => integration.platform === platform) + const selectedIntegration = useMemo( + () => activeIntegrations.find((integration) => integration.id === voiceAiIntegrationId), + [activeIntegrations, voiceAiIntegrationId], + ) + + const externalAgentPlatformLabel = + selectedPlatform && [IntegrationPlatform.ELEVENLABS, IntegrationPlatform.VAPI, IntegrationPlatform.RETELL].includes(selectedPlatform) + ? getIntegrationPlatformLabel(selectedPlatform) + : 'provider' + const shouldLoadExternalAgents = + [IntegrationPlatform.ELEVENLABS, IntegrationPlatform.VAPI, IntegrationPlatform.RETELL].includes(selectedPlatform as IntegrationPlatform) && + !!selectedIntegration?.id + + const { + data: externalAgentPayload, + isLoading: externalAgentLoading, + isError: externalAgentError, + error: externalAgentQueryError, + } = useQuery({ + queryKey: ['integration-external-agents', selectedIntegration?.id], + queryFn: () => apiClient.listIntegrationExternalAgents(selectedIntegration!.id), + enabled: shouldLoadExternalAgents, + }) + + const externalAgents: ExternalProviderAgent[] = externalAgentPayload?.agents || [] + const externalAgentErrorMessage = + (externalAgentQueryError as AxiosError<{ detail?: string }>)?.response?.data?.detail || + `Could not load ${externalAgentPlatformLabel} agents.` + + const onIntegrationSelect = (integrationId: string) => { + onIntegrationChange(integrationId) + onAgentIdChange('') + setManualAgentIdEnabled(false) + } + return (
@@ -104,7 +144,7 @@ export default function PlatformConnectStep({ onAgentIdChange(e.target.value)} - placeholder={`Enter ${label} agent ID`} - className="w-full px-3 py-2 text-sm border border-gray-300 rounded-lg bg-white focus:ring-2 focus:ring-primary-500 font-mono" - /> + {[IntegrationPlatform.ELEVENLABS, IntegrationPlatform.VAPI, IntegrationPlatform.RETELL].includes(selectedPlatform as IntegrationPlatform) ? ( +
+ {!manualAgentIdEnabled && ( + + )} + {externalAgentError && ( +

+ {externalAgentErrorMessage} You can enter the agent ID manually. +

+ )} + {!externalAgentLoading && !externalAgentError && externalAgents.length === 0 && ( +

+ No agents were returned by {externalAgentPlatformLabel} for this API key. Verify + the key and that agents exist in your {externalAgentPlatformLabel} workspace. +

+ )} + {(manualAgentIdEnabled || externalAgentError || externalAgents.length === 0) && ( + onAgentIdChange(e.target.value)} + placeholder={`Enter ${externalAgentPlatformLabel} agent ID`} + className="w-full px-3 py-2 text-sm border border-gray-300 rounded-lg bg-white focus:ring-2 focus:ring-primary-500 font-mono" + /> + )} + {!manualAgentIdEnabled && externalAgents.length > 0 && ( + + )} +
+ ) : ( + onAgentIdChange(e.target.value)} + placeholder={`Enter ${label} agent ID`} + className="w-full px-3 py-2 text-sm border border-gray-300 rounded-lg bg-white focus:ring-2 focus:ring-primary-500 font-mono" + /> + )}
)} diff --git a/frontend/src/pages/observability/ObservabilityCallDetail.tsx b/frontend/src/pages/observability/ObservabilityCallDetail.tsx index 9b1e7e00..f7b98afe 100644 --- a/frontend/src/pages/observability/ObservabilityCallDetail.tsx +++ b/frontend/src/pages/observability/ObservabilityCallDetail.tsx @@ -1,24 +1,110 @@ import { useNavigate, useParams } from 'react-router-dom' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { useEffect, useState } from 'react' +import { useEffect, useMemo, useState } from 'react' import { ArrowLeft, Phone, Clock, PhoneIncoming, PhoneOutgoing, - MessageSquare, Trash2, Download, Tag, - Loader, XCircle, Sparkles, X, + MessageSquare, Trash2, Tag, + Loader, XCircle, Sparkles, X, Download, RotateCw, } from 'lucide-react' import { motion, AnimatePresence } from 'framer-motion' +import { + BarChart, + Bar, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + ResponsiveContainer, + PieChart, + Pie, + Cell, +} from 'recharts' import Button from '../../components/Button' import ConfirmModal from '../../components/ConfirmModal' import { apiClient } from '../../lib/api' -import RetellCallDetails from '../../components/call-recordings/RetellCallDetails' -import VapiCallDetails from '../../components/call-recordings/VapiCallDetails' -import VobizCallDetails from '../../components/call-recordings/VobizCallDetails' +import TraceRecordingPanel from '../../components/observability/TraceRecordingPanel' +import { buildWaveformSegments } from '../../components/observability/waveformSegments' +import { + buildSpanTree, + findTurnSpanForTranscriptIndex, + flattenSpanTree, +} from '../../components/observability/traceDisplay' import { getIntegrationPlatformLabel, getIntegrationPlatformLogo } from '../../config/providers' -import { IntegrationPlatform, ObservabilityCall } from '../../types/api' +import { IntegrationPlatform, ObservabilityCall, ObservabilityCallTrace } from '../../types/api' +import { useObservabilityCallAudioBlob } from '../../hooks/useObservabilityCallAudioBlob' import { useRecordingPresignedUrl } from '../../hooks/useRecordingPresignedUrl' import { CallAgentLink } from './CallAgentLink' +function getTraceFetchErrorDisplay(error: unknown): { + title: string + body: string + hint?: string +} { + const axiosError = error as { response?: { status?: number; data?: { detail?: string } }; message?: string } + const status = axiosError.response?.status + const detail = + typeof axiosError.response?.data?.detail === 'string' + ? axiosError.response.data.detail + : axiosError.message + + if (status === 404) { + const isSyntheticMissing = + typeof detail === 'string' && + (detail.includes('No trace linked') || + detail.toLowerCase().includes('no provider trace') || + detail.toLowerCase().includes('no synthetic')) + return { + title: isSyntheticMissing ? 'Provider trace not ready' : 'Trace not found in store', + body: + detail || + (isSyntheticMissing + ? 'Full Retell/Vapi metrics may not have been pulled yet. Click Refresh on the call to fetch provider report data and synthetic trace.' + : 'This call has a trace ID, but Tempo has no spans for it. The trace may have expired, been purged, or never exported.'), + hint: isSyntheticMissing + ? 'Ensure the call is linked to an agent with a Retell integration, then use Refresh.' + : 'Run a new voice-bundle test call while Tempo is running and tracing is enabled. Tempo retains traces for 24h by default (see observability/tempo/tempo.yml).', + } + } + + if (status === 502) { + const isConnectionError = + typeof detail === 'string' && + (detail.includes('Could not reach') || detail.toLowerCase().includes('connect')) + return { + title: isConnectionError ? 'Trace store unreachable' : 'Trace backend error', + body: detail || 'The trace query API returned an upstream error.', + hint: isConnectionError + ? 'Restart the API after changing config.yml (query_backend / tempo_query_url). For local dev: docker compose -f docker-compose.yml -f docker-compose.observability.yml up -d tempo' + : 'Ensure Tempo is reachable at tempo_query_url in config.yml. Tempo running in Docker does not guarantee spans were exported to it.', + } + } + + return { + title: 'Could not load trace', + body: detail || 'An unexpected error occurred while fetching spans.', + hint: + 'For local dev: docker compose -f docker-compose.yml -f docker-compose.observability.yml up -d tempo', + } +} + +type DetailTab = 'overview' | 'transcript' | 'trace' + +function looksLikeRetellCallData(callData: Record | undefined): boolean { + if (!callData || typeof callData !== 'object') return false + return Boolean( + callData.latency || + callData.call_analysis || + callData.call_cost || + (callData.call_id && Array.isArray(callData.transcript_object)), + ) +} + +function looksLikeVapiCallData(callData: Record | undefined): boolean { + if (!callData || typeof callData !== 'object') return false + return Boolean(callData.assistantId || callData.assistant_id || callData.artifact || callData.endedReason) +} + export default function ObservabilityCallDetail() { const navigate = useNavigate() const { callShortId } = useParams<{ callShortId: string }>() @@ -26,7 +112,12 @@ export default function ObservabilityCallDetail() { const [showDelete, setShowDelete] = useState(false) const [showEvalModal, setShowEvalModal] = useState(false) const [selectedEvaluator, setSelectedEvaluator] = useState('') - const [liveTranscript, setLiveTranscript] = useState>([]) + const [liveTranscript, setLiveTranscript] = useState< + Array<{ role: string; content: string; timestamp?: string; start_time?: number }> + >([]) + const [activeTab, setActiveTab] = useState('overview') + const [selectedTraceSpanId, setSelectedTraceSpanId] = useState(null) + const [highlightedTranscriptIndex, setHighlightedTranscriptIndex] = useState(null) const liveEvents = new Set([ 'outbound_initiated', @@ -52,6 +143,72 @@ export default function ObservabilityCallDetail() { }, }) + const linkedTraceId = callRecording?.trace_id || callRecording?.call_data?.trace_id || null + const resolvedProviderPlatform = useMemo(() => { + const stored = (callRecording?.provider_platform || '').toLowerCase() + if (stored && stored !== 'external') return stored + const callData = callRecording?.call_data as Record | undefined + if (looksLikeRetellCallData(callData)) return 'retell' + if (looksLikeVapiCallData(callData)) return 'vapi' + return stored + }, [callRecording?.provider_platform, callRecording?.call_data]) + const hasElevenLabsProviderTraceCandidate = + resolvedProviderPlatform === 'elevenlabs' && + !!callRecording?.provider_call_id + const liveTranscriptCount = Array.isArray(callRecording?.call_data?.live_transcript) + ? callRecording.call_data.live_transcript.length + : 0 + const isLiveIngestPlatform = ['pipecat', 'livekit', 'external'].includes(resolvedProviderPlatform) + const hasSyntheticProviderTraceCandidate = + ((resolvedProviderPlatform === 'vapi' || resolvedProviderPlatform === 'retell') && + !!callRecording?.provider_call_id) || + (isLiveIngestPlatform && + (liveTranscriptCount > 0 || !!linkedTraceId || !!callRecording?.call_data?.provider_trace)) + + const { + data: callTrace, + isLoading: traceLoading, + isError: traceError, + error: traceFetchError, + refetch: refetchTrace, + } = useQuery({ + queryKey: [ + 'observability-call-trace', + callShortId, + linkedTraceId, + resolvedProviderPlatform, + callRecording?.updated_at, + ], + queryFn: () => apiClient.getObservabilityCallTrace(callShortId!), + enabled: + !!callShortId && + (!!linkedTraceId || hasElevenLabsProviderTraceCandidate || hasSyntheticProviderTraceCandidate), + retry: false, + refetchInterval: () => { + const call = queryClient.getQueryData(['observability-call', callShortId]) + if (!call) return false + const live = call.is_live || liveEvents.has((call.call_event || '').toLowerCase()) + return live ? 3000 : false + }, + }) + + useEffect(() => { + if (!callShortId || !callRecording) return + if (!linkedTraceId && !hasSyntheticProviderTraceCandidate && !hasElevenLabsProviderTraceCandidate) { + return + } + void refetchTrace() + }, [ + callShortId, + callRecording?.updated_at, + callRecording?.provider_platform, + resolvedProviderPlatform, + linkedTraceId, + hasSyntheticProviderTraceCandidate, + hasElevenLabsProviderTraceCandidate, + refetchTrace, + ]) + useEffect(() => { const existing = callRecording?.call_data?.live_transcript if (!Array.isArray(existing) || existing.length === 0) return @@ -92,8 +249,34 @@ export default function ObservabilityCallDetail() { }, [callShortId, callRecording?.call_event, callRecording?.is_live]) const storageKey = callRecording?.call_data?.recording_s3_key ?? undefined - const providerRecordingUrl = callRecording?.call_data?.recording_url ?? null + const providerRecordingUrl = useMemo(() => { + const callData = callRecording?.call_data + if (!callData || typeof callData !== 'object') return null + const artifact = (callData as any).artifact + const candidate = [ + (callData as any).recording_url, + (callData as any).recording_multi_channel_url, + (callData as any).recordingUrl, + (callData as any).stereoRecordingUrl, + (callData as any).monoRecordingUrl, + artifact?.recordingUrl, + artifact?.stereoRecordingUrl, + artifact?.monoRecordingUrl, + artifact?.recording?.url, + artifact?.recording?.stereoUrl, + artifact?.recording?.monoUrl, + ].find((value) => typeof value === 'string' && value.trim().length > 0) + return typeof candidate === 'string' ? candidate : null + }, [callRecording?.call_data]) const hasStorageRecording = !!storageKey + const hasRecordingCandidate = hasStorageRecording || !!providerRecordingUrl + const isLiveCallActive = + !!callRecording?.is_live || liveEvents.has((callRecording?.call_event || '').toLowerCase()) + + const { + data: archivedAudioBlobUrl, + isLoading: archivedAudioLoading, + } = useObservabilityCallAudioBlob(callShortId, !isLiveCallActive && hasRecordingCandidate) const { data: presignedRecording, @@ -124,6 +307,27 @@ export default function ObservabilityCallDetail() { }, }) + const refreshMutation = useMutation({ + mutationFn: () => apiClient.refreshObservabilityCall(callShortId!), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['observability-call', callShortId] }) + queryClient.invalidateQueries({ queryKey: ['observability-call-trace', callShortId] }) + }, + }) + + const callDataEarly = callRecording?.call_data + const callStartMs = useMemo(() => { + const started = callDataEarly?.startedAt || callDataEarly?.started_at + if (!started) return null + const ms = new Date(started).getTime() + return Number.isNaN(ms) ? null : ms + }, [callDataEarly?.startedAt, callDataEarly?.started_at]) + + const traceSpans = useMemo(() => { + if (!callTrace?.spans?.length) return [] + return flattenSpanTree(buildSpanTree(callTrace)) + }, [callTrace]) + if (isLoading) { return (
@@ -156,42 +360,267 @@ export default function ObservabilityCallDetail() { } const callData = callRecording.call_data - const liveTranscriptEntries: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> = - Array.isArray(callData?.live_transcript) ? callData.live_transcript : [] + const providerCallData = callData as any + type TranscriptTurn = { role: 'user' | 'agent'; content: string; start_time?: number } + const startedAtValue = callData?.startedAt || callData?.started_at + const callStartMsFromData = + startedAtValue && !Number.isNaN(new Date(startedAtValue).getTime()) + ? new Date(startedAtValue).getTime() + : null + const liveTranscriptEntries: Array<{ + role: string + content: string + timestamp?: string + start_time?: number + }> = Array.isArray(callData?.live_transcript) ? callData.live_transcript : [] + const extractTranscriptText = (value: unknown): string => { + if (typeof value === 'string') return value.trim() + if (typeof value === 'number' || typeof value === 'boolean') return String(value) + if (Array.isArray(value)) { + return value + .map((entry) => extractTranscriptText(entry)) + .filter(Boolean) + .join(' ') + .trim() + } + if (value && typeof value === 'object') { + const record = value as Record + const candidates = [record.text, record.content, record.message, record.transcript, record.value] + for (const candidate of candidates) { + const parsed = extractTranscriptText(candidate) + if (parsed) return parsed + } + } + return '' + } const messagesFromLive = liveTranscriptEntries - .filter((entry) => entry?.content) .map((entry) => ({ role: entry.role === 'user' ? 'user' : 'assistant', - content: entry.content, - start_time: entry.start_time - ?? (entry.timestamp ? new Date(entry.timestamp).getTime() : undefined), + content: extractTranscriptText(entry.content), + start_time: + typeof entry.start_time === 'number' + ? entry.start_time + : entry.timestamp + ? new Date(entry.timestamp).getTime() + : undefined, })) - const messages: any[] | undefined = Array.isArray(callData?.messages) && callData.messages.length > 0 + .filter((entry) => entry.content.length > 0) + const normalizedCallDataMessages = Array.isArray(callData?.messages) ? callData.messages + .map((entry: any) => ({ + role: String(entry?.role || '').toLowerCase(), + content: extractTranscriptText(entry?.content ?? entry?.message ?? entry), + start_time: (() => { + const startCandidate = entry?.start_time ?? entry?.timestamp ?? entry?.time ?? entry?.secondsFromStart + if (typeof startCandidate !== 'number') return undefined + if (startCandidate > 1e10) return startCandidate + if (startCandidate > 1e6 && callStartMsFromData != null) return startCandidate + if (startCandidate < 1e6 && callStartMsFromData != null) return callStartMsFromData + startCandidate * 1000 + return startCandidate * 1000 + })(), + })) + .filter((entry: any) => entry.content.length > 0 && entry.role !== 'system') + : [] + const messages: any[] | undefined = normalizedCallDataMessages.length > 0 + ? normalizedCallDataMessages : messagesFromLive.length > 0 ? messagesFromLive : undefined - const playbackUrl = presignedRecording?.url || providerRecordingUrl - const audioLoading = hasStorageRecording && presignedLoading && !playbackUrl + + const providerMessagesTurns = Array.isArray(providerCallData?.messages) + ? providerCallData.messages + .filter((entry: any) => { + if (!entry || typeof entry !== 'object') return false + const role = String(entry.role || '').toLowerCase() + if (role === 'system') return false + const content = extractTranscriptText(entry.content ?? entry.message ?? entry) + return content.length > 0 + }) + .map((entry: any) => { + const roleRaw = String(entry.role || '').toLowerCase() + const role = + roleRaw === 'user' || roleRaw === 'caller' || roleRaw === 'customer' + ? 'user' + : 'assistant' + const startCandidate = entry.start_time ?? entry.timestamp ?? entry.time + const startMs = + typeof startCandidate === 'number' + ? (startCandidate > 1e10 ? startCandidate : startCandidate * 1000) + : undefined + return { + role, + content: extractTranscriptText(entry.content ?? entry.message ?? entry), + start_time: startMs, + } + }) + : [] + + const artifactMessagesTurns = + Array.isArray(providerCallData?.artifact?.messages) + ? providerCallData.artifact.messages + .filter((entry: any) => { + if (!entry || typeof entry !== 'object') return false + const role = String(entry.role || '').toLowerCase() + if (role === 'system') return false + const content = extractTranscriptText(entry.content ?? entry.message ?? entry) + return content.length > 0 + }) + .map((entry: any) => { + const roleRaw = String(entry.role || '').toLowerCase() + const role = + roleRaw === 'user' || roleRaw === 'caller' || roleRaw === 'customer' + ? 'user' + : 'assistant' + const startCandidate = entry.start_time ?? entry.timestamp ?? entry.time + const startMs = + typeof startCandidate === 'number' + ? (startCandidate > 1e10 ? startCandidate : startCandidate * 1000) + : undefined + return { + role, + content: extractTranscriptText(entry.content ?? entry.message ?? entry), + start_time: startMs, + } + }) + : [] + + const transcriptObjectTurns = Array.isArray(providerCallData?.transcript_object) + ? providerCallData.transcript_object + .filter((entry: any) => entry?.content || entry?.text) + .map((entry: any) => { + const roleRaw = String(entry.role || entry.speaker || '').toLowerCase() + const role = roleRaw === 'user' ? 'user' : 'assistant' + const startSecs = typeof entry.start === 'number' ? entry.start : null + return { + role, + content: String(entry.content ?? entry.text ?? ''), + start_time: + startSecs !== null && callStartMsFromData !== null + ? callStartMsFromData + startSecs * 1000 + : undefined, + } + }) + : [] + + const rawTranscriptTurns = Array.isArray(providerCallData?.raw_data?.transcript) + ? providerCallData.raw_data.transcript + .filter((entry: any) => entry?.message) + .map((entry: any) => { + const role = String(entry.role || '').toLowerCase() === 'user' ? 'user' : 'assistant' + const startSecs = typeof entry.time_in_call_secs === 'number' ? entry.time_in_call_secs : null + return { + role, + content: String(entry.message), + start_time: + startSecs !== null && callStartMsFromData !== null + ? callStartMsFromData + startSecs * 1000 + : undefined, + } + }) + : [] + + const transcriptTextTurns = + typeof providerCallData?.transcript === 'string' && providerCallData.transcript.trim() + ? providerCallData.transcript + .split('\n') + .map((line: string) => line.trim()) + .filter(Boolean) + .map((line: string) => { + const lower = line.toLowerCase() + const isUser = lower.startsWith('user:') + const isAgent = lower.startsWith('agent:') + if (isUser || isAgent) { + return { + role: isUser ? 'user' : 'assistant', + content: line.split(':').slice(1).join(':').trim(), + } + } + return { role: 'assistant', content: line } + }) + : [] + + const artifactTranscriptTurns = + typeof providerCallData?.artifact?.transcript === 'string' && providerCallData.artifact.transcript.trim() + ? providerCallData.artifact.transcript + .split('\n') + .map((line: string) => line.trim()) + .filter(Boolean) + .map((line: string) => { + const lower = line.toLowerCase() + const isUser = lower.startsWith('user:') + const isAgent = lower.startsWith('agent:') + if (isUser || isAgent) { + return { + role: isUser ? 'user' : 'assistant', + content: line.split(':').slice(1).join(':').trim(), + } + } + return { role: 'assistant', content: line } + }) + : [] + const playbackUrl = archivedAudioBlobUrl || presignedRecording?.url || providerRecordingUrl + const audioLoading = + (hasStorageRecording && presignedLoading && !playbackUrl) || + (hasRecordingCandidate && archivedAudioLoading && !playbackUrl) const isLiveCall = callRecording.is_live || liveEvents.has((callRecording.call_event || '').toLowerCase()) - const toTranscriptTurn = (entry: { role: string; content: string; start_time?: number }) => ({ - role: entry.role === 'user' ? 'user' as const : 'agent' as const, - content: entry.content, - start_time: entry.start_time, - }) + const toTranscriptTurn = ( + entry: { role?: string; content?: unknown; start_time?: number } | null | undefined, + ): TranscriptTurn | null => { + if (!entry || typeof entry !== 'object') return null + const content = + typeof entry.content === 'string' + ? entry.content.trim() + : entry.content == null + ? '' + : String(entry.content).trim() + if (!content) return null + return { + role: entry.role === 'user' ? 'user' : 'agent', + content, + start_time: typeof entry.start_time === 'number' ? entry.start_time : undefined, + } + } + + const persistedTurnsSource = + messages && messages.length > 0 + ? messages + : messagesFromLive.length > 0 + ? messagesFromLive + : providerMessagesTurns.length > 0 + ? providerMessagesTurns + : artifactMessagesTurns.length > 0 + ? artifactMessagesTurns + : artifactTranscriptTurns.length > 0 + ? artifactTranscriptTurns + : transcriptObjectTurns.length > 0 + ? transcriptObjectTurns + : rawTranscriptTurns.length > 0 + ? rawTranscriptTurns + : transcriptTextTurns - const persistedTurns = (messages || messagesFromLive).map(toTranscriptTurn) - const liveTurns = liveTranscript + const persistedTurns: TranscriptTurn[] = persistedTurnsSource + .map(toTranscriptTurn) + .filter((turn: TranscriptTurn | null): turn is TranscriptTurn => Boolean(turn)) + const liveTurns: TranscriptTurn[] = liveTranscript .filter((entry) => entry?.content) - .map((entry) => toTranscriptTurn({ - role: entry.role, - content: entry.content, - start_time: entry.timestamp ? new Date(entry.timestamp).getTime() : undefined, - })) + .map((entry) => + toTranscriptTurn({ + role: entry.role, + content: entry.content, + start_time: + typeof entry.start_time === 'number' + ? entry.start_time + : entry.timestamp + ? new Date(entry.timestamp).getTime() + : undefined, + }), + ) + .filter((turn): turn is TranscriptTurn => Boolean(turn)) - const transcriptTurns = isLiveCall && liveTurns.length > 0 ? liveTurns : persistedTurns + const transcriptTurns: TranscriptTurn[] = + isLiveCall && liveTurns.length > 0 ? liveTurns : persistedTurns const hasTranscript = transcriptTurns.length > 0 const computeDuration = (): string | null => { @@ -223,6 +652,96 @@ export default function ObservabilityCallDetail() { const duration = computeDuration() + let durationSeconds: number | null = null + if (typeof callData?.duration_seconds === 'number') { + durationSeconds = callData.duration_seconds + } else { + const started = callData?.startedAt || callData?.started_at + const ended = callData?.endedAt || callData?.ended_at + if (started && ended) { + const diff = (new Date(ended).getTime() - new Date(started).getTime()) / 1000 + if (!Number.isNaN(diff) && diff > 0) durationSeconds = diff + } + } + + const waveformSegments = buildWaveformSegments(callData, transcriptTurns, durationSeconds) + + const agentDisplayName = callRecording.agent?.name || 'Agent' + const storedProviderTrace = callData?.provider_trace + + const hasTraceTab = + !!linkedTraceId || + !!storedProviderTrace || + hasElevenLabsProviderTraceCandidate || + hasSyntheticProviderTraceCandidate || + (!!callTrace && callTrace.spans.length > 0) + const hasTraceData = + !!storedProviderTrace || + (!!callTrace && (callTrace.spans?.length ?? 0) > 0) + const syntheticTraceSource = callTrace?.trace_source?.endsWith('_synthetic') + ? callTrace.trace_source + : null + const providerTraceSource = typeof storedProviderTrace?.trace_source === 'string' + ? storedProviderTrace.trace_source + : typeof storedProviderTrace?.source === 'string' + ? storedProviderTrace.source + : null + const hasArchivedTrace = storedProviderTrace?.storage === 's3' + const hasRealProviderTrace = !!providerTraceSource && !providerTraceSource.endsWith('_synthetic') + const showTabs = + hasTranscript || + hasTraceTab || + isLiveCall || + (!isLiveCall && hasRecordingCandidate) + const canRefreshProviderPayload = Boolean( + callRecording.provider_call_id && + (callRecording.provider_platform || resolvedProviderPlatform !== 'external'), + ) + const traceAvailabilityLabel = hasTraceData + ? hasArchivedTrace + ? 'Archived provider trace' + : syntheticTraceSource || providerTraceSource?.endsWith('_synthetic') + ? 'Synthetic provider trace' + : hasRealProviderTrace + ? 'Real provider trace' + : 'Deep trace available' + : hasSyntheticProviderTraceCandidate + ? 'Synthetic trace on refresh' + : callRecording.provider_platform + ? 'Provider report only (Level 1)' + : 'Trace unavailable' + const traceAvailabilityTone = hasTraceData + ? 'bg-emerald-50 text-emerald-700 border-emerald-200' + : 'bg-amber-50 text-amber-700 border-amber-200' + + const handleTranscriptTurnClick = (index: number, role: 'user' | 'agent') => { + setHighlightedTranscriptIndex(index) + if (traceSpans.length === 0) return + const userTurnIndex = + role === 'user' + ? transcriptTurns.slice(0, index + 1).filter((t: TranscriptTurn) => t.role === 'user').length - 1 + : Math.max( + 0, + transcriptTurns.slice(0, index + 1).filter((t: TranscriptTurn) => t.role === 'user').length - 1, + ) + const turnSpan = findTurnSpanForTranscriptIndex(userTurnIndex, traceSpans) + if (turnSpan?.span_id) { + setSelectedTraceSpanId(turnSpan.span_id) + setActiveTab('trace') + } + } + + const traceErrorDisplay = traceFetchError + ? getTraceFetchErrorDisplay(traceFetchError) + : { title: '', body: '' } + + const tabClass = (tab: DetailTab) => + `px-4 py-2.5 text-sm font-medium border-b-2 transition-colors ${ + activeTab === tab + ? 'border-indigo-600 text-indigo-700' + : 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-200' + }` + return (
{/* Header */} @@ -258,6 +777,17 @@ export default function ObservabilityCallDetail() { Run Evaluation )} + {canRefreshProviderPayload && ( + + )}
+
+

Observability

+
+ + {traceAvailabilityLabel} + +
+ Transcript: {transcriptTurns.length} turns +
+
+

Provider Call ID

+ {linkedTraceId && ( +
+

Trace ID

+

+ {linkedTraceId} +

+
+ )} {duration && (

Duration

@@ -331,99 +880,87 @@ export default function ObservabilityCallDetail() {
- {/* Provider-specific call details (Call Analysis, Cost, Latency, System Details) */} - {callRecording.provider_platform === 'retell' && callData && ( -
-

- - Provider Call Details - - retell - -

- -
- )} - - {callRecording.provider_platform === 'vapi' && callData && ( -
-

- - Provider Call Details - - vapi - -

- + {showTabs && ( +
+
)} - {callRecording.provider_platform === 'vobiz' && callData && ( -
- -
- )} + {(!showTabs || activeTab === 'overview') && ( + <> + {callData && ( + + )} - {(hasStorageRecording || providerRecordingUrl) && !isLiveCall && ( -
-

- - Call Recording -

- {audioLoading ? ( -
- - Loading recording... -
- ) : playbackUrl ? ( - - ) : null} -
+ + )} - {(hasTranscript || isLiveCall) && ( -
- {/* Left: Transcript */} -
-
-
-
- - - {isLiveCall ? 'Live Transcript' : 'Transcript'} - - {isLiveCall && ( - - Live - - )} - - {transcriptTurns.length} turns - -
- {(hasStorageRecording || providerRecordingUrl) && isLiveCall && playbackUrl && ( -
-
+ {showTabs && activeTab === 'transcript' && (hasTranscript || isLiveCall) && ( +
+
+
+
+ + + {isLiveCall ? 'Live Transcript' : 'Transcript'} + + {isLiveCall && ( + Live )} + + {transcriptTurns.length} turns +
+
+ {traceSpans.length > 0 && ( +

Click a message to open trace

+ )} + {!isLiveCall && audioLoading && ( + + )} + {!isLiveCall && playbackUrl && ( + <> +
+
-
- {transcriptTurns.length === 0 ? ( -

Waiting for speech…

- ) : ( - transcriptTurns.map((turn, index) => { +
+ {transcriptTurns.length === 0 ? ( +

Waiting for speech…

+ ) : ( + transcriptTurns.map((turn, index) => { const isUser = turn.role === 'user' + const isLinked = highlightedTranscriptIndex === index + const canLink = traceSpans.length > 0 return ( -
handleTranscriptTurnClick(index, turn.role)} + className={`max-w-[80%] rounded-2xl px-4 py-2.5 text-left transition-shadow ${ isUser ? 'bg-indigo-600 text-white rounded-br-sm' : 'bg-white border border-gray-200 text-gray-800 rounded-bl-sm' + } ${canLink ? 'hover:ring-2 hover:ring-indigo-300 cursor-pointer' : 'cursor-default'} ${ + isLinked ? 'ring-2 ring-violet-400 shadow-md' : '' }`} >
- {isUser ? 'Caller' : 'Agent'} + {isUser ? 'Caller' : agentDisplayName} {turn.start_time && ( @@ -455,114 +997,67 @@ export default function ObservabilityCallDetail() { )}

{turn.content}

-
+ ) }) - )} -
+ )}
+
+ )} - {/* Right: Call Summary */} -
-
-

- - Call Summary -

-
- {duration && ( -
-

- Duration -

-

{duration}

-
- )} - - {callData?.startedAt && ( -
-

- Started At -

-

- {new Date(callData.startedAt).toLocaleString()} -

-
- )} - - {callData?.endedAt && ( -
-

- Ended At -

-

- {new Date(callData.endedAt).toLocaleString()} -

-
- )} - - {(callData?.from_phone_number || callData?.to_phone_number) && ( -
-

- Phone Numbers -

-
- {callData.from_phone_number && ( -
- - - {callData.from_phone_number} - -
- )} - {callData.to_phone_number && ( -
- - - {callData.to_phone_number} - -
- )} -
-
- )} - - {callData?.endedReason && ( -
-

- End Reason -

- -
- )} - - {callData?.metadata && - Object.keys(callData.metadata).length > 0 && ( -
-

- Metadata -

-
- {Object.entries(callData.metadata).map(([key, value]) => ( -
- -
- {key}: - - {String(value)} - -
-
- ))} -
-
- )} - -
+ {showTabs && activeTab === 'trace' && ( +
+ {hasTraceTab ? ( + refetchTrace()} + selectedSpanId={selectedTraceSpanId} + onSelectedSpanIdChange={setSelectedTraceSpanId} + /> + ) : ( +
+ Trace is not available for this call yet.
-
+ )} +
+ )} + + {!showTabs && ( +
+ refetchTrace()} + selectedSpanId={selectedTraceSpanId} + onSelectedSpanIdChange={setSelectedTraceSpanId} + />
)} @@ -676,8 +1171,642 @@ export default function ObservabilityCallDetail() { ) } +function CallSummaryPanel({ + callData, + duration, +}: { + callData: ObservabilityCall['call_data'] + duration: string | null +}) { + if (!callData && !duration) return null + + return ( +
+
+

+ + Call Summary +

+
+ {duration && ( +
+

Duration

+

{duration}

+
+ )} + {callData?.startedAt && ( +
+

Started At

+

{new Date(callData.startedAt).toLocaleString()}

+
+ )} + {callData?.endedAt && ( +
+

Ended At

+

{new Date(callData.endedAt).toLocaleString()}

+
+ )} + {(callData?.from_phone_number || callData?.to_phone_number) && ( +
+

Phone Numbers

+
+ {callData.from_phone_number && ( +
+ + {callData.from_phone_number} +
+ )} + {callData.to_phone_number && ( +
+ + {callData.to_phone_number} +
+ )} +
+
+ )} + {callData?.endedReason && ( +
+

End Reason

+ +
+ )} + {callData?.metadata && typeof callData.metadata === 'object' && !Array.isArray(callData.metadata) && Object.keys(callData.metadata).length > 0 && ( +
+

Metadata

+
+ {Object.entries(callData.metadata as Record).map(([key, value]) => ( +
+ +
+ {key}: + {String(value)} +
+
+ ))} +
+
+ )} +
+
+
+ ) +} + +type InsightItem = { label: string; value: string } +type ChartDatum = { name: string; value: number } + +function ProviderInsightsPanel({ + platform, + callData, +}: { + platform?: string + callData: ObservabilityCall['call_data'] +}) { + const normalizedPlatform = String(platform || '').toLowerCase() + const providerLabel = normalizedPlatform ? getIntegrationPlatformLabel(normalizedPlatform as IntegrationPlatform) : 'Provider' + const raw = (callData || {}) as Record + const chartColors = ['#4f46e5', '#06b6d4', '#f59e0b', '#10b981', '#f97316', '#a855f7'] + + const formatValue = (value: unknown): string => { + if (value == null) return 'N/A' + if (typeof value === 'number') return Number.isFinite(value) ? value.toLocaleString() : 'N/A' + if (typeof value === 'boolean') return value ? 'Yes' : 'No' + const text = String(value).trim() + return text.length > 0 ? text : 'N/A' + } + + const formatCurrency = (value: unknown, digits = 4): string => { + if (typeof value !== 'number' || !Number.isFinite(value)) return 'N/A' + return `$${value.toFixed(digits)}` + } + + const toNumberOrNull = (value: unknown): number | null => { + if (typeof value !== 'number' || !Number.isFinite(value)) return null + return value + } + + const pushIfPresent = (items: InsightItem[], label: string, value: unknown) => { + if (value == null) return + if (typeof value === 'string' && value.trim().length === 0) return + items.push({ label, value: formatValue(value) }) + } + + const addChartPoint = (data: ChartDatum[], name: string, value: unknown) => { + const numericValue = toNumberOrNull(value) + if (numericValue == null) return + data.push({ name, value: numericValue }) + } + + const sections: Array<{ title: string; items: InsightItem[] }> = [] + const isElevenLabs = normalizedPlatform === 'elevenlabs' + const isRetell = normalizedPlatform === 'retell' + const elevenlabsMetadata = raw.raw_data?.metadata ?? raw.metadata + const elevenlabsCharging = elevenlabsMetadata?.charging + const elevenlabsRawTranscript = Array.isArray(raw.raw_data?.transcript) + ? raw.raw_data.transcript + : Array.isArray(raw.transcript) && raw.transcript.some((entry: unknown) => entry && typeof entry === 'object') + ? raw.transcript + : [] + const elevenlabsTokenTotals = { + prompt: 0, + completion: 0, + cached: 0, + } + if (isElevenLabs) { + const usageCandidates = [ + elevenlabsCharging?.llm_usage?.irreversible_generation, + elevenlabsCharging?.llm_usage?.initiated_generation, + ] + usageCandidates.forEach((usage: any) => { + const modelUsage = usage?.model_usage + if (!modelUsage || typeof modelUsage !== 'object') return + Object.values(modelUsage as Record).forEach((model: any) => { + elevenlabsTokenTotals.prompt += toNumberOrNull(model?.input?.tokens) ?? 0 + elevenlabsTokenTotals.cached += toNumberOrNull(model?.input_cache_read?.tokens) ?? 0 + elevenlabsTokenTotals.completion += toNumberOrNull(model?.output_total?.tokens) ?? 0 + }) + }) + } + + const summaryItems: InsightItem[] = [] + pushIfPresent(summaryItems, 'Summary', raw.analysis?.summary ?? raw.call_analysis?.call_summary ?? raw.summary) + pushIfPresent( + summaryItems, + 'Success Evaluation', + raw.analysis?.success_evaluation ?? raw.analysis?.successEvaluation ?? raw.call_analysis?.call_successful, + ) + pushIfPresent(summaryItems, 'User Sentiment', raw.call_analysis?.user_sentiment) + pushIfPresent(summaryItems, 'Ended Reason', raw.ended_reason ?? raw.endedReason ?? raw.disconnection_reason ?? raw.reason) + pushIfPresent(summaryItems, 'Call Type', raw.call_type ?? raw.type) + if (summaryItems.length > 0) sections.push({ title: 'Call Analysis', items: summaryItems }) + + const tokenItems: InsightItem[] = [] + const costBreakdown = raw.cost_breakdown || raw.costBreakdown || {} + const promptTokensValue = + costBreakdown.llm_prompt_tokens ?? + costBreakdown.llmPromptTokens ?? + (elevenlabsTokenTotals.prompt > 0 ? elevenlabsTokenTotals.prompt : undefined) + const completionTokensValue = + costBreakdown.llm_completion_tokens ?? + costBreakdown.llmCompletionTokens ?? + (elevenlabsTokenTotals.completion > 0 ? elevenlabsTokenTotals.completion : undefined) + const cachedTokensValue = + costBreakdown.llm_cached_prompt_tokens ?? + costBreakdown.llmCachedPromptTokens ?? + (elevenlabsTokenTotals.cached > 0 ? elevenlabsTokenTotals.cached : undefined) + pushIfPresent(tokenItems, 'LLM Prompt Tokens', promptTokensValue) + pushIfPresent(tokenItems, 'LLM Completion Tokens', completionTokensValue) + pushIfPresent( + tokenItems, + 'LLM Cached Prompt Tokens', + cachedTokensValue, + ) + pushIfPresent(tokenItems, 'TTS Characters', costBreakdown.tts_characters ?? costBreakdown.ttsCharacters) + if (tokenItems.length > 0) sections.push({ title: 'Token & Usage', items: tokenItems }) + const tokenChartData: ChartDatum[] = [] + addChartPoint(tokenChartData, 'Prompt', promptTokensValue) + addChartPoint(tokenChartData, 'Completion', completionTokensValue) + addChartPoint(tokenChartData, 'Cached', cachedTokensValue) + addChartPoint(tokenChartData, 'TTS Chars', costBreakdown.tts_characters ?? costBreakdown.ttsCharacters) + + const costItems: InsightItem[] = [] + const elevenlabsCreditsValue = + toNumberOrNull(raw.cost) ?? + toNumberOrNull(elevenlabsMetadata?.cost) ?? + (() => { + const callCharge = toNumberOrNull(elevenlabsCharging?.call_charge) ?? 0 + const llmCharge = toNumberOrNull(elevenlabsCharging?.llm_charge) ?? 0 + const platformCharge = toNumberOrNull(elevenlabsCharging?.platform_charge) ?? 0 + const total = callCharge + llmCharge + platformCharge + return total > 0 ? total : null + })() + if (isElevenLabs) { + pushIfPresent( + costItems, + 'Total Cost (Credits)', + elevenlabsCreditsValue != null ? `${elevenlabsCreditsValue.toLocaleString()} credits` : undefined, + ) + } else { + pushIfPresent(costItems, 'Total Cost', formatCurrency(raw.cost ?? raw.call_cost?.combined_cost ?? costBreakdown.total)) + } + pushIfPresent(costItems, 'Transport Cost', formatCurrency(costBreakdown.transport)) + pushIfPresent(costItems, 'STT Cost', formatCurrency(costBreakdown.stt)) + pushIfPresent(costItems, 'LLM Cost', formatCurrency(costBreakdown.llm)) + pushIfPresent(costItems, 'TTS Cost', formatCurrency(costBreakdown.tts)) + pushIfPresent(costItems, 'Provider Fee', formatCurrency(costBreakdown.vapi)) + if (isElevenLabs) { + pushIfPresent( + costItems, + 'Call (TTS + Infra)', + elevenlabsCharging?.call_charge != null ? `${Number(elevenlabsCharging.call_charge).toLocaleString()} credits` : undefined, + ) + pushIfPresent( + costItems, + 'LLM', + elevenlabsCharging?.llm_charge != null ? `${Number(elevenlabsCharging.llm_charge).toLocaleString()} credits` : undefined, + ) + pushIfPresent( + costItems, + 'Platform', + elevenlabsCharging?.platform_charge != null + ? `${Number(elevenlabsCharging.platform_charge).toLocaleString()} credits` + : undefined, + ) + pushIfPresent(costItems, 'LLM Unit Price (USD)', formatCurrency(elevenlabsCharging?.llm_price, 6)) + } + if (Array.isArray(raw.call_cost?.product_costs)) { + raw.call_cost.product_costs.forEach((entry: any) => { + if (!entry || typeof entry !== 'object') return + pushIfPresent(costItems, `${formatValue(entry.product)} Cost`, formatCurrency(entry.cost, 3)) + pushIfPresent(costItems, `${formatValue(entry.product)} Unit Price`, formatCurrency(entry.unit_price, 6)) + }) + } + if (costItems.length > 0) sections.push({ title: 'Costs', items: costItems }) + + const formatElevenLabsCategoryLabel = (category: string): string => + category + .split('_') + .map((part) => (part ? part[0].toUpperCase() + part.slice(1) : part)) + .join(' ') + + const buildElevenLabsCostChartData = (): ChartDatum[] => { + const data: ChartDatum[] = [] + const categoryUsage = elevenlabsCharging?.platform_usage?.category_usage + if (categoryUsage && typeof categoryUsage === 'object') { + Object.entries(categoryUsage as Record).forEach(([category, usage]) => { + addChartPoint(data, formatElevenLabsCategoryLabel(category), usage?.credits) + }) + } + + const llmCharge = toNumberOrNull(elevenlabsCharging?.llm_charge) + const callCharge = toNumberOrNull(elevenlabsCharging?.call_charge) + const platformCharge = toNumberOrNull(elevenlabsCharging?.platform_charge) + + if (data.length === 0) { + addChartPoint(data, 'Call (TTS + Infra)', callCharge) + addChartPoint(data, 'LLM', llmCharge) + addChartPoint(data, 'Platform', platformCharge) + } else if (llmCharge != null && llmCharge > 0) { + addChartPoint(data, 'LLM', llmCharge) + } + + if (data.length > 0 && elevenlabsCreditsValue != null) { + const segmentTotal = data.reduce((sum, entry) => sum + entry.value, 0) + const remainder = elevenlabsCreditsValue - segmentTotal + if (remainder > 0) addChartPoint(data, 'Other', remainder) + } + + if (data.length === 0 && elevenlabsCreditsValue != null) { + addChartPoint(data, 'Total Credits', elevenlabsCreditsValue) + } + + return data + } + + const costChartData: ChartDatum[] = [] + if (isElevenLabs) { + costChartData.push(...buildElevenLabsCostChartData()) + } else { + addChartPoint(costChartData, 'Transport', costBreakdown.transport) + addChartPoint(costChartData, 'STT', costBreakdown.stt) + addChartPoint(costChartData, 'LLM', costBreakdown.llm) + addChartPoint(costChartData, 'TTS', costBreakdown.tts) + addChartPoint(costChartData, 'Provider', costBreakdown.vapi) + if (Array.isArray(raw.call_cost?.product_costs)) { + raw.call_cost.product_costs.forEach((entry: any) => { + if (!entry || typeof entry !== 'object') return + addChartPoint(costChartData, formatValue(entry.product), entry.cost) + }) + } + } + + const formatCostChartValue = (value: number) => + isElevenLabs ? `${value.toLocaleString()} credits` : `$${value.toFixed(4)}` + + const latencyItems: InsightItem[] = [] + const latencyStats = raw.analysis?.latency_stats || raw.artifact?.performanceMetrics || {} + pushIfPresent(latencyItems, 'Model Latency Avg (ms)', latencyStats.model_latency_avg ?? latencyStats.modelLatencyAverage) + pushIfPresent(latencyItems, 'Voice Latency Avg (ms)', latencyStats.voice_latency_avg ?? latencyStats.voiceLatencyAverage) + pushIfPresent( + latencyItems, + 'Transcriber Latency Avg (ms)', + latencyStats.transcriber_latency_avg ?? latencyStats.transcriberLatencyAverage, + ) + pushIfPresent( + latencyItems, + 'Endpointing Latency Avg (ms)', + latencyStats.endpointing_latency_avg ?? latencyStats.endpointingLatencyAverage, + ) + pushIfPresent(latencyItems, 'Turn Latency Avg (ms)', latencyStats.turn_latency_avg ?? latencyStats.turnLatencyAverage) + pushIfPresent(latencyItems, 'P50 (ms)', latencyStats.p50) + pushIfPresent(latencyItems, 'P90 (ms)', latencyStats.p90) + pushIfPresent(latencyItems, 'P95 (ms)', latencyStats.p95) + pushIfPresent(latencyItems, 'P99 (ms)', latencyStats.p99) + + const retellLatency = raw.latency || {} + ;['e2e', 'asr', 'llm', 'tts'].forEach((key) => { + const stats = retellLatency[key] + if (!stats || typeof stats !== 'object') return + pushIfPresent(latencyItems, `${key.toUpperCase()} P50 (ms)`, stats.p50) + pushIfPresent(latencyItems, `${key.toUpperCase()} P90 (ms)`, stats.p90) + pushIfPresent(latencyItems, `${key.toUpperCase()} Max (ms)`, stats.max) + }) + if (isElevenLabs && latencyItems.length === 0) { + const asrSamples: number[] = [] + const llmSamples: number[] = [] + const ttsSamples: number[] = [] + elevenlabsRawTranscript.forEach((entry: any) => { + const metrics = entry?.conversation_turn_metrics?.metrics + if (!metrics || typeof metrics !== 'object') return + const pickMetricMs = (keys: string[]) => { + for (const key of keys) { + const elapsed = toNumberOrNull(metrics?.[key]?.elapsed_time) + if (elapsed != null) return elapsed * 1000 + } + return null + } + const asrMs = pickMetricMs(['convai_turn_asr_latency', 'convai_asr_trailing_service_latency']) + const llmMs = pickMetricMs([ + 'convai_llm_service_tt_last_sentence', + 'convai_llm_service_ttf_sentence', + 'convai_llm_service_ttfb', + ]) + const ttsMs = pickMetricMs(['convai_tts_service_ttfb']) + if (asrMs != null) asrSamples.push(asrMs) + if (llmMs != null) llmSamples.push(llmMs) + if (ttsMs != null) ttsSamples.push(ttsMs) + }) + const avg = (arr: number[]) => (arr.length ? arr.reduce((acc, n) => acc + n, 0) / arr.length : null) + const avgAsr = avg(asrSamples) + const avgLlm = avg(llmSamples) + const avgTts = avg(ttsSamples) + pushIfPresent(latencyItems, 'ASR Latency Avg (ms)', avgAsr != null ? Math.round(avgAsr) : undefined) + pushIfPresent(latencyItems, 'LLM Latency Avg (ms)', avgLlm != null ? Math.round(avgLlm) : undefined) + pushIfPresent(latencyItems, 'TTS Latency Avg (ms)', avgTts != null ? Math.round(avgTts) : undefined) + const turnAvg = + avgAsr != null || avgLlm != null || avgTts != null + ? Math.round((avgAsr ?? 0) + (avgLlm ?? 0) + (avgTts ?? 0)) + : null + pushIfPresent(latencyItems, 'Turn Latency Avg (ms)', turnAvg ?? undefined) + } + if (latencyItems.length > 0) sections.push({ title: 'Latency', items: latencyItems }) + const latencyChartData: ChartDatum[] = [] + addChartPoint(latencyChartData, 'Model', latencyStats.model_latency_avg ?? latencyStats.modelLatencyAverage) + addChartPoint(latencyChartData, 'Voice', latencyStats.voice_latency_avg ?? latencyStats.voiceLatencyAverage) + addChartPoint( + latencyChartData, + 'Transcriber', + latencyStats.transcriber_latency_avg ?? latencyStats.transcriberLatencyAverage, + ) + addChartPoint( + latencyChartData, + 'Endpointing', + latencyStats.endpointing_latency_avg ?? latencyStats.endpointingLatencyAverage, + ) + addChartPoint(latencyChartData, 'Turn', latencyStats.turn_latency_avg ?? latencyStats.turnLatencyAverage) + ;['e2e', 'asr', 'llm', 'tts'].forEach((key) => { + const stats = retellLatency[key] + if (!stats || typeof stats !== 'object') return + addChartPoint(latencyChartData, `${key.toUpperCase()} P50`, stats.p50) + }) + if (isElevenLabs && latencyChartData.length === 0 && latencyItems.length > 0) { + const latencyByLabel: Record = { + 'ASR Latency Avg (ms)': 'ASR', + 'LLM Latency Avg (ms)': 'LLM', + 'TTS Latency Avg (ms)': 'TTS', + 'Turn Latency Avg (ms)': 'Turn', + } + latencyItems.forEach((item) => { + const label = latencyByLabel[item.label] + if (!label) return + const numeric = Number(item.value.replace(/[^0-9.]/g, '')) + if (!Number.isNaN(numeric)) addChartPoint(latencyChartData, label, numeric) + }) + } + + const systemItems: InsightItem[] = [] + pushIfPresent(systemItems, 'Provider Call ID', raw.call_id ?? raw.id) + pushIfPresent(systemItems, 'Assistant ID', raw.assistant_id ?? raw.assistantId ?? raw.agent_id) + pushIfPresent(systemItems, 'Status', raw.call_status ?? raw.status) + pushIfPresent(systemItems, 'Start Time', raw.start_timestamp ?? raw.startedAt) + pushIfPresent(systemItems, 'End Time', raw.end_timestamp ?? raw.endedAt) + pushIfPresent(systemItems, 'Duration (seconds)', raw.duration_seconds ?? raw.call_cost?.total_duration_seconds ?? (typeof raw.duration_ms === 'number' ? Math.round(raw.duration_ms / 1000) : undefined)) + if (systemItems.length > 0) sections.push({ title: 'System Details', items: systemItems }) + + const metaItems: InsightItem[] = [] + const metadata = raw.metadata + if (metadata && typeof metadata === 'object' && !Array.isArray(metadata)) { + Object.entries(metadata).forEach(([key, value]) => { + metaItems.push({ label: key, value: formatValue(value) }) + }) + } + if (metaItems.length > 0) sections.push({ title: 'Metadata', items: metaItems }) + + const totalCostValue = isElevenLabs + ? elevenlabsCreditsValue + : isRetell + ? (toNumberOrNull(raw.call_cost?.combined_cost) ?? toNumberOrNull(raw.cost) ?? toNumberOrNull(costBreakdown.total)) + : (toNumberOrNull(raw.cost) ?? toNumberOrNull(costBreakdown.total)) + const promptTokens = toNumberOrNull(promptTokensValue) ?? 0 + const completionTokens = toNumberOrNull(completionTokensValue) ?? 0 + const cachedTokens = toNumberOrNull(cachedTokensValue) ?? 0 + const totalTokens = promptTokens + completionTokens + cachedTokens + const parseLatencyItem = (label: string): number | null => { + const item = latencyItems.find((entry) => entry.label === label) + if (!item) return null + const numeric = Number(item.value.replace(/[^0-9.]/g, '')) + return Number.isNaN(numeric) ? null : numeric + } + const derivedTurnLatency = + parseLatencyItem('Turn Latency Avg (ms)') ?? + parseLatencyItem('P50 (ms)') ?? + (() => { + const values = ['ASR Latency Avg (ms)', 'LLM Latency Avg (ms)', 'TTS Latency Avg (ms)'] + .map(parseLatencyItem) + .filter((v): v is number => v != null) + if (values.length === 0) return null + return values.reduce((acc, value) => acc + value, 0) + })() + const turnLatency = + toNumberOrNull(latencyStats.turn_latency_avg ?? latencyStats.turnLatencyAverage) ?? + toNumberOrNull(latencyStats.p50) ?? + (isRetell ? toNumberOrNull(retellLatency.e2e?.p50) : null) ?? + derivedTurnLatency + const metricSectionTitles = new Set(['Token & Usage', 'Costs', 'Latency']) + const coreSections = sections.filter((section) => !metricSectionTitles.has(section.title)) + const metricSections = sections.filter((section) => metricSectionTitles.has(section.title)) + + if (sections.length === 0) return null + + return ( +
+

+ + Provider Insights + + {providerLabel} + +

+ +
+
+
+

+ {isElevenLabs ? 'Total Cost (Credits)' : 'Total Cost'} +

+

+ {totalCostValue == null + ? 'N/A' + : isElevenLabs + ? `${totalCostValue.toLocaleString()} credits` + : `$${totalCostValue.toFixed(4)}`} +

+
+
+

Total LLM Tokens

+

+ {isRetell && totalTokens === 0 + ? 'Not reported' + : totalTokens > 0 + ? totalTokens.toLocaleString() + : 'N/A'} +

+
+
+

Turn Latency

+

{turnLatency == null ? 'N/A' : `${Math.round(turnLatency).toLocaleString()} ms`}

+
+
+ + {(costChartData.length > 0 || tokenChartData.length > 0 || latencyChartData.length > 0) && ( +
+

Dashboards

+
+ {costChartData.length > 0 && ( +
+

+ {isElevenLabs ? 'Credits Distribution' : 'Cost Distribution'} +

+
+ + + 1 ? 2 : 0} + > + {costChartData.map((_entry, index) => ( + + ))} + + formatCostChartValue(value)} /> + + +
+
+ )} + + {tokenChartData.length > 0 && ( +
+

Token & Usage Volume

+
+ + + + + + value.toLocaleString()} /> + + + +
+
+ )} + + {latencyChartData.length > 0 && ( +
+

Latency Breakdown (ms)

+
+ + + + + + `${value.toFixed(1)} ms`} /> + + + +
+
+ )} +
+
+ )} + + {coreSections.map((section) => ( +
+

{section.title}

+
+ {section.items.map((item) => ( +
+

+ {item.label} +

+

{item.value}

+
+ ))} +
+
+ ))} + + {metricSections.length > 0 && ( +
+ + Detailed Metric Tables + +
+ {metricSections.map((section) => ( +
+

{section.title}

+
+ {section.items.map((item) => ( +
+

+ {item.label} +

+

{item.value}

+
+ ))} +
+
+ ))} +
+
+ )} + +
+ + Raw Provider Payload + +
+ 
+              {JSON.stringify(raw, null, 2)}
+
+
+
+
+
+ ) +} + function EventBadge({ event }: { event?: string }) { if (!event) return — + const normalizedEvent = String(event).toLowerCase() const variants: Record< string, @@ -706,8 +1835,8 @@ function EventBadge({ event }: { event?: string }) { }, } - const variant = variants[event.toLowerCase()] || { - label: event, + const variant = variants[normalizedEvent] || { + label: String(event), bg: 'bg-gray-50', text: 'text-gray-600', border: 'border-gray-200', @@ -725,6 +1854,7 @@ function EventBadge({ event }: { event?: string }) { } function EndReasonBadge({ reason }: { reason: string }) { + const normalizedReason = String(reason || '').toLowerCase() const colors: Record = { 'customer-hungup': 'bg-amber-50 text-amber-700 border-amber-200', 'assistant-ended-call': 'bg-blue-50 text-blue-700 border-blue-200', @@ -732,8 +1862,8 @@ function EndReasonBadge({ reason }: { reason: string }) { error: 'bg-rose-50 text-rose-700 border-rose-200', } - const colorClass = colors[reason.toLowerCase()] || 'bg-gray-50 text-gray-700 border-gray-200' - const label = reason + const colorClass = colors[normalizedReason] || 'bg-gray-50 text-gray-700 border-gray-200' + const label = String(reason) .replace(/-/g, ' ') .replace(/\b\w/g, (l) => l.toUpperCase()) @@ -748,7 +1878,7 @@ function EndReasonBadge({ reason }: { reason: string }) { function PlatformBadge({ platform }: { platform?: string }) { if (!platform) return N/A - const normalized = platform.toLowerCase() as IntegrationPlatform + const normalized = String(platform).toLowerCase() as IntegrationPlatform const label = getIntegrationPlatformLabel(normalized) const logo = getIntegrationPlatformLogo(normalized) diff --git a/frontend/src/pages/observability/ObservabilityCalls.tsx b/frontend/src/pages/observability/ObservabilityCalls.tsx index e9908949..5988ce15 100644 --- a/frontend/src/pages/observability/ObservabilityCalls.tsx +++ b/frontend/src/pages/observability/ObservabilityCalls.tsx @@ -3,7 +3,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { useNavigate } from 'react-router-dom' import { Eye, RefreshCw, PhoneCall, Info, Activity, CheckCircle, - Clock, Loader, Trash2, PhoneOff, PhoneIncoming, + Clock, Loader, Trash2, PhoneOff, PhoneIncoming, Copy, Check, ChevronDown, ChevronUp, } from 'lucide-react' import { motion } from 'framer-motion' @@ -11,20 +11,87 @@ import Button from '../../components/Button' import ConfirmModal from '../../components/ConfirmModal' import { apiClient } from '../../lib/api' import { getIntegrationPlatformLabel, getIntegrationPlatformLogo } from '../../config/providers' -import { IntegrationPlatform, ObservabilityCall } from '../../types/api' +import { + IntegrationPlatform, + ObservabilityCall, + ObservabilityCallsSummary, + ObservabilityLiveLatencyResponse, +} from '../../types/api' import { CallAgentLink } from './CallAgentLink' +const LIVE_INGEST_PLATFORMS = new Set(['pipecat', 'livekit', 'external']) +const PROVIDER_PLATFORMS = new Set(['retell', 'vapi', 'elevenlabs', 'efficientai']) + +type CallSourceFilter = 'all' | 'providers' | 'live' | 'simulated' | 'in_progress' + +function isSimulatedLiveCall(call: ObservabilityCall): boolean { + const platform = (call.provider_platform || '').toLowerCase() + const providerCallId = call.provider_call_id || '' + return platform === 'pipecat' && providerCallId.startsWith('pipecat-live-') +} + +function isProviderCall(call: ObservabilityCall): boolean { + const platform = (call.provider_platform || '').toLowerCase() + if (PROVIDER_PLATFORMS.has(platform)) return true + return (call.source || '').toUpperCase() === 'PLAYGROUND' +} + +function isLiveIngestCall(call: ObservabilityCall): boolean { + const platform = (call.provider_platform || '').toLowerCase() + return LIVE_INGEST_PLATFORMS.has(platform) && !isSimulatedLiveCall(call) +} + +function matchesSourceFilter(call: ObservabilityCall, filter: CallSourceFilter): boolean { + if (filter === 'all') return true + if (filter === 'providers') return isProviderCall(call) + if (filter === 'live') return isLiveIngestCall(call) || Boolean(call.is_live) + if (filter === 'simulated') return isSimulatedLiveCall(call) + if (filter === 'in_progress') return Boolean(call.is_live) + return true +} + export default function ObservabilityCalls() { const navigate = useNavigate() const queryClient = useQueryClient() const [selectedCallId, setSelectedCallId] = useState(null) const [eventFilter, setEventFilter] = useState<'all' | 'call_ended' | 'call_started' | 'other'>('all') + const [sourceFilter, setSourceFilter] = useState('all') + const [copiedWebhook, setCopiedWebhook] = useState(null) + const [showSetupGuide, setShowSetupGuide] = useState(false) + + const webhookBaseUrl = useMemo(() => { + const envBase = (import.meta as { env?: { VITE_API_BASE_URL?: string } }).env?.VITE_API_BASE_URL + if (envBase && envBase.trim()) return envBase.replace(/\/$/, '') + if (typeof window !== 'undefined') return window.location.origin + return '' + }, []) + + const webhookEndpoints = useMemo( + () => [ + '/api/v1/observability/calls/webhook/{api_key}', + '/api/v1/observability/calls/webhook/retell/{api_key}', + '/api/v1/observability/calls/webhook/elevenlabs/{api_key}', + '/api/v1/observability/calls/webhook/vapi/{api_key}', + ], + [], + ) + + const copyWebhookUrl = async (value: string) => { + try { + await navigator.clipboard.writeText(value) + setCopiedWebhook(value) + setTimeout(() => setCopiedWebhook((current) => (current === value ? null : current)), 2000) + } catch { + // no-op: clipboard may be blocked by browser permissions + } + } const deleteMutation = useMutation({ mutationFn: (callShortId: string) => apiClient.deleteObservabilityCall(callShortId), onSuccess: () => { setSelectedCallId(null) queryClient.invalidateQueries({ queryKey: ['observability-calls'] }) + queryClient.invalidateQueries({ queryKey: ['observability-calls-summary'] }) }, }) @@ -42,20 +109,57 @@ export default function ObservabilityCalls() { }, }) + const { data: summary } = useQuery({ + queryKey: ['observability-calls-summary'], + queryFn: () => apiClient.getObservabilityCallsSummary(), + }) + const liveDashboardEnabled = Boolean(summary?.live_feature_flags?.live_dashboard_enabled) + const liveAggregatesEnabled = Boolean(summary?.live_feature_flags?.live_aggregates_enabled) + const { data: liveLatency } = useQuery({ + queryKey: ['observability-live-latency'], + queryFn: () => apiClient.getObservabilityLiveLatencyMetrics(), + enabled: liveDashboardEnabled && liveAggregatesEnabled, + refetchInterval: 5000, + }) + const summaryStats = useMemo(() => { - const total = calls.length - const ended = calls.filter((c) => c.call_event === 'call_ended').length - const started = calls.filter((c) => c.call_event === 'call_started').length + const total = summary?.total_calls ?? calls.length + const ended = summary?.event_breakdown?.call_ended ?? calls.filter((c) => c.call_event === 'call_ended').length + const started = summary?.event_breakdown?.call_started ?? calls.filter((c) => c.call_event === 'call_started').length const other = total - ended - started - return { total, ended, started, other } - }, [calls]) + return { + total, + ended, + started, + other, + totalMinutes: summary?.total_minutes ?? 0, + avgDurationMs: summary?.avg_duration_ms ?? summary?.avg_latency_ms ?? 0, + traceLinkedCalls: summary?.trace_linked_calls ?? calls.filter((c) => !!c.trace_id).length, + traceLinkRatePct: summary?.trace_link_rate_pct ?? (total > 0 ? (calls.filter((c) => !!c.trace_id).length / total) * 100 : 0), + traceAvailableCalls: summary?.trace_available_calls ?? calls.filter((c) => !!c.trace_id).length, + traceAvailableRatePct: + summary?.trace_available_rate_pct ?? + (total > 0 ? (calls.filter((c) => !!c.trace_id).length / total) * 100 : 0), + evaluatedCalls: summary?.evaluated_calls ?? calls.filter((c) => !!c.evaluator_result_id).length, + evaluatedRatePct: summary?.evaluated_rate_pct ?? (total > 0 ? (calls.filter((c) => !!c.evaluator_result_id).length / total) * 100 : 0), + } + }, [calls, summary]) + + const sourceFilterCounts = useMemo(() => ({ + all: calls.length, + providers: calls.filter(isProviderCall).length, + live: calls.filter((c) => isLiveIngestCall(c) || Boolean(c.is_live)).length, + simulated: calls.filter(isSimulatedLiveCall).length, + in_progress: calls.filter((c) => Boolean(c.is_live)).length, + }), [calls]) const filteredCalls = useMemo(() => { - if (eventFilter === 'all') return calls - if (eventFilter === 'call_ended') return calls.filter((c) => c.call_event === 'call_ended') - if (eventFilter === 'call_started') return calls.filter((c) => c.call_event === 'call_started') - return calls.filter((c) => c.call_event !== 'call_ended' && c.call_event !== 'call_started') - }, [calls, eventFilter]) + const bySource = calls.filter((c) => matchesSourceFilter(c, sourceFilter)) + if (eventFilter === 'all') return bySource + if (eventFilter === 'call_ended') return bySource.filter((c) => c.call_event === 'call_ended') + if (eventFilter === 'call_started') return bySource.filter((c) => c.call_event === 'call_started') + return bySource.filter((c) => c.call_event !== 'call_ended' && c.call_event !== 'call_started') + }, [calls, eventFilter, sourceFilter]) const formatTimestamp = (timestamp: string): string => { const date = new Date(timestamp) @@ -95,19 +199,53 @@ export default function ObservabilityCalls() { {/* Webhook info */}
-
- POST call data to{' '} - /api/v1/observability/calls{' '} - with the{' '} - X-EFFICIENTAI-API-KEY{' '} - header. Payloads are stored per organization and surfaced here. +
+

Use webhook URLs with your API key:

+
+ {webhookEndpoints.map((path) => { + const fullUrl = `${webhookBaseUrl}${path}` + const copied = copiedWebhook === fullUrl + return ( +
+ + {fullUrl} + + +
+ ) + })} +
+ Include trace_id in payloads to link traces. + + {showSetupGuide && ( +
+

Required fields: id, trace_id, and messages.

+

Use ISO-8601 UTC for startedAt and endedAt.

+

Provider-specific payload mapping is documented in docs/telemetry/provider-webhook-map.md.

+
+ )}
{/* Summary Stats */} {!isLoading && calls.length > 0 && (
-

Ended

-

{summaryStats.ended}

+

Total Minutes

+

+ {summaryStats.totalMinutes.toFixed(1)} +

+

minutes

-

Started

-

{summaryStats.started}

+

Avg Duration

+

+ {Math.round(summaryStats.avgDurationMs)} +

+

ms

-

Other

-

{summaryStats.other}

+

Ended

+

{summaryStats.ended}

+
+
+
+

Trace Available

+

{summaryStats.traceAvailableCalls}

+
+
+ +
+
+

{summaryStats.traceAvailableRatePct.toFixed(1)}% available

+
+
+
+
+

Evaluated

+

{summaryStats.evaluatedCalls}

+
+
+ +
+
+

{summaryStats.evaluatedRatePct.toFixed(1)}% of calls

+
)} + {liveDashboardEnabled && liveAggregatesEnabled && liveLatency && ( +
+
+

Live Quality (Rolling)

+ 1m / 5m windows +
+
+ + + +
+
+ Samples: {liveLatency.windows['60s'].sample_count} (1m) / {liveLatency.windows['300s'].sample_count} (5m) +
+
+ )} + {/* Call Records Table */}
@@ -168,25 +365,49 @@ export default function ObservabilityCalls() {

Call Records

{calls.length > 0 && ( -
- {([ - { key: 'all' as const, label: 'All', count: summaryStats.total }, - { key: 'call_ended' as const, label: 'Ended', count: summaryStats.ended }, - { key: 'call_started' as const, label: 'Started', count: summaryStats.started }, - { key: 'other' as const, label: 'Other', count: summaryStats.other }, - ] as const).map(({ key, label, count }) => ( - - ))} +
+
+ {([ + { key: 'all' as const, label: 'All sources', count: sourceFilterCounts.all }, + { key: 'providers' as const, label: 'Providers', count: sourceFilterCounts.providers }, + { key: 'live' as const, label: 'Live ingest', count: sourceFilterCounts.live }, + { key: 'simulated' as const, label: 'Simulated', count: sourceFilterCounts.simulated }, + { key: 'in_progress' as const, label: 'In progress', count: sourceFilterCounts.in_progress }, + ] as const).map(({ key, label, count }) => ( + + ))} +
+ | +
+ {([ + { key: 'all' as const, label: 'All events', count: summaryStats.total }, + { key: 'call_ended' as const, label: 'Ended', count: summaryStats.ended }, + { key: 'call_started' as const, label: 'Started', count: summaryStats.started }, + { key: 'other' as const, label: 'Other', count: summaryStats.other }, + ] as const).map(({ key, label, count }) => ( + + ))} +
)}
@@ -208,7 +429,10 @@ export default function ObservabilityCalls() {

No matching calls found.

)}
+ + {(callRecording.provider_platform || '').toLowerCase() === 'elevenlabs' && ( +
+

Provider Trace (ElevenLabs)

+ {providerTraceLoading ? ( +
+ + Loading provider trace... +
+ ) : providerTraceError ? ( +

+ Provider trace is not available yet. It appears after the conversation is complete. +

+ ) : providerTrace?.spans?.length ? ( + + ) : ( +

No provider trace spans found for this call.

+ )} +
+ )}
+} + +export interface ExternalProviderAgentListResponse { + agents: ExternalProviderAgent[] + has_more: boolean + next_cursor?: string | null +} + // VoiceBundle Types export enum ModelProvider { OPENAI = 'openai', @@ -2156,12 +2170,34 @@ export interface ObservabilityCallData { endedReason?: string recording_s3_key?: string recording_url?: string + integration_id?: string + provider_trace?: { + source?: string + trace_source?: string + trace_id?: string + ingested_at?: string + storage?: 'inline' | 's3' + trace_s3_key?: string + normalized_trace?: Record + otlp_traces?: Record + } duration_seconds?: number + trace_id?: string agent_name?: string _agent_ref?: string | number direction?: string messages?: Array<{ role: string; content: string; start_time?: number; end_time?: number }> live_transcript?: Array<{ role: string; content: string; timestamp?: string; start_time?: number }> + speaker_segments?: Array<{ + speaker?: string + role?: string + text?: string + content?: string + start?: number + end?: number + start_time?: number + end_time?: number + }> metadata?: Record call_short_id?: string } @@ -2176,7 +2212,10 @@ export interface ObservabilityCall { source?: string | null provider_platform?: string | null provider_call_id?: string | null + trace_id?: string | null + last_live_event_ts?: string | null agent_id?: string | null + evaluator_result_id?: string | null agent?: ObservabilityCallAgent | null created_at?: string | null updated_at?: string | null @@ -2184,3 +2223,67 @@ export interface ObservabilityCall { live_transcript?: Array<{ role: string; content: string; timestamp?: string }> display_name?: string | null } + +export interface ObservabilityCallsSummary { + total_calls: number + total_minutes: number + avg_duration_ms?: number + avg_latency_ms: number + trace_linked_calls?: number + trace_link_rate_pct?: number + trace_available_calls?: number + trace_available_rate_pct?: number + evaluated_calls?: number + evaluated_rate_pct?: number + event_breakdown?: { + call_ended?: number + call_failed?: number + call_started?: number + other?: number + } + live_feature_flags?: { + live_ingest_enabled?: boolean + live_aggregates_enabled?: boolean + live_dashboard_enabled?: boolean + } +} + +export interface ObservabilityLiveLatencyMetricBreakdown { + p50_ms?: number | null + p90_ms?: number | null + p95_ms?: number | null + sample_count: number +} + +export interface ObservabilityLiveLatencyWindow extends ObservabilityLiveLatencyMetricBreakdown { + window_seconds: number + metrics: Record +} + +export interface ObservabilityLiveLatencyResponse { + scope: 'workspace' | 'agent' + agent_id?: string + platform?: string | null + windows: { + '60s': ObservabilityLiveLatencyWindow + '300s': ObservabilityLiveLatencyWindow + } +} + +export interface ObservabilityTraceSpan { + span_id: string | null + parent_span_id: string | null + name: string + start_time: number | null + end_time: number | null + duration_ms: number | null + attributes: Record + status: string | null +} + +export interface ObservabilityCallTrace { + trace_id: string + root_span_id?: string | null + spans: ObservabilityTraceSpan[] + trace_source?: string | null +} diff --git a/observability/grafana/provisioning/datasources/datasource.yml b/observability/grafana/provisioning/datasources/datasource.yml index 3f775d92..a1fe12b1 100644 --- a/observability/grafana/provisioning/datasources/datasource.yml +++ b/observability/grafana/provisioning/datasources/datasource.yml @@ -29,3 +29,9 @@ datasources: httpHeaderName1: "X-Scope-OrgID" secureJsonData: httpHeaderValue1: "platform" + + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false diff --git a/observability/tempo/tempo.yml b/observability/tempo/tempo.yml new file mode 100644 index 00000000..8959bd28 --- /dev/null +++ b/observability/tempo/tempo.yml @@ -0,0 +1,22 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 24h + +storage: + trace: + backend: local + local: + path: /var/tempo/traces diff --git a/pyproject.toml b/pyproject.toml index 94e8e3c6..980c8cc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,8 @@ dependencies = [ "typing-extensions>=4.9.0", "onnxruntime>=1.17.0", "prometheus-fastapi-instrumentator>=7.0.0", + "opentelemetry-sdk>=1.26.0", + "opentelemetry-exporter-otlp-proto-http>=1.26.0", "plivo>=4.47.0", "openpyxl>=3.1.0", "weasyprint>=62.0", diff --git a/scripts/elevenlabs_monitor_bridge.py b/scripts/elevenlabs_monitor_bridge.py new file mode 100644 index 00000000..1450b2e5 --- /dev/null +++ b/scripts/elevenlabs_monitor_bridge.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Forward ElevenLabs monitor websocket events to EfficientAI live ingest. + +Usage: + EFFICIENTAI_API_KEY=... \ + EFFICIENTAI_WORKSPACE_ID=... \ + ELEVENLABS_API_KEY=... \ + python scripts/elevenlabs_monitor_bridge.py --conversation-id conv_xxx +""" + +from __future__ import annotations + +import argparse +import asyncio +import os + +from app.services.observability.elevenlabs_monitor_bridge import ElevenLabsMonitorBridge + + +def _args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Bridge ElevenLabs live monitor events to EfficientAI.") + parser.add_argument("--conversation-id", required=True, help="ElevenLabs conversation id") + parser.add_argument( + "--base-url", + default=os.environ.get("EFFICIENTAI_BASE_URL", "http://localhost:8000"), + help="EfficientAI API base URL", + ) + parser.add_argument( + "--platform", + default=os.environ.get("EFFICIENTAI_PROVIDER_PLATFORM", "elevenlabs"), + help="Platform label sent to live ingest", + ) + return parser.parse_args() + + +async def _main() -> None: + args = _args() + efficientai_api_key = os.environ.get("EFFICIENTAI_API_KEY") + workspace_id = os.environ.get("EFFICIENTAI_WORKSPACE_ID") + elevenlabs_api_key = os.environ.get("ELEVENLABS_API_KEY") + + if not efficientai_api_key: + raise RuntimeError("EFFICIENTAI_API_KEY is required") + if not elevenlabs_api_key: + raise RuntimeError("ELEVENLABS_API_KEY is required") + + bridge = ElevenLabsMonitorBridge( + conversation_id=args.conversation_id, + elevenlabs_api_key=elevenlabs_api_key, + efficientai_api_key=efficientai_api_key, + workspace_id=workspace_id, + efficientai_base_url=args.base_url, + provider_platform=args.platform, + ) + await bridge.run() + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/pipecat_efficientai_observer.py b/scripts/pipecat_efficientai_observer.py new file mode 100644 index 00000000..34e82108 --- /dev/null +++ b/scripts/pipecat_efficientai_observer.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Bridge Pipecat bots to EfficientAI live observability ingest. + +Copy this file into your Pipecat bot project. + +Environment: + EFFICIENTAI_API_KEY + EFFICIENTAI_WORKSPACE_ID + EFFICIENTAI_BASE_URL=http://localhost:8000 + +Minimum event sequence: + 1. call.started + 2. turn.user / turn.assistant (one or more — required for transcript + trace) + 3. call.ended + +Recommended wiring (Pipecat 0.0.99+ turn events): + + observer = EfficientAILiveObserver(call_id=session_id, platform="pipecat") + await observer.start_call() + wire_turn_events(user_aggregator, assistant_aggregator, observer) + ... + await observer.end_call() + +Alternative (frame tap — place once in pipeline, after STT and LLM): + + pipeline = Pipeline([..., observer.as_pipecat_processor(), ...]) +""" + +from __future__ import annotations + +import json +import logging +import os +import uuid +from datetime import UTC, datetime +from typing import Any, Optional + +import httpx + +logger = logging.getLogger("efficientai.pipecat_observer") + + +def _load_pipecat_modules(): + """Import frame/processor symbols from pipecat (external bots) or efficientai (fork).""" + errors: list[str] = [] + for module_prefix in ("pipecat", "efficientai"): + try: + frames = __import__(f"{module_prefix}.frames.frames", fromlist=["frames"]) + processors = __import__( + f"{module_prefix}.processors.frame_processor", + fromlist=["frame_processor"], + ) + return { + "TranscriptionFrame": frames.TranscriptionFrame, + "LLMTextFrame": frames.LLMTextFrame, + "TTSTextFrame": frames.TTSTextFrame, + "AggregatedTextFrame": getattr(frames, "AggregatedTextFrame", None), + "LLMFullResponseEndFrame": frames.LLMFullResponseEndFrame, + "TextFrame": getattr(frames, "TextFrame", None), + "FrameDirection": processors.FrameDirection, + "FrameProcessor": processors.FrameProcessor, + } + except ImportError as exc: + errors.append(f"{module_prefix}: {exc}") + raise ImportError( + "Install pipecat-ai or efficientai to use Pipecat integration helpers. " + + "; ".join(errors) + ) + + +class EfficientAILiveObserver: + """HTTP client + Pipecat hooks for live observability.""" + + def __init__( + self, + *, + call_id: str, + platform: str = "pipecat", + agent_ref: Optional[str] = None, + trace_id: Optional[str] = None, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + workspace_id: Optional[str] = None, + ) -> None: + self.call_id = call_id + self.platform = platform + self.agent_ref = agent_ref + self.trace_id = trace_id + self.base_url = (base_url or os.environ.get("EFFICIENTAI_BASE_URL") or "http://localhost:8000").rstrip("/") + self.api_key = api_key or os.environ.get("EFFICIENTAI_API_KEY", "") + self.workspace_id = workspace_id or os.environ.get("EFFICIENTAI_WORKSPACE_ID", "") + self._seq = 0 + self.call_short_id: Optional[str] = None + self._agent_buffer = "" + self._turn_count = 0 + + async def start_call(self) -> Optional[str]: + ack = await self._post("call.started", {"startedAt": self._now_iso(), "status": "in_progress"}) + self.call_short_id = ack.get("call_short_id") + if ack.get("trace_id"): + self.trace_id = str(ack["trace_id"]) + logger.info("EfficientAI call started call_id=%s short_id=%s", self.call_id, self.call_short_id) + return self.call_short_id + + async def emit_turn(self, role: str, content: str, *, latency: Optional[dict[str, Any]] = None) -> None: + if not content.strip(): + return + normalized = role.strip().lower() + event_type = "turn.assistant" if normalized in {"assistant", "agent", "bot"} else "turn.user" + payload: dict[str, Any] = { + "content": content.strip(), + "role": "assistant" if event_type == "turn.assistant" else "user", + } + if latency: + payload["latency"] = latency + await self._post(event_type, payload) + self._turn_count += 1 + logger.info("EfficientAI turn emitted role=%s seq=%s chars=%s", payload["role"], self._seq, len(content)) + + async def end_call( + self, + *, + recording_url: Optional[str] = None, + duration_seconds: Optional[float] = None, + trace_id: Optional[str] = None, + ) -> None: + if trace_id: + self.trace_id = trace_id + payload: dict[str, Any] = {"endedAt": self._now_iso(), "status": "ended"} + if recording_url: + payload["recording_url"] = recording_url + if duration_seconds is not None: + payload["duration_seconds"] = duration_seconds + await self._post("call.ended", payload) + if self._turn_count == 0: + logger.warning( + "EfficientAI call ended with 0 turns (call_id=%s). " + "Wire wire_turn_events() or as_pipecat_processor() so transcript/trace populate.", + self.call_id, + ) + else: + logger.info("EfficientAI call ended call_id=%s turns=%s", self.call_id, self._turn_count) + + async def _post(self, event_type: str, payload: dict[str, Any]) -> dict[str, Any]: + if not self.api_key: + raise RuntimeError("EFFICIENTAI_API_KEY is required") + self._seq += 1 + body = { + "event_id": f"pipecat-{self.call_id}-{self._seq}-{uuid.uuid4().hex[:8]}", + "call_id": self.call_id, + "event_type": event_type, + "seq": self._seq, + "event_ts": self._now_iso(), + "platform": self.platform, + "payload": payload, + } + if self.trace_id: + body["trace_id"] = self.trace_id + if self.agent_ref: + body["agent_ref"] = self.agent_ref + + headers = { + "X-API-Key": self.api_key, + "Content-Type": "application/json", + } + if self.workspace_id: + headers["X-Workspace-Id"] = self.workspace_id + + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post( + f"{self.base_url}/api/v1/observability/live/events", + headers=headers, + content=json.dumps(body), + ) + resp.raise_for_status() + return resp.json() + + @staticmethod + def _now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + def as_pipecat_processor(self): + """Return a FrameProcessor that emits turns from common Pipecat frame types.""" + mods = _load_pipecat_modules() + TranscriptionFrame = mods["TranscriptionFrame"] + LLMTextFrame = mods["LLMTextFrame"] + TTSTextFrame = mods["TTSTextFrame"] + AggregatedTextFrame = mods["AggregatedTextFrame"] + LLMFullResponseEndFrame = mods["LLMFullResponseEndFrame"] + TextFrame = mods["TextFrame"] + FrameDirection = mods["FrameDirection"] + FrameProcessor = mods["FrameProcessor"] + + agent_frame_types = tuple( + cls + for cls in (LLMTextFrame, TTSTextFrame, AggregatedTextFrame) + if cls is not None + ) + immediate_agent_types = tuple( + cls for cls in (TTSTextFrame, AggregatedTextFrame) if cls is not None + ) + observer = self + + class _Processor(FrameProcessor): + async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + try: + if isinstance(frame, TranscriptionFrame) and getattr(frame, "text", ""): + await observer.emit_turn("user", frame.text) + elif agent_frame_types and isinstance(frame, agent_frame_types) and getattr(frame, "text", ""): + if direction == FrameDirection.DOWNSTREAM: + if immediate_agent_types and isinstance(frame, immediate_agent_types): + await observer.emit_turn("agent", frame.text) + else: + observer._agent_buffer += frame.text + elif isinstance(frame, LLMFullResponseEndFrame) and observer._agent_buffer.strip(): + await observer.emit_turn("agent", observer._agent_buffer.strip()) + observer._agent_buffer = "" + elif ( + TextFrame is not None + and isinstance(frame, TextFrame) + and not isinstance(frame, TranscriptionFrame) + and getattr(frame, "text", "") + and direction == FrameDirection.DOWNSTREAM + and frame.__class__.__name__ not in {"InterimTranscriptionFrame"} + ): + # Fallback for bots that only emit generic TextFrame downstream. + await observer.emit_turn("agent", frame.text) + except Exception as exc: + logger.warning("EfficientAI frame emit failed: %s", exc) + await self.push_frame(frame, direction) + + return _Processor() + + +def wire_turn_events(user_aggregator, assistant_aggregator, observer: EfficientAILiveObserver) -> None: + """Wire Pipecat 0.0.99+ context aggregator turn events (recommended).""" + + @user_aggregator.event_handler("on_user_turn_stopped") + async def _on_user_turn_stopped(aggregator, strategy, message): + content = getattr(message, "content", None) + if isinstance(content, str) and content.strip(): + await observer.emit_turn("user", content) + + @assistant_aggregator.event_handler("on_assistant_turn_stopped") + async def _on_assistant_turn_stopped(aggregator, message, *_extra): + content = getattr(message, "content", None) + if isinstance(content, str) and content.strip(): + await observer.emit_turn("assistant", content) + + +def wire_transcript_processor(transcript_processor, observer: EfficientAILiveObserver) -> None: + """Wire deprecated TranscriptProcessor.on_transcript_update (Pipecat < 0.0.99).""" + + @transcript_processor.event_handler("on_transcript_update") + async def _on_transcript_update(processor, frame): + messages = getattr(frame, "messages", None) or [] + for msg in messages: + content = getattr(msg, "content", None) + role = getattr(msg, "role", None) or "user" + if isinstance(content, str) and content.strip(): + await observer.emit_turn(role, content) + + +if __name__ == "__main__": + import asyncio + + logging.basicConfig(level=logging.INFO) + + async def _demo() -> None: + obs = EfficientAILiveObserver(call_id=f"pipecat-demo-{uuid.uuid4().hex[:8]}") + await obs.start_call() + await obs.emit_turn("user", "Hello from external Pipecat bridge") + await obs.emit_turn("assistant", "Hi there!", latency={"llm_ms": 320, "tts_ms": 180}) + await obs.end_call() + print("done", obs.call_short_id) + + asyncio.run(_demo()) diff --git a/scripts/test_elevenlabs_trace_e2e.sh b/scripts/test_elevenlabs_trace_e2e.sh new file mode 100644 index 00000000..ad6d745d --- /dev/null +++ b/scripts/test_elevenlabs_trace_e2e.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: +# ELEVENLABS_API_KEY=... ./scripts/test_elevenlabs_trace_e2e.sh conv_xxx +# +# This helper validates that ElevenLabs OTLP spans are present +# for a completed conversation. + +if [[ $# -lt 1 ]]; then + echo "Usage: ELEVENLABS_API_KEY=... $0 " + exit 1 +fi + +if [[ -z "${ELEVENLABS_API_KEY:-}" ]]; then + echo "ELEVENLABS_API_KEY is required" + exit 1 +fi + +CONV_ID="$1" +BASE_URL="${ELEVENLABS_BASE_URL:-https://api.elevenlabs.io}" + +echo "Fetching conversation OTLP payload for: ${CONV_ID}" + +RAW_JSON="$(mktemp)" +curl -sS "${BASE_URL}/v1/convai/conversations/${CONV_ID}?format=opentelemetry" \ + -H "xi-api-key: ${ELEVENLABS_API_KEY}" > "${RAW_JSON}" + +echo "Conversation status:" +jq -r '.status' "${RAW_JSON}" + +echo "Span names:" +jq -r '.otlp_traces.resourceSpans[]?.scopeSpans[]?.spans[]?.name' "${RAW_JSON}" | sort -u + +USER_TURNS="$(jq '[.otlp_traces.resourceSpans[]?.scopeSpans[]?.spans[]? | select(.name=="elevenlabs.recv.user_transcript")] | length' "${RAW_JSON}")" +AGENT_TURNS="$(jq '[.otlp_traces.resourceSpans[]?.scopeSpans[]?.spans[]? | select(.name=="elevenlabs.recv.agent_response")] | length' "${RAW_JSON}")" + +echo "User turn spans: ${USER_TURNS}" +echo "Agent turn spans: ${AGENT_TURNS}" + +if [[ "${USER_TURNS}" -lt 1 || "${AGENT_TURNS}" -lt 1 ]]; then + echo "Missing expected turn spans." + exit 2 +fi + +echo "OTLP trace payload looks valid." diff --git a/scripts/test_live_observability_feed.py b/scripts/test_live_observability_feed.py new file mode 100755 index 00000000..8f6406fd --- /dev/null +++ b/scripts/test_live_observability_feed.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +"""Simulate an ongoing Pipecat call and stream the live transcript feed. + +Posts incremental events to POST /api/v1/observability/live/events while +subscribing to GET /api/v1/observability/calls/{call_short_id}/live-events (SSE). + +Usage: + export EFFICIENTAI_API_KEY="..." # Settings → API Keys + export EFFICIENTAI_WORKSPACE_ID="..." # optional; auto-resolved from /workspaces + python3 scripts/test_live_observability_feed.py + + # Or login instead of API key: + export EFFICIENTAI_EMAIL="you@example.com" + export EFFICIENTAI_PASSWORD="..." + python3 scripts/test_live_observability_feed.py + +Optional: + BASE_URL=http://localhost:8000 TURN_DELAY_SEC=3 NUM_TURNS=4 python3 ... +""" + +from __future__ import annotations + +import json +import os +import sys +import threading +import time +import uuid +from datetime import UTC, datetime +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8000").rstrip("/") +API_PREFIX = "/api/v1" +TURN_DELAY_SEC = float(os.environ.get("TURN_DELAY_SEC", "3")) +NUM_TURNS = int(os.environ.get("NUM_TURNS", "4")) +_PLACEHOLDER_KEYS = frozenset( + { + "", + "your-api-key", + "your-key-from-settings", + "changeme", + "replace-me", + } +) + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _request( + method: str, + path: str, + *, + headers: dict[str, str] | None = None, + body: dict[str, Any] | None = None, + stream: bool = False, +): + url = f"{BASE_URL}{path}" + data = None + req_headers = {"Accept": "application/json", **(headers or {})} + if body is not None: + data = json.dumps(body).encode("utf-8") + req_headers["Content-Type"] = "application/json" + req = Request(url, data=data, headers=req_headers, method=method) + if stream: + return urlopen(req, timeout=300) + try: + with urlopen(req, timeout=60) as resp: + raw = resp.read().decode("utf-8") + if not raw.strip(): + return resp.status, {} + return resp.status, json.loads(raw) + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + if exc.code == 401 and "Invalid API key" in detail: + raise RuntimeError( + f"{method} {path} failed HTTP 401: Invalid API key. " + "Copy the full key from Settings → API Keys (shown only once at creation). " + "If you regenerated the key, update EFFICIENTAI_API_KEY in your shell." + ) from exc + raise RuntimeError(f"{method} {path} failed HTTP {exc.code}: {detail}") from exc + except URLError as exc: + raise RuntimeError(f"{method} {path} failed: {exc}") from exc + + +def _auth_headers(api_key: str | None, bearer: str | None, workspace_id: str | None) -> dict[str, str]: + headers: dict[str, str] = {} + if api_key: + headers["X-API-Key"] = api_key + elif bearer: + headers["Authorization"] = f"Bearer {bearer}" + else: + raise RuntimeError("Set EFFICIENTAI_API_KEY or EFFICIENTAI_EMAIL + EFFICIENTAI_PASSWORD") + if workspace_id: + headers["X-Workspace-Id"] = workspace_id + return headers + + +def _login() -> str: + email = os.environ.get("EFFICIENTAI_EMAIL", "").strip() + password = os.environ.get("EFFICIENTAI_PASSWORD", "") + if not email or not password: + raise RuntimeError("Set EFFICIENTAI_API_KEY or EFFICIENTAI_EMAIL + EFFICIENTAI_PASSWORD") + _, payload = _request( + "POST", + f"{API_PREFIX}/auth/login", + body={"email": email, "password": password}, + ) + token = payload.get("access_token") + if not token: + raise RuntimeError("Login succeeded but no access_token in response") + return token + + +def _resolve_workspace(headers: dict[str, str]) -> str: + preset = os.environ.get("EFFICIENTAI_WORKSPACE_ID", "").strip() + if preset: + return preset + _, payload = _request("GET", f"{API_PREFIX}/workspaces", headers=headers) + workspaces = payload if isinstance(payload, list) else payload.get("items", []) + if not workspaces: + raise RuntimeError("No workspaces found; set EFFICIENTAI_WORKSPACE_ID") + ws_id = workspaces[0].get("id") + ws_name = workspaces[0].get("name", "unknown") + print(f"Using workspace: {ws_name} ({ws_id})") + return str(ws_id) + + +def _post_live_event(headers: dict[str, str], payload: dict[str, Any]) -> dict[str, Any]: + _, body = _request("POST", f"{API_PREFIX}/observability/live/events", headers=headers, body=payload) + return body + + +def _transcript_len(snap: dict[str, Any]) -> int: + top = snap.get("live_transcript") + if isinstance(top, list): + return len(top) + call_data = snap.get("call_data") + if isinstance(call_data, dict): + nested = call_data.get("live_transcript") + if isinstance(nested, list): + return len(nested) + return 0 + + +def _poll_call(headers: dict[str, str], call_short_id: str) -> dict[str, Any]: + _, body = _request("GET", f"{API_PREFIX}/observability/calls/{call_short_id}", headers=headers) + return body + + +class LiveFeedWatcher: + """Subscribe to SSE live-events and print transcript turns as they arrive.""" + + def __init__(self, headers: dict[str, str], call_short_id: str) -> None: + self._headers = headers + self._call_short_id = call_short_id + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self.seen_turns = 0 + + def start(self) -> None: + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread: + self._thread.join(timeout=5) + + def _run(self) -> None: + path = f"{API_PREFIX}/observability/calls/{self._call_short_id}/live-events" + url = f"{BASE_URL}{path}" + req = Request(url, headers={**self._headers, "Accept": "text/event-stream"}, method="GET") + try: + with urlopen(req, timeout=300) as resp: + for raw_line in resp: + if self._stop.is_set(): + break + line = raw_line.decode("utf-8", errors="replace").strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if not data: + continue + try: + turn = json.loads(data) + except json.JSONDecodeError: + continue + self.seen_turns += 1 + role = turn.get("role", "?") + content = (turn.get("content") or "")[:120] + ts = turn.get("event_ts") or turn.get("timestamp") or "" + print(f" [SSE feed] {role}: {content!r} ({ts})", flush=True) + except Exception as exc: # noqa: BLE001 — background watcher + if not self._stop.is_set(): + print(f" [SSE feed] stream ended: {exc}", flush=True) + + +def _validate_api_key(api_key: str | None) -> None: + if not api_key: + return + if api_key.lower() in _PLACEHOLDER_KEYS: + raise RuntimeError( + 'EFFICIENTAI_API_KEY is still the placeholder "your-api-key". ' + "Paste the full key shown once when you created it in Settings → API Keys." + ) + if len(api_key) < 20: + raise RuntimeError( + "EFFICIENTAI_API_KEY looks too short. Copy the full key from Settings → API Keys " + "(not the key name/label)." + ) + + +def main() -> int: + api_key = os.environ.get("EFFICIENTAI_API_KEY", "").strip() or None + bearer = None + _validate_api_key(api_key) + if not api_key: + print("Logging in with EFFICIENTAI_EMAIL...") + bearer = _login() + + headers = _auth_headers(api_key, bearer, None) + workspace_id = _resolve_workspace(headers) + headers["X-Workspace-Id"] = workspace_id + + print(f"\n== Checking live feature flags ({BASE_URL}) ==") + _, summary = _request("GET", f"{API_PREFIX}/observability/calls/summary", headers=headers) + flags = summary.get("live_feature_flags") or {} + print(json.dumps(flags, indent=2)) + if not flags.get("live_ingest_enabled"): + print( + "\nERROR: OBSERVABILITY_LIVE_INGEST_ENABLED is false on the API.\n" + "Restart the api service with live flags enabled (see docker-compose.yml).\n", + file=sys.stderr, + ) + return 1 + + call_id = f"pipecat-live-{int(time.time())}" + seq = 0 + + def next_event(event_type: str, payload: dict[str, Any], event_suffix: str) -> dict[str, Any]: + nonlocal seq + seq += 1 + return { + "event_id": f"evt-{call_id}-{event_suffix}", + "call_id": call_id, + "event_type": event_type, + "seq": seq, + "event_ts": _now_iso(), + "platform": "pipecat", + "payload": payload, + } + + print(f"\n== Starting simulated call: {call_id} ==") + started = _post_live_event( + headers, + next_event("call.started", {"direction": "inbound", "startedAt": _now_iso()}, "start"), + ) + call_short_id = started.get("call_short_id") + trace_id = started.get("trace_id") + print(f" call_short_id: {call_short_id}") + print(f" trace_id: {trace_id}") + print(f" UI: {BASE_URL}/observability/calls/{call_short_id}") + + if not call_short_id: + print("ERROR: ingest did not return call_short_id", file=sys.stderr) + return 1 + + watcher = LiveFeedWatcher(headers, call_short_id) + print("\n== SSE live feed (open Observability UI — list auto-refreshes every 3s for live calls) ==") + watcher.start() + + dialog = [ + ("turn.user", {"content": "Hello, can you hear me?"}), + ("turn.assistant", {"content": "Yes, I can hear you clearly.", "latency": {"llm_ms": 380, "tts_ms": 210}}), + ("turn.user", {"content": "What is the weather like today?"}), + ("turn.assistant", {"content": "I do not have live weather data, but I can help you look it up.", "latency": {"llm_ms": 520, "tts_ms": 190}}), + ("turn.user", {"content": "Thanks, that is all."}), + ("turn.assistant", {"content": "You're welcome. Goodbye!", "latency": {"llm_ms": 290, "tts_ms": 160}}), + ] + turns_to_send = dialog[: NUM_TURNS * 2] + + for idx, (event_type, payload) in enumerate(turns_to_send): + time.sleep(TURN_DELAY_SEC) + ack = _post_live_event(headers, next_event(event_type, payload, f"turn-{idx + 1}")) + role = event_type.split(".")[-1] + snippet = (payload.get("content") or "")[:80] + print(f"\n [posted] seq={seq} {role}: {snippet!r} duplicate={ack.get('duplicate')}", flush=True) + + snap = _poll_call(headers, call_short_id) + last_ts = snap.get("last_live_event_ts") + print( + f" [poll] call_event={snap.get('call_event')} " + f"turns={_transcript_len(snap)} last_live_event_ts={last_ts}", + flush=True, + ) + + if flags.get("live_aggregates_enabled"): + _, metrics = _request( + "GET", + f"{API_PREFIX}/observability/live/metrics/latency?platform=pipecat", + headers=headers, + ) + sample_count = metrics.get("windows", {}).get("300s", {}).get("sample_count", 0) + p90 = metrics.get("windows", {}).get("300s", {}).get("metrics", {}).get("llm_ms", {}).get("p90") + print(f" [metrics] rolling 300s samples={sample_count} llm_p90={p90}", flush=True) + + time.sleep(TURN_DELAY_SEC) + ended = _post_live_event( + headers, + next_event("call.ended", {"endedAt": _now_iso(), "status": "ended"}, "end"), + ) + print(f"\n== Call ended duplicate={ended.get('duplicate')} ==") + + time.sleep(1) + watcher.stop() + final = _poll_call(headers, call_short_id) + print(f"\nFinal state: call_event={final.get('call_event')} turns={_transcript_len(final)}") + print(f"SSE turns received: {watcher.seen_turns}") + print(f"\nOpen in UI: {BASE_URL}/observability/calls/{call_short_id}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except RuntimeError as exc: + print(f"\nERROR: {exc}", file=sys.stderr) + print( + "\nSet credentials before running (no inline comments on export lines):\n" + " export EFFICIENTAI_API_KEY='paste-full-key-here'\n" + " export EFFICIENTAI_WORKSPACE_ID='your-workspace-uuid'\n" + "\nYou are already in efficientAI/ if pwd shows that path — skip 'cd efficientAI'.\n", + file=sys.stderr, + ) + raise SystemExit(1) from exc diff --git a/scripts/test_observability_e2e.sh b/scripts/test_observability_e2e.sh new file mode 100755 index 00000000..65851e09 --- /dev/null +++ b/scripts/test_observability_e2e.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Quick smoke test for Product Observability APIs. +# +# Usage: +# export EFFICIENTAI_API_KEY="your-key-from-settings" +# ./scripts/test_observability_e2e.sh +# +# Optional: +# BASE_URL=http://localhost:8000 ./scripts/test_observability_e2e.sh + +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8000}" +API_PREFIX="${API_PREFIX:-/api/v1/observability}" +API_KEY="${EFFICIENTAI_API_KEY:-}" + +if [[ -z "$API_KEY" ]]; then + echo "Set EFFICIENTAI_API_KEY (Settings → API Keys in the UI)." >&2 + exit 1 +fi + +TRACE_ID="$(python3 - <<'PY' +import secrets +print(secrets.token_hex(16)) +PY +)" +CALL_ID="test-$(date +%s)" + +echo "== Health ==" +curl -sf "$BASE_URL/health" | head -c 120 +echo "" + +echo "== Webhook ingest (flat payload + trace_id) ==" +INGEST=$(curl -s -w "\nHTTP:%{http_code}" -X POST \ + "$BASE_URL${API_PREFIX}/calls/webhook/$API_KEY" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $API_KEY" \ + -d "{ + \"id\": \"$CALL_ID\", + \"provider_platform\": \"external\", + \"startedAt\": \"2026-08-07T09:00:00.000Z\", + \"endedAt\": \"2026-08-07T09:01:30.000Z\", + \"trace_id\": \"$TRACE_ID\", + \"messages\": [{\"role\": \"user\", \"content\": \"hello\"}, {\"role\": \"bot\", \"content\": \"hi there\"}] + }") +HTTP_CODE="${INGEST##*HTTP:}" +BODY="${INGEST%HTTP:*}" +echo "$BODY" | python3 -m json.tool 2>/dev/null || echo "$BODY" +if [[ "$HTTP_CODE" != "201" ]]; then + echo "Ingest failed (HTTP $HTTP_CODE)" >&2 + exit 1 +fi + +CALL_SHORT_ID="$(echo "$BODY" | python3 -c "import json,sys; print(json.load(sys.stdin).get('call_short_id',''))")" +if [[ -z "$CALL_SHORT_ID" ]]; then + echo "Could not parse call_short_id from ingest response" >&2 + exit 1 +fi + +echo "" +echo "== List calls ==" +curl -sf "$BASE_URL${API_PREFIX}/calls" -H "X-API-Key: $API_KEY" | python3 -m json.tool | head -40 + +echo "" +echo "== Summary ==" +SUMMARY_JSON="$(curl -sf "$BASE_URL${API_PREFIX}/calls/summary" -H "X-API-Key: $API_KEY")" +echo "$SUMMARY_JSON" | python3 -m json.tool +python3 - <<'PY' "$SUMMARY_JSON" +import json +import sys + +payload = json.loads(sys.argv[1]) +required = {"total_calls", "total_minutes", "avg_latency_ms"} +missing = sorted(required - payload.keys()) +if missing: + raise SystemExit(f"Summary payload missing fields: {', '.join(missing)}") +PY + +echo "" +echo "== Trace fetch (expect 502 if Tempo/cloud has no spans for this trace_id) ==" +TRACE_RESP=$(curl -s -w "\nHTTP:%{http_code}" \ + "$BASE_URL${API_PREFIX}/calls/$CALL_SHORT_ID/trace" \ + -H "X-API-Key: $API_KEY") +TRACE_HTTP="${TRACE_RESP##*HTTP:}" +TRACE_BODY="${TRACE_RESP%HTTP:*}" +if [[ "$TRACE_HTTP" == "200" ]]; then + echo "$TRACE_BODY" | python3 -m json.tool | head -60 +else + echo "Trace query returned HTTP $TRACE_HTTP (normal for webhook-only test without live spans)" + echo "$TRACE_BODY" +fi + +echo "" +echo "== Retell synthetic trace smoke ==" +RETELL_CALL_ID="retell-smoke-$(date +%s)" +RETELL_INGEST=$(curl -s -w "\nHTTP:%{http_code}" -X POST \ + "$BASE_URL${API_PREFIX}/calls/webhook/retell/$API_KEY" \ + -H "Content-Type: application/json" \ + -d "{ + \"event\": \"call_ended\", + \"call\": { + \"call_id\": \"$RETELL_CALL_ID\", + \"call_status\": \"ended\", + \"start_timestamp\": 1714423232000, + \"end_timestamp\": 1714423257000, + \"transcript_object\": [ + {\"role\": \"user\", \"content\": \"hello\", \"words\": [{\"start\": 0.4, \"end\": 0.9}]}, + {\"role\": \"agent\", \"content\": \"hi there\", \"words\": [{\"start\": 1.2, \"end\": 2.0}]} + ], + \"latency\": {\"asr\": {\"p50\": 120}, \"llm\": {\"p50\": 350}, \"tts\": {\"p50\": 220}} + } + }") +RETELL_HTTP="${RETELL_INGEST##*HTTP:}" +RETELL_BODY="${RETELL_INGEST%HTTP:*}" +if [[ "$RETELL_HTTP" != "201" ]]; then + echo "Retell ingest failed (HTTP $RETELL_HTTP)" >&2 + echo "$RETELL_BODY" + exit 1 +fi +RETELL_SHORT_ID="$(echo "$RETELL_BODY" | python3 -c "import json,sys; print(json.load(sys.stdin).get('call_short_id',''))")" +RETELL_TRACE=$(curl -s -w "\nHTTP:%{http_code}" \ + "$BASE_URL${API_PREFIX}/calls/$RETELL_SHORT_ID/trace" \ + -H "X-API-Key: $API_KEY") +RETELL_TRACE_HTTP="${RETELL_TRACE##*HTTP:}" +RETELL_TRACE_BODY="${RETELL_TRACE%HTTP:*}" +if [[ "$RETELL_TRACE_HTTP" != "200" ]]; then + echo "Retell synthetic trace fetch failed (HTTP $RETELL_TRACE_HTTP)" >&2 + echo "$RETELL_TRACE_BODY" + exit 1 +fi +python3 - <<'PY' "$RETELL_TRACE_BODY" +import json, sys +payload = json.loads(sys.argv[1]) +assert payload.get("trace_source") == "retell_synthetic", payload +assert any(span.get("name") == "stt" for span in payload.get("spans", [])), payload +PY + +echo "" +echo "== Vapi synthetic trace smoke ==" +VAPI_CALL_ID="vapi-smoke-$(date +%s)" +VAPI_INGEST=$(curl -s -w "\nHTTP:%{http_code}" -X POST \ + "$BASE_URL${API_PREFIX}/calls/webhook/vapi/$API_KEY" \ + -H "Content-Type: application/json" \ + -d "{ + \"id\": \"$VAPI_CALL_ID\", + \"status\": \"ended\", + \"startedAt\": \"2026-08-07T09:00:00.000Z\", + \"endedAt\": \"2026-08-07T09:00:10.000Z\", + \"messages\": [ + {\"role\": \"user\", \"message\": \"hello\", \"secondsFromStart\": 0.5, \"duration\": 900}, + {\"role\": \"assistant\", \"message\": \"hi there\", \"secondsFromStart\": 1.6, \"duration\": 1200} + ], + \"artifact\": { + \"performanceMetrics\": { + \"modelLatencyAverage\": 320, + \"voiceLatencyAverage\": 480, + \"transcriberLatencyAverage\": 210, + \"endpointingLatencyAverage\": 140 + } + } + }") +VAPI_HTTP="${VAPI_INGEST##*HTTP:}" +VAPI_BODY="${VAPI_INGEST%HTTP:*}" +if [[ "$VAPI_HTTP" != "201" ]]; then + echo "Vapi ingest failed (HTTP $VAPI_HTTP)" >&2 + echo "$VAPI_BODY" + exit 1 +fi +VAPI_SHORT_ID="$(echo "$VAPI_BODY" | python3 -c "import json,sys; print(json.load(sys.stdin).get('call_short_id',''))")" +VAPI_TRACE=$(curl -s -w "\nHTTP:%{http_code}" \ + "$BASE_URL${API_PREFIX}/calls/$VAPI_SHORT_ID/trace" \ + -H "X-API-Key: $API_KEY") +VAPI_TRACE_HTTP="${VAPI_TRACE##*HTTP:}" +VAPI_TRACE_BODY="${VAPI_TRACE%HTTP:*}" +if [[ "$VAPI_TRACE_HTTP" != "200" ]]; then + echo "Vapi synthetic trace fetch failed (HTTP $VAPI_TRACE_HTTP)" >&2 + echo "$VAPI_TRACE_BODY" + exit 1 +fi +python3 - <<'PY' "$VAPI_TRACE_BODY" +import json, sys +payload = json.loads(sys.argv[1]) +assert payload.get("trace_source") == "vapi_synthetic", payload +assert any(span.get("name") == "llm" for span in payload.get("spans", [])), payload +PY + +echo "" +echo "Done." +echo " Call short id: $CALL_SHORT_ID" +echo " Trace id: $TRACE_ID" +echo " UI: $BASE_URL/observability/calls/$CALL_SHORT_ID" diff --git a/src/efficientai/utils/tracing/service_attributes.py b/src/efficientai/utils/tracing/service_attributes.py index a8645cdc..a3387015 100644 --- a/src/efficientai/utils/tracing/service_attributes.py +++ b/src/efficientai/utils/tracing/service_attributes.py @@ -11,6 +11,7 @@ where applicable and EfficientAI-specific conventions for additional context. """ +import os from typing import TYPE_CHECKING, Any, Dict, List, Optional # Import for type checking only @@ -23,6 +24,18 @@ from opentelemetry.trace import Span +def _should_include_transcript_attributes() -> bool: + value = os.getenv("OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS") + if value is not None: + return value.strip().lower() in {"1", "true", "yes", "on"} + try: + from app.config import settings # Imported lazily to avoid hard dependency in SDK-only contexts. + + return bool(getattr(settings, "OBSERVABILITY_TRACING_INCLUDE_TRANSCRIPTS", True)) + except Exception: + return True + + def _get_gen_ai_system_from_service_name(service_name: str) -> str: """Extract the standardized gen_ai.system value from a service class name. @@ -94,6 +107,7 @@ def add_tts_span_attributes( span.set_attribute("gen_ai.operation.name", operation_name) span.set_attribute("gen_ai.output.type", "speech") span.set_attribute("voice_id", voice_id) + span.set_attribute("tts.provider", service_name.replace("TTSService", "").lower()) # Add optional attributes if text: @@ -101,6 +115,7 @@ def add_tts_span_attributes( if character_count is not None: span.set_attribute("metrics.character_count", character_count) + span.set_attribute("tts.characters", character_count) if ttfb is not None: span.set_attribute("metrics.ttfb", ttfb) @@ -152,10 +167,12 @@ def add_stt_span_attributes( span.set_attribute("gen_ai.request.model", model) span.set_attribute("gen_ai.operation.name", operation_name) span.set_attribute("vad_enabled", vad_enabled) + span.set_attribute("stt.provider", service_name.replace("STTService", "").lower()) # Add optional attributes - if transcript: + if transcript and _should_include_transcript_attributes(): span.set_attribute("transcript", transcript) + span.set_attribute("stt.transcript", transcript) if is_final is not None: span.set_attribute("is_final", is_final) diff --git a/src/efficientai/utils/tracing/service_decorators.py b/src/efficientai/utils/tracing/service_decorators.py index fc6382f1..be412802 100644 --- a/src/efficientai/utils/tracing/service_decorators.py +++ b/src/efficientai/utils/tracing/service_decorators.py @@ -43,6 +43,23 @@ R = TypeVar("R") +def _resolve_operation_span_name(operation: str, service_obj: object) -> str: + if operation in {"llm_tool_call", "llm_tool_result"}: + return "tool_call" + + bundle_type = getattr(service_obj, "_observability_bundle_type", None) + if bundle_type == "s2s" and operation in { + "llm", + "stt", + "tts", + "llm_setup", + "llm_request", + "llm_response", + }: + return "s2s" + return operation + + def _noop_decorator(func): """No-op fallback decorator when tracing is unavailable. @@ -142,7 +159,7 @@ async def tracing_context(self, text): return service_class_name = self.__class__.__name__ - span_name = "tts" + span_name = _resolve_operation_span_name("tts", self) # Get parent context turn_context = get_current_turn_context() @@ -249,7 +266,7 @@ async def wrapper(self, transcript, is_final, language=None): return await f(self, transcript, is_final, language) service_class_name = self.__class__.__name__ - span_name = "stt" + span_name = _resolve_operation_span_name("stt", self) # Get the turn context first, then fall back to service context turn_context = get_current_turn_context() @@ -331,7 +348,7 @@ async def wrapper(self, context, *args, **kwargs): return await f(self, context, *args, **kwargs) service_class_name = self.__class__.__name__ - span_name = "llm" + span_name = _resolve_operation_span_name("llm", self) # Get the parent context - turn context if available, otherwise service context turn_context = get_current_turn_context() @@ -538,7 +555,7 @@ async def wrapper(self, *args, **kwargs): return await func(self, *args, **kwargs) service_class_name = self.__class__.__name__ - span_name = f"{operation}" + span_name = _resolve_operation_span_name(operation, self) # Get the parent context - turn context if available, otherwise service context turn_context = get_current_turn_context() @@ -664,6 +681,7 @@ async def wrapper(self, *args, **kwargs): # Add information about the first function call call = function_calls[0] operation_attrs["tool.function_name"] = call.name + operation_attrs["function.name"] = call.name operation_attrs["tool.call_id"] = call.id operation_attrs["tool.calls_count"] = len(function_calls) @@ -679,8 +697,10 @@ async def wrapper(self, *args, **kwargs): if len(args_str) > 1000: args_str = args_str[:1000] + "..." operation_attrs["tool.arguments"] = args_str + operation_attrs["function.input"] = args_str except Exception: operation_attrs["tool.arguments"] = str(call.args)[:1000] + operation_attrs["function.input"] = str(call.args)[:1000] elif operation == "llm_tool_result" and args: # Extract tool result information @@ -695,6 +715,7 @@ async def wrapper(self, *args, **kwargs): operation_attrs["tool.call_id"] = tool_call_id if tool_call_name: operation_attrs["tool.function_name"] = tool_call_name + operation_attrs["function.name"] = tool_call_name # Parse and capture the result if result_content: @@ -705,6 +726,7 @@ async def wrapper(self, *args, **kwargs): if len(result_str) > 2000: # Larger limit for results result_str = result_str[:2000] + "..." operation_attrs["tool.result"] = result_str + operation_attrs["function.output"] = result_str # Add result status/success indicator if present if isinstance(result, dict): @@ -843,7 +865,7 @@ async def wrapper(self, *args, **kwargs): return await func(self, *args, **kwargs) service_class_name = self.__class__.__name__ - span_name = f"{operation}" + span_name = _resolve_operation_span_name(operation, self) # Get the parent context - turn context if available, otherwise service context turn_context = get_current_turn_context() diff --git a/src/efficientai/utils/tracing/setup.py b/src/efficientai/utils/tracing/setup.py index 350dd569..46340cc2 100644 --- a/src/efficientai/utils/tracing/setup.py +++ b/src/efficientai/utils/tracing/setup.py @@ -71,9 +71,12 @@ def setup_tracing( } ) - # Set up the tracer provider with the resource - tracer_provider = TracerProvider(resource=resource) - trace.set_tracer_provider(tracer_provider) + existing_provider = trace.get_tracer_provider() + if isinstance(existing_provider, TracerProvider): + tracer_provider = existing_provider + else: + tracer_provider = TracerProvider(resource=resource) + trace.set_tracer_provider(tracer_provider) # Add console exporter if requested (good for debugging) if console_export: diff --git a/src/efficientai/utils/tracing/turn_trace_observer.py b/src/efficientai/utils/tracing/turn_trace_observer.py index 12b1f279..1fc50c99 100644 --- a/src/efficientai/utils/tracing/turn_trace_observer.py +++ b/src/efficientai/utils/tracing/turn_trace_observer.py @@ -27,7 +27,7 @@ if is_tracing_available(): from opentelemetry import trace - from opentelemetry.trace import Span, SpanContext + from opentelemetry.trace import Span, SpanContext, format_trace_id class TurnTraceObserver(BaseObserver): @@ -66,6 +66,7 @@ def __init__( # Conversation tracking properties self._conversation_span: Optional["Span"] = None self._conversation_id = conversation_id + self._conversation_trace_id: Optional[str] = None self._additional_span_attributes = additional_span_attributes or {} if turn_tracker: @@ -108,6 +109,9 @@ def start_conversation_tracing(self, conversation_id: Optional[str] = None): # Create a new span for this conversation self._conversation_span = self._tracer.start_span("conversation") + self._conversation_trace_id = format_trace_id( + self._conversation_span.get_span_context().trace_id + ) # Set span attributes self._conversation_span.set_attribute("conversation.id", conversation_id) @@ -154,6 +158,10 @@ def end_conversation_tracing(self): logger.debug(f"Ended tracing for Conversation {self._conversation_id}") self._conversation_id = None + def get_conversation_trace_id(self) -> Optional[str]: + """Get the active conversation trace id when tracing is enabled.""" + return self._conversation_trace_id + async def _handle_turn_started(self, turn_number: int): """Handle a turn start event by creating a new span.""" if not is_tracing_available() or not self._tracer: diff --git a/tests/fixtures/elevenlabs/agents_list.json b/tests/fixtures/elevenlabs/agents_list.json new file mode 100644 index 00000000..7e7211b2 --- /dev/null +++ b/tests/fixtures/elevenlabs/agents_list.json @@ -0,0 +1,12 @@ +{ + "agents": [ + { + "agent_id": "agent_7101k5zvyjhmfg983brhmhkd98n6", + "name": "Customer Support Agent", + "archived": false, + "created_at_unix_secs": 1714423200 + } + ], + "has_more": false, + "next_cursor": null +} \ No newline at end of file diff --git a/tests/fixtures/elevenlabs/conv.json b/tests/fixtures/elevenlabs/conv.json new file mode 100644 index 00000000..7e7066d9 --- /dev/null +++ b/tests/fixtures/elevenlabs/conv.json @@ -0,0 +1,73 @@ +{ + "conversation_id": "conv_9001k1zph3fkeh5s8xg9z90swaqa", + "agent_id": "agent_7101k5zvyjhmfg983brhmhkd98n6", + "status": "done", + "metadata": { + "start_time_unix_secs": 1714423232, + "call_duration_secs": 25, + "cost_fiat": 0.12 + }, + "transcript": [ + { + "role": "user", + "time_in_call_secs": 2, + "message": "Hello there", + "conversation_turn_metrics": { + "elapsed_time": 0.22, + "metrics": { + "convai_turn_asr_latency": { + "elapsed_time": 0.22 + } + }, + "llm_usage": { + "model_usage": { + "gpt-4o-mini": { + "input": { + "tokens": 12 + }, + "output_total": { + "tokens": 8 + } + } + } + }, + "convai_asr_provider": "elevenlabs-asr" + } + }, + { + "role": "agent", + "time_in_call_secs": 4, + "message": "Hi, how can I help you today?", + "conversation_turn_metrics": { + "elapsed_time": 0.38, + "metrics": { + "convai_llm_service_ttfb": { + "elapsed_time": 0.18 + }, + "convai_tts_service_ttfb": { + "elapsed_time": 0.12 + } + }, + "convai_tts_model": "eleven_flash_v2_5" + } + }, + { + "role": "user", + "time_in_call_secs": 12, + "message": "Tell me a quick fun fact.", + "conversation_turn_metrics": { + "elapsed_time": 0.24 + } + }, + { + "role": "agent", + "time_in_call_secs": 15, + "message": "Honey never spoils, even after thousands of years.", + "conversation_turn_metrics": { + "elapsed_time": 0.41, + "convai_tts_model": "eleven_flash_v2_5" + } + } + ], + "otlp_traces": null +} \ No newline at end of file diff --git a/tests/fixtures/elevenlabs/conv_otel.json b/tests/fixtures/elevenlabs/conv_otel.json new file mode 100644 index 00000000..fa46b23a --- /dev/null +++ b/tests/fixtures/elevenlabs/conv_otel.json @@ -0,0 +1,112 @@ +{ + "conversation_id": "conv_9001k1zph3fkeh5s8xg9z90swaqa", + "agent_id": "agent_7101k5zvyjhmfg983brhmhkd98n6", + "status": "done", + "otlp_traces": { + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "elevenlabs-convai" + } + }, + { + "key": "elevenlabs.conversation_id", + "value": { + "stringValue": "conv_9001k1zph3fkeh5s8xg9z90swaqa" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "elevenlabs.convai", + "version": "1.0.0" + }, + "spans": [ + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "1111111111111111", + "name": "elevenlabs.conversation", + "startTimeUnixNano": "1714423232000000000", + "endTimeUnixNano": "1714423257000000000", + "attributes": [ + { + "key": "elevenlabs.source", + "value": { + "stringValue": "post_call_webhook" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "2222222222222222", + "parentSpanId": "1111111111111111", + "name": "elevenlabs.recv.user_transcript", + "startTimeUnixNano": "1714423234000000000", + "endTimeUnixNano": "1714423234500000000", + "attributes": [ + { + "key": "elevenlabs.user.text", + "value": { + "stringValue": "Hello there" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "3333333333333333", + "parentSpanId": "1111111111111111", + "name": "elevenlabs.recv.agent_response", + "startTimeUnixNano": "1714423235000000000", + "endTimeUnixNano": "1714423237000000000", + "attributes": [ + { + "key": "elevenlabs.agent.text", + "value": { + "stringValue": "Hi, how can I help you today?" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "4444444444444444", + "parentSpanId": "3333333333333333", + "name": "elevenlabs.tool.lookup_fact", + "startTimeUnixNano": "1714423235600000000", + "endTimeUnixNano": "1714423235900000000", + "attributes": [ + { + "key": "tool.name", + "value": { + "stringValue": "lookup_fact" + } + } + ], + "status": { + "code": 1 + } + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/elevenlabs/conversations_list.json b/tests/fixtures/elevenlabs/conversations_list.json new file mode 100644 index 00000000..a4e15ebf --- /dev/null +++ b/tests/fixtures/elevenlabs/conversations_list.json @@ -0,0 +1,18 @@ +{ + "conversations": [ + { + "conversation_id": "conv_12345", + "agent_id": "agent_abc", + "start_time_unix_secs": 1724034000, + "call_duration_secs": 123, + "message_count": 12, + "status": "done", + "call_successful": "success", + "metadata": { + "source": "js_sdk" + } + } + ], + "has_more": false, + "next_cursor": null +} diff --git a/tests/fixtures/elevenlabs/post_call_transcription_otel.json b/tests/fixtures/elevenlabs/post_call_transcription_otel.json new file mode 100644 index 00000000..42b8d5ba --- /dev/null +++ b/tests/fixtures/elevenlabs/post_call_transcription_otel.json @@ -0,0 +1,116 @@ +{ + "type": "post_call_transcription_otel", + "event_timestamp": 1714423258, + "data": { + "conversation_id": "conv_9001k1zph3fkeh5s8xg9z90swaqa", + "agent_id": "agent_7101k5zvyjhmfg983brhmhkd98n6", + "status": "done", + "otlp_traces": { + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "elevenlabs-convai" + } + }, + { + "key": "elevenlabs.conversation_id", + "value": { + "stringValue": "conv_9001k1zph3fkeh5s8xg9z90swaqa" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "elevenlabs.convai", + "version": "1.0.0" + }, + "spans": [ + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "1111111111111111", + "name": "elevenlabs.conversation", + "startTimeUnixNano": "1714423232000000000", + "endTimeUnixNano": "1714423257000000000", + "attributes": [ + { + "key": "elevenlabs.source", + "value": { + "stringValue": "post_call_webhook" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "2222222222222222", + "parentSpanId": "1111111111111111", + "name": "elevenlabs.recv.user_transcript", + "startTimeUnixNano": "1714423234000000000", + "endTimeUnixNano": "1714423234500000000", + "attributes": [ + { + "key": "elevenlabs.user.text", + "value": { + "stringValue": "Hello there" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "3333333333333333", + "parentSpanId": "1111111111111111", + "name": "elevenlabs.recv.agent_response", + "startTimeUnixNano": "1714423235000000000", + "endTimeUnixNano": "1714423237000000000", + "attributes": [ + { + "key": "elevenlabs.agent.text", + "value": { + "stringValue": "Hi, how can I help you today?" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "4444444444444444", + "parentSpanId": "3333333333333333", + "name": "elevenlabs.tool.lookup_fact", + "startTimeUnixNano": "1714423235600000000", + "endTimeUnixNano": "1714423235900000000", + "attributes": [ + { + "key": "tool.name", + "value": { + "stringValue": "lookup_fact" + } + } + ], + "status": { + "code": 1 + } + } + ] + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/test_api/conftest.py b/tests/test_api/conftest.py index a6490272..f6894a27 100644 --- a/tests/test_api/conftest.py +++ b/tests/test_api/conftest.py @@ -552,6 +552,7 @@ def _make_call_recording(**overrides): call_short_id=overrides.get("call_short_id", "123456"), status=overrides.get("status", CallRecordingStatus.PENDING), source=overrides.get("source", CallRecordingSource.WEBHOOK), + call_event=overrides.get("call_event"), call_data=overrides.get("call_data", {}), provider_call_id=overrides.get("provider_call_id"), provider_platform=overrides.get("provider_platform"), diff --git a/tests/test_api/test_integrations_routes.py b/tests/test_api/test_integrations_routes.py index e2b433bd..b11a9e6f 100644 --- a/tests/test_api/test_integrations_routes.py +++ b/tests/test_api/test_integrations_routes.py @@ -1,10 +1,14 @@ """API tests for integrations routes.""" import importlib +import json +from pathlib import Path from app.api.v1.routes import integrations as integrations_route +from app.models.database import ProviderSyncJob, Workspace prompt_sync_module = importlib.import_module("app.services.voice_providers.prompt_sync") +FIXTURE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "elevenlabs" def test_create_and_list_integrations(authenticated_client): @@ -124,3 +128,186 @@ def test_preview_integration_agent_prompt_empty(authenticated_client, monkeypatc ) assert response.status_code == 422 + + +def test_list_integration_external_agents_elevenlabs(authenticated_client, monkeypatch, make_integration): + integration = make_integration(platform="elevenlabs", api_key="encrypted") + fixture = json.loads((FIXTURE_DIR / "agents_list.json").read_text()) + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def list_agents(self, **_kwargs): + return { + "agents": [ + { + "id": fixture["agents"][0]["agent_id"], + "name": fixture["agents"][0]["name"], + "archived": fixture["agents"][0]["archived"], + "created_at": fixture["agents"][0]["created_at_unix_secs"], + "metadata": fixture["agents"][0], + } + ], + "has_more": fixture["has_more"], + "next_cursor": fixture["next_cursor"], + } + + monkeypatch.setattr(integrations_route, "decrypt_api_key", lambda _v: "decrypted") + monkeypatch.setattr(integrations_route, "get_voice_provider", lambda _p: _Provider) + + response = authenticated_client.get(f"/api/v1/integrations/{integration.id}/external-agents") + assert response.status_code == 200 + payload = response.json() + assert payload["agents"][0]["id"].startswith("agent_") + assert payload["agents"][0]["name"] == "Customer Support Agent" + + +def test_list_integration_external_agents_non_elevenlabs_rejected( + authenticated_client, make_integration +): + integration = make_integration(platform="murf", api_key="encrypted") + response = authenticated_client.get(f"/api/v1/integrations/{integration.id}/external-agents") + assert response.status_code == 400 + + +def test_list_integration_external_agents_vapi(authenticated_client, monkeypatch, make_integration): + integration = make_integration(platform="vapi", api_key="encrypted") + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def list_agents(self, **_kwargs): + return { + "agents": [ + { + "id": "assist_123", + "name": "Support Assistant", + "archived": False, + "created_at": "2026-01-01T00:00:00.000Z", + "metadata": {"id": "assist_123", "name": "Support Assistant"}, + } + ], + "has_more": False, + "next_cursor": None, + } + + monkeypatch.setattr(integrations_route, "decrypt_api_key", lambda _v: "decrypted") + monkeypatch.setattr(integrations_route, "get_voice_provider", lambda _p: _Provider) + + response = authenticated_client.get(f"/api/v1/integrations/{integration.id}/external-agents") + assert response.status_code == 200 + payload = response.json() + assert payload["agents"][0]["id"] == "assist_123" + assert payload["agents"][0]["name"] == "Support Assistant" + + +def test_list_integration_external_agents_retell(authenticated_client, monkeypatch, make_integration): + integration = make_integration(platform="retell", api_key="encrypted") + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def list_agents(self, **_kwargs): + return { + "agents": [ + { + "id": "agent_retell_1", + "name": "Retell Agent", + "archived": False, + "created_at": "2026-01-02T00:00:00.000Z", + "metadata": {"agent_id": "agent_retell_1", "agent_name": "Retell Agent"}, + } + ], + "has_more": False, + "next_cursor": None, + } + + monkeypatch.setattr(integrations_route, "decrypt_api_key", lambda _v: "decrypted") + monkeypatch.setattr(integrations_route, "get_voice_provider", lambda _p: _Provider) + + response = authenticated_client.get(f"/api/v1/integrations/{integration.id}/external-agents") + assert response.status_code == 200 + payload = response.json() + assert payload["agents"][0]["id"] == "agent_retell_1" + + +def test_start_elevenlabs_agent_sync_creates_job(authenticated_client, monkeypatch, make_integration): + integration = make_integration(platform="elevenlabs", api_key="encrypted") + scheduled = {"job_id": None} + + class _Task: + @staticmethod + def delay(job_id): + scheduled["job_id"] = job_id + return None + + monkeypatch.setattr(integrations_route, "sync_elevenlabs_agents_task", _Task()) + response = authenticated_client.post( + f"/api/v1/integrations/{integration.id}/sync/elevenlabs/agents" + ) + assert response.status_code == 200 + body = response.json() + assert body["phase"] == "agents" + assert body["provider_platform"] == "elevenlabs" + assert scheduled["job_id"] == body["id"] + + +def test_start_elevenlabs_conversation_sync_creates_job(authenticated_client, monkeypatch, make_integration): + integration = make_integration(platform="elevenlabs", api_key="encrypted") + called = {"count": 0} + + class _Chain: + def delay(self): + called["count"] += 1 + return None + + monkeypatch.setattr(integrations_route, "chain", lambda *_args, **_kwargs: _Chain()) + response = authenticated_client.post( + f"/api/v1/integrations/{integration.id}/sync/elevenlabs/conversations", + json={"insights_only": True, "agent_ids": ["agent_abc"]}, + ) + assert response.status_code == 200 + body = response.json() + assert body["phase"] == "catalog" + assert body["config"]["agent_ids"] == ["agent_abc"] + assert called["count"] == 1 + + +def test_get_and_cancel_provider_sync_job(authenticated_client, db_session, make_integration): + integration = make_integration(platform="elevenlabs", api_key="encrypted") + org_id = integration.organization_id + workspace = db_session.query(Workspace).filter( + Workspace.organization_id == org_id, + Workspace.is_default.is_(True), + ).first() + assert workspace is not None + ws_id = workspace.id + + job = ProviderSyncJob( + organization_id=org_id, + workspace_id=ws_id, + integration_id=integration.id, + provider_platform="elevenlabs", + status="running", + phase="catalog", + config={"since_unix": 0}, + cursor_state={}, + ) + db_session.add(job) + db_session.commit() + db_session.refresh(job) + + get_response = authenticated_client.get( + f"/api/v1/integrations/{integration.id}/sync/jobs/{job.id}" + ) + assert get_response.status_code == 200 + assert get_response.json()["id"] == str(job.id) + + cancel_response = authenticated_client.post( + f"/api/v1/integrations/{integration.id}/sync/jobs/{job.id}/cancel" + ) + assert cancel_response.status_code == 200 + assert cancel_response.json()["status"] == "cancelled" diff --git a/tests/test_api/test_observability_routes.py b/tests/test_api/test_observability_routes.py index 2586f22c..c29b0e5d 100644 --- a/tests/test_api/test_observability_routes.py +++ b/tests/test_api/test_observability_routes.py @@ -1,5 +1,25 @@ """API tests for observability routes.""" +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +from types import SimpleNamespace +import wave + +from app.api.v1.routes import observability + +FIXTURE_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "elevenlabs" + + +def _write_wav(path: Path, samples: np.ndarray, sample_rate: int = 16000) -> None: + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(samples.astype(np.int16).tobytes()) + def test_list_get_delete_observability_calls(authenticated_client, make_call_recording): call_recording = make_call_recording( @@ -21,3 +41,804 @@ def test_list_get_delete_observability_calls(authenticated_client, make_call_rec ) assert delete_response.status_code == 200 assert delete_response.json()["message"] == "Call deleted" + + +def test_calls_summary_returns_aggregates(authenticated_client, make_call_recording): + make_call_recording( + call_short_id="111111", + source="webhook", + call_data={"duration_seconds": 60, "messages": [{"role": "user", "content": "hello"}]}, + ) + make_call_recording( + call_short_id="222222", + source="webhook", + call_data={"duration_seconds": 30, "messages": [{"role": "user", "content": "hi"}]}, + ) + + response = authenticated_client.get("/api/v1/observability/calls/summary") + assert response.status_code == 200 + payload = response.json() + assert payload["total_calls"] == 2 + assert payload["total_minutes"] == 1.5 + assert payload["avg_latency_ms"] == 45000.0 + assert payload["avg_duration_ms"] == 45000.0 + assert payload["trace_available_calls"] == 0 + + +def test_get_call_trace_returns_normalized_payload( + authenticated_client, make_call_recording, monkeypatch +): + call_recording = make_call_recording( + call_short_id="333333", + source="webhook", + call_data={ + "messages": [{"role": "user", "content": "hello"}], + "trace_id": "0af7651916cd43dd8448eb211c80319c", + }, + ) + + async def _mock_query_trace_cloud(trace_id, api_key): + del api_key + return { + "trace_id": trace_id, + "root_span_id": "root-1", + "spans": [ + { + "span_id": "root-1", + "parent_span_id": None, + "name": "conversation", + "start_time": 1000.0, + "end_time": 2000.0, + "duration_ms": 1000.0, + "attributes": {"conversation.id": "abc"}, + "status": "ok", + } + ], + } + + monkeypatch.setattr( + observability, + "_query_trace_cloud", + _mock_query_trace_cloud, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_id"] == "0af7651916cd43dd8448eb211c80319c" + assert payload["root_span_id"] == "root-1" + assert payload["spans"][0]["name"] == "conversation" + + +def test_trace_route_returns_404_when_no_trace_id(authenticated_client, make_call_recording): + call_recording = make_call_recording( + call_short_id="444444", + source="webhook", + call_data={"messages": [{"role": "user", "content": "hello"}]}, + ) + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 404 + assert response.json()["detail"] == "No trace linked to this call" + + +def test_get_call_trace_returns_elevenlabs_trace_without_trace_id( + authenticated_client, make_call_recording, monkeypatch +): + call_recording = make_call_recording( + call_short_id="el1111", + source="webhook", + provider_platform="elevenlabs", + provider_call_id="conv_9001k1zph3fkeh5s8xg9z90swaqa", + call_data={"messages": [{"role": "user", "content": "hello"}]}, + ) + fixture = json.loads((FIXTURE_DIR / "conv_otel.json").read_text()) + + async def _mock_query_elevenlabs_trace_for_call(**kwargs): + del kwargs + return { + "trace_id": "0af7651916cd43dd8448eb211c80319c", + "root_span_id": "1111111111111111", + "trace_source": "elevenlabs", + "spans": fixture["otlp_traces"]["resourceSpans"][0]["scopeSpans"][0]["spans"], + } + + monkeypatch.setattr( + observability, + "_query_elevenlabs_trace_for_call", + _mock_query_elevenlabs_trace_for_call, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_source"] == "elevenlabs" + assert payload["trace_id"] == "0af7651916cd43dd8448eb211c80319c" + + +def test_get_call_trace_returns_vapi_synthetic_trace_without_trace_id( + authenticated_client, make_call_recording +): + call_recording = make_call_recording( + call_short_id="vapi44", + source="webhook", + provider_platform="vapi", + provider_call_id="call_vapi_123", + call_data={ + "id": "call_vapi_123", + "status": "ended", + "startedAt": "2026-08-07T09:00:00.000Z", + "endedAt": "2026-08-07T09:00:10.000Z", + "messages": [ + {"role": "user", "message": "hello", "secondsFromStart": 0.5, "duration": 900}, + {"role": "assistant", "message": "hi there", "secondsFromStart": 1.6, "duration": 1200}, + ], + "artifact": { + "performanceMetrics": { + "modelLatencyAverage": 320, + "voiceLatencyAverage": 480, + "transcriberLatencyAverage": 210, + "endpointingLatencyAverage": 140, + "turnLatencyAverage": 2200, + } + }, + }, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_source"] == "vapi_synthetic" + assert payload["trace_id"] == "vapi-call_vapi_123" + assert any(span["name"] == "llm" for span in payload["spans"]) + + detail = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}" + ) + assert detail.status_code == 200 + provider_trace = detail.json()["call_data"].get("provider_trace") + assert isinstance(provider_trace, dict) + assert provider_trace.get("trace_source") == "vapi_synthetic" + + +def test_get_call_trace_returns_retell_synthetic_trace_without_trace_id( + authenticated_client, make_call_recording +): + call_recording = make_call_recording( + call_short_id="ret444", + source="webhook", + provider_platform="retell", + provider_call_id="call_retell_123", + call_data={ + "call_id": "call_retell_123", + "call_status": "ended", + "start_timestamp": 1_714_423_232_000, + "end_timestamp": 1_714_423_257_000, + "transcript_object": [ + {"role": "user", "content": "hello", "words": [{"word": "hello", "start": 0.4, "end": 0.9}]}, + {"role": "agent", "content": "hi there", "words": [{"word": "hi", "start": 1.2, "end": 2.0}]}, + ], + "latency": { + "asr": {"p50": 180}, + "llm": {"p50": 420}, + "tts": {"p50": 260}, + }, + }, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_source"] == "retell_synthetic" + assert payload["trace_id"] == "retell-call_retell_123" + assert any(span["name"] == "stt" for span in payload["spans"]) + + detail = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}" + ) + assert detail.status_code == 200 + provider_trace = detail.json()["call_data"].get("provider_trace") + assert isinstance(provider_trace, dict) + assert provider_trace.get("trace_source") == "retell_synthetic" + + +def test_get_call_trace_prefers_stored_provider_trace(authenticated_client, make_call_recording): + call_recording = make_call_recording( + call_short_id="stort1", + source="webhook", + provider_platform="retell", + provider_call_id="call_retell_999", + call_data={ + "provider_trace": { + "trace_source": "retell_synthetic", + "normalized_trace": { + "trace_id": "retell-call_retell_999", + "root_span_id": "root", + "spans": [ + { + "span_id": "root", + "parent_span_id": None, + "name": "conversation", + "start_time": 1000.0, + "end_time": 2000.0, + "duration_ms": 1000.0, + "attributes": {"trace.provider": "retell"}, + "status": "1", + } + ], + "trace_source": "retell_synthetic", + }, + } + }, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_id"] == "retell-call_retell_999" + assert payload["trace_source"] == "retell_synthetic" + + +def test_get_call_trace_returns_404_when_trace_missing_in_store( + authenticated_client, make_call_recording, monkeypatch +): + import httpx + + call_recording = make_call_recording( + call_short_id="555555", + source="playground", + call_data={ + "messages": [{"role": "user", "content": "hello"}], + "trace_id": "0af7651916cd43dd8448eb211c80319c", + }, + ) + + async def _mock_query_trace_tempo(trace_id): + del trace_id + request = httpx.Request("GET", "http://tempo/api/traces/test") + response = httpx.Response(404, request=request) + raise httpx.HTTPStatusError("not found", request=request, response=response) + + monkeypatch.setattr(observability.settings, "TRACING_QUERY_BACKEND", "tempo") + monkeypatch.setattr(observability, "_query_trace_tempo", _mock_query_trace_tempo) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"].lower() + + +def test_live_audio_endpoint_serves_partial_wav( + authenticated_client, make_call_recording, tmp_path +): + user_path = tmp_path / "user.wav" + bot_path = tmp_path / "bot.wav" + samples = np.array([2000, -2000, 2000, -2000], dtype=np.int16) + _write_wav(user_path, samples) + _write_wav(bot_path, samples) + + call_recording = make_call_recording( + call_short_id="777777", + source="webhook", + call_event="call_in_progress", + call_data={ + "live_user_audio_path": str(user_path), + "live_bot_audio_path": str(bot_path), + "live_transcript": [], + }, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/live-audio" + ) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("audio/wav") + assert float(response.headers["x-audio-duration-sec"]) >= 0 + assert response.content.startswith(b"RIFF") + + +def test_observability_audio_uses_elevenlabs_proxy( + authenticated_client, make_integration, make_agent, make_call_recording, monkeypatch +): + from fastapi.responses import Response + + integration = make_integration(platform="elevenlabs", api_key="encrypted-api-key") + agent = make_agent( + integration=integration, + voice_ai_integration_id=integration.id, + voice_ai_agent_id="agent_abc", + ) + call_recording = make_call_recording( + call_short_id="obsaud", + source="webhook", + agent_id=agent.id, + provider_platform="elevenlabs", + provider_call_id="conv_123", + call_data={"recording_urls": {"conversation_audio": "https://api.elevenlabs.io/v1/convai/conversations/conv_123/audio"}}, + ) + + def _proxy(**kwargs): + assert kwargs["call_recording"].call_short_id == "obsaud" + return Response(content=b"mp3-bytes", media_type="audio/mpeg") + + monkeypatch.setattr( + "app.services.observability.provider_audio_proxy.stream_elevenlabs_audio_proxy", + _proxy, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/audio" + ) + assert response.status_code == 200 + assert response.content == b"mp3-bytes" + assert response.headers["content-type"].startswith("audio/mpeg") + + +def test_webhook_ingest_persists_trace_id(authenticated_client, api_key): + payload = { + "id": "provider-call-999", + "provider_platform": "external", + "startedAt": "2026-08-07T09:00:00.000Z", + "endedAt": "2026-08-07T09:01:30.000Z", + "trace_id": "0af7651916cd43dd8448eb211c80319c", + "messages": [{"role": "user", "content": "hello"}], + } + response = authenticated_client.post( + f"/api/v1/observability/calls/webhook/{api_key}", + json=payload, + ) + assert response.status_code == 201 + body = response.json() + assert body["trace_id"] == "0af7651916cd43dd8448eb211c80319c" + + +def test_ingest_elevenlabs_otel_webhook(authenticated_client, api_key): + payload = json.loads((FIXTURE_DIR / "post_call_transcription_otel.json").read_text()) + response = authenticated_client.post( + f"/api/v1/observability/calls/webhook/elevenlabs/{api_key}", + json=payload, + ) + assert response.status_code == 201 + body = response.json() + assert body["provider_platform"] == "elevenlabs" + assert body["provider_call_id"] == payload["data"]["conversation_id"] + assert body["call_data"]["provider_trace"]["source"] == "elevenlabs_post_call_webhook" + + +def test_observe_endpoint_persists_trace_id(authenticated_client): + payload = { + "id": "observe-call-999", + "provider_platform": "retell", + "startedAt": "2026-08-07T09:00:00.000Z", + "endedAt": "2026-08-07T09:01:30.000Z", + "trace_id": "0af7651916cd43dd8448eb211c80319c", + "messages": [{"role": "assistant", "content": "hello"}], + } + response = authenticated_client.post("/api/v1/observability/observe", json=payload) + assert response.status_code == 201 + body = response.json() + assert body["provider_call_id"] == "observe-call-999" + assert body["trace_id"] == "0af7651916cd43dd8448eb211c80319c" + + +def test_refresh_observability_call_pulls_provider_metrics( + authenticated_client, make_integration, make_agent, make_call_recording, monkeypatch +): + integration = make_integration(platform="vapi", api_key="encrypted-api-key") + agent = make_agent(integration=integration, voice_ai_integration_id=integration.id) + call_recording = make_call_recording( + call_short_id="obsrf1", + source="webhook", + agent_id=agent.id, + provider_platform="vapi", + provider_call_id="call_123", + call_data={"status": "queued"}, + ) + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def retrieve_call_metrics(self, call_id): + assert call_id == "call_123" + return {"id": "call_123", "status": "ended", "messages": [{"role": "assistant", "message": "done"}]} + + monkeypatch.setattr(observability, "decrypt_api_key", lambda _v: "decrypted") + monkeypatch.setattr(observability, "get_voice_provider", lambda _p: _Provider) + + response = authenticated_client.post(f"/api/v1/observability/calls/{call_recording.call_short_id}/refresh") + assert response.status_code == 200 + payload = response.json() + assert payload["call_data"]["status"] == "ended" + assert payload["provider_platform"] == "vapi" + + +def test_refresh_observability_call_requires_provider_info( + authenticated_client, make_call_recording +): + call_recording = make_call_recording( + call_short_id="obsrf2", + source="webhook", + call_data={"status": "queued"}, + provider_platform=None, + provider_call_id=None, + ) + + response = authenticated_client.post(f"/api/v1/observability/calls/{call_recording.call_short_id}/refresh") + assert response.status_code == 400 + assert "provider information" in response.json()["detail"].lower() + + +def test_vapi_webhook_terminal_event_triggers_refresh_fallback( + authenticated_client, api_key, make_integration, make_agent, monkeypatch +): + integration = make_integration(platform="vapi", api_key="encrypted-api-key") + agent = make_agent( + integration=integration, + voice_ai_integration_id=integration.id, + voice_ai_agent_id="assist_123", + ) + + monkeypatch.setattr(observability, "decrypt_api_key", lambda _v: "decrypted") + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def retrieve_call_metrics(self, call_id): + assert call_id == "call_vapi_terminal" + return { + "id": call_id, + "status": "ended", + "analysis": {"summary": "Complete"}, + "costBreakdown": {"transport": 0.001}, + "artifact": {"messages": [{"role": "assistant", "message": "done"}]}, + } + + monkeypatch.setattr(observability, "get_voice_provider", lambda _p: _Provider) + + payload = { + "id": "call_vapi_terminal", + "agent_id": "assist_123", + "status": "ended", + "messages": [], + } + response = authenticated_client.post( + f"/api/v1/observability/calls/webhook/vapi/{api_key}", + json=payload, + ) + assert response.status_code == 201 + body = response.json() + assert body["call_event"] == "call_ended" + assert body["call_data"]["status"] == "ended" + assert body["call_data"]["analysis"]["summary"] == "Complete" + + +def test_retell_webhook_terminal_event_triggers_refresh_fallback( + authenticated_client, api_key, make_integration, make_agent, monkeypatch +): + integration = make_integration(platform="retell", api_key="encrypted-api-key") + agent = make_agent( + integration=integration, + voice_ai_integration_id=integration.id, + voice_ai_agent_id="agent_retell_123", + ) + + monkeypatch.setattr(observability, "decrypt_api_key", lambda _v: "decrypted") + + class _Provider: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def retrieve_call_metrics(self, call_id): + assert call_id == "call_retell_terminal" + return { + "call_id": call_id, + "call_status": "ended", + "call_analysis": {"call_summary": "Resolved"}, + "call_cost": {"combined_cost": 0.0123}, + "transcript_object": [{"role": "agent", "content": "done"}], + } + + monkeypatch.setattr(observability, "get_voice_provider", lambda _p: _Provider) + + payload = { + "event": "call_analyzed", + "call": { + "call_id": "call_retell_terminal", + "agent_id": "agent_retell_123", + "call_status": "ended", + }, + } + response = authenticated_client.post( + f"/api/v1/observability/calls/webhook/retell/{api_key}", + json=payload, + ) + assert response.status_code == 201 + body = response.json() + assert body["call_event"] == "call_ended" + assert body["call_data"]["call_status"] == "ended" + assert body["call_data"]["call_analysis"]["call_summary"] == "Resolved" + + +def test_webhook_call_ended_auto_queues_evaluation( + authenticated_client, + api_key, + make_agent, + make_evaluator, + db_session, + monkeypatch, +): + agent = make_agent() + evaluator = make_evaluator( + agent_id=agent.id, + workspace_id=agent.workspace_id, + ) + agent.observability_auto_evaluator_id = evaluator.id + db_session.commit() + + monkeypatch.setattr( + observability.process_evaluator_result_task, + "delay", + lambda _result_id: SimpleNamespace(id="task-123"), + ) + + payload = { + "id": "provider-call-auto", + "provider_platform": "external", + "agent_id": str(agent.id), + "startedAt": "2026-08-07T09:00:00.000Z", + "endedAt": "2026-08-07T09:01:30.000Z", + "messages": [{"role": "user", "content": "hello"}], + } + ingest_response = authenticated_client.post( + f"/api/v1/observability/calls/webhook/{api_key}", + json=payload, + ) + assert ingest_response.status_code == 201 + call_short_id = ingest_response.json()["call_short_id"] + + detail_response = authenticated_client.get(f"/api/v1/observability/calls/{call_short_id}") + assert detail_response.status_code == 200 + assert detail_response.json()["evaluator_result_id"] is not None + + +def test_live_event_ingest_is_idempotent_and_issues_trace_id(authenticated_client, monkeypatch): + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_INGEST_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS", 999999999) + now_iso = datetime.now(UTC).isoformat().replace("+00:00", "Z") + payload = { + "event_id": "evt_live_001", + "call_id": "lk_call_001", + "event_type": "call.started", + "seq": 1, + "event_ts": now_iso, + "platform": "livekit", + "payload": {"startedAt": now_iso}, + } + + first = authenticated_client.post("/api/v1/observability/live/events", json=payload) + assert first.status_code == 202 + first_body = first.json() + assert first_body["accepted"] is True + assert first_body["duplicate"] is False + assert isinstance(first_body["trace_id"], str) + assert len(first_body["trace_id"]) == 32 + + second = authenticated_client.post("/api/v1/observability/live/events", json=payload) + assert second.status_code == 202 + second_body = second.json() + assert second_body["accepted"] is True + assert second_body["duplicate"] is True + assert second_body["call_short_id"] == first_body["call_short_id"] + + +def test_live_event_ingest_rejects_stale_sequence(authenticated_client, monkeypatch): + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_INGEST_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_OUT_OF_ORDER_SEQ", 2) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS", 999999999) + base_ts = datetime.now(UTC) + first = { + "event_id": "evt_live_100", + "call_id": "pc_call_100", + "event_type": "turn.user", + "seq": 10, + "event_ts": base_ts.isoformat().replace("+00:00", "Z"), + "platform": "pipecat", + "payload": {"content": "hello"}, + } + stale = { + "event_id": "evt_live_101", + "call_id": "pc_call_100", + "event_type": "turn.assistant", + "seq": 2, + "event_ts": (base_ts + timedelta(seconds=1)).isoformat().replace("+00:00", "Z"), + "platform": "pipecat", + "payload": {"content": "hi"}, + } + first_resp = authenticated_client.post("/api/v1/observability/live/events", json=first) + assert first_resp.status_code == 202 + stale_resp = authenticated_client.post("/api/v1/observability/live/events", json=stale) + assert stale_resp.status_code == 409 + + +def test_live_latency_metrics_endpoints(authenticated_client, monkeypatch, make_agent): + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_INGEST_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_AGGREGATES_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS", 999999999) + agent = make_agent() + + base_ts = datetime.now(UTC) + events = [ + { + "event_id": "evt_lat_1", + "call_id": "lk_latency_call", + "event_type": "turn.user", + "seq": 1, + "event_ts": base_ts.isoformat().replace("+00:00", "Z"), + "platform": "livekit", + "agent_ref": str(agent.id), + "payload": {"content": "hello", "latency": {"llm": 200, "tts": 150}}, + }, + { + "event_id": "evt_lat_2", + "call_id": "lk_latency_call", + "event_type": "turn.assistant", + "seq": 2, + "event_ts": (base_ts + timedelta(seconds=1)).isoformat().replace("+00:00", "Z"), + "platform": "livekit", + "agent_ref": str(agent.id), + "payload": {"content": "hi", "latency": {"llm": 500, "tts": 320}}, + }, + ] + for item in events: + resp = authenticated_client.post("/api/v1/observability/live/events", json=item) + assert resp.status_code == 202 + + metrics_resp = authenticated_client.get("/api/v1/observability/live/metrics/latency") + assert metrics_resp.status_code == 200 + metrics_body = metrics_resp.json() + assert metrics_body["windows"]["300s"]["sample_count"] >= 2 + assert "llm_ms" in metrics_body["windows"]["300s"]["metrics"] + + agent_metrics = authenticated_client.get( + f"/api/v1/observability/live/agents/{agent.id}/latency" + ) + assert agent_metrics.status_code == 200 + agent_body = agent_metrics.json() + assert agent_body["scope"] == "agent" + assert agent_body["agent_id"] == str(agent.id) + + +def test_live_event_ingest_records_slo_breach(authenticated_client, monkeypatch): + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_INGEST_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_AGGREGATES_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_SLO_ALERTS_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_SLO_MIN_SAMPLE_COUNT", 1) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_SLO_P90_LLM_MS", 100) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS", 999999999) + + now_iso = datetime.now(UTC).isoformat().replace("+00:00", "Z") + payload = { + "event_id": "evt_slo_1", + "call_id": "live_call_slo_1", + "event_type": "turn.assistant", + "seq": 1, + "event_ts": now_iso, + "platform": "livekit", + "payload": {"content": "hello", "latency": {"llm": 450}}, + } + response = authenticated_client.post("/api/v1/observability/live/events", json=payload) + assert response.status_code == 202 + assert response.json()["slo_breach_detected"] is True + + +def test_get_call_trace_returns_pipecat_live_synthetic_trace( + authenticated_client, make_call_recording +): + trace_id = "60e7082206844a85bbd9eaa13888c5ed" + call_recording = make_call_recording( + call_short_id="pipe77", + source="webhook", + provider_platform="pipecat", + provider_call_id="pipecat-live-1786946667", + trace_id=trace_id, + call_data={ + "trace_id": trace_id, + "startedAt": "2026-08-17T06:04:27.000Z", + "endedAt": "2026-08-17T06:04:48.000Z", + "status": "ended", + "live_transcript": [ + { + "role": "user", + "content": "Hello, can you hear me?", + "event_ts": "2026-08-17T06:04:30.641055+00:00", + }, + { + "role": "assistant", + "content": "Yes, I can hear you clearly.", + "event_ts": "2026-08-17T06:04:33.708357+00:00", + "latency": {"llm_ms": 380, "tts_ms": 210}, + }, + ], + }, + ) + + response = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}/trace" + ) + assert response.status_code == 200 + payload = response.json() + assert payload["trace_source"] == "pipecat_live_synthetic" + assert payload["trace_id"] == trace_id + assert any(span["name"] == "llm" for span in payload["spans"]) + assert any(span["name"] == "tts" for span in payload["spans"]) + + detail = authenticated_client.get( + f"/api/v1/observability/calls/{call_recording.call_short_id}" + ) + assert detail.status_code == 200 + provider_trace = detail.json()["call_data"].get("provider_trace") + assert isinstance(provider_trace, dict) + assert provider_trace.get("trace_source") == "pipecat_live_synthetic" + + +def test_live_event_ingest_persists_synthetic_trace_on_call_end(authenticated_client, monkeypatch): + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_INGEST_ENABLED", True) + monkeypatch.setattr(observability.settings, "OBSERVABILITY_LIVE_EVENT_MAX_TS_DRIFT_SECONDS", 999999999) + call_id = "pipecat-live-end-1" + base_ts = datetime.now(UTC) + + events = [ + { + "event_id": "evt_live_end_1", + "call_id": call_id, + "event_type": "call.started", + "seq": 1, + "event_ts": base_ts.isoformat().replace("+00:00", "Z"), + "platform": "pipecat", + "payload": {"startedAt": base_ts.isoformat().replace("+00:00", "Z")}, + }, + { + "event_id": "evt_live_end_2", + "call_id": call_id, + "event_type": "turn.assistant", + "seq": 2, + "event_ts": (base_ts + timedelta(seconds=1)).isoformat().replace("+00:00", "Z"), + "platform": "pipecat", + "payload": {"content": "hello", "latency": {"llm_ms": 300, "tts_ms": 150}}, + }, + { + "event_id": "evt_live_end_3", + "call_id": call_id, + "event_type": "call.ended", + "seq": 3, + "event_ts": (base_ts + timedelta(seconds=2)).isoformat().replace("+00:00", "Z"), + "platform": "pipecat", + "payload": {"endedAt": (base_ts + timedelta(seconds=2)).isoformat().replace("+00:00", "Z")}, + }, + ] + + call_short_id = None + for item in events: + resp = authenticated_client.post("/api/v1/observability/live/events", json=item) + assert resp.status_code == 202 + call_short_id = resp.json()["call_short_id"] + + trace_resp = authenticated_client.get(f"/api/v1/observability/calls/{call_short_id}/trace") + assert trace_resp.status_code == 200 + trace_payload = trace_resp.json() + assert trace_payload["trace_source"] == "pipecat_live_synthetic" + assert any(span["name"] == "llm" for span in trace_payload["spans"]) diff --git a/tests/test_services/test_ai/test_llm_service.py b/tests/test_services/test_ai/test_llm_service.py index f9dcb666..3ab3da93 100644 --- a/tests/test_services/test_ai/test_llm_service.py +++ b/tests/test_services/test_ai/test_llm_service.py @@ -343,3 +343,72 @@ def _fake_completion(**kwargs): assert captured["model"] == "openai/gpt-5-mini" assert captured["api_base"] == "https://eaitest-resource.openai.azure.com/openai/v1" assert "azure_endpoint" not in captured + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("openai/gpt-5.6", True), + ("openai/gpt-5.6-sol", True), + ("openai/gpt-5-mini", True), + ("openai/gpt-5-chat-latest", False), + ("openai/gpt-4o-mini", False), + ("openai/o3-mini", True), + ], +) +def test_model_only_supports_default_temperature(model, expected): + assert llm_module._model_only_supports_default_temperature(model) is expected + + +def test_normalize_temperature_for_model_drops_non_default(): + call_kwargs = {"temperature": 0.7, "model": "openai/gpt-5.6"} + llm_module._normalize_temperature_for_model("openai/gpt-5.6", call_kwargs) + assert "temperature" not in call_kwargs + + +def test_normalize_temperature_for_model_keeps_default_and_other_models(): + gpt4_kwargs = {"temperature": 0.7} + llm_module._normalize_temperature_for_model("openai/gpt-4o-mini", gpt4_kwargs) + assert gpt4_kwargs["temperature"] == 0.7 + + gpt5_default_kwargs = {"temperature": 1} + llm_module._normalize_temperature_for_model("openai/gpt-5.6", gpt5_default_kwargs) + assert gpt5_default_kwargs["temperature"] == 1 + + +def test_generate_response_omits_temperature_for_gpt_5_6(monkeypatch): + service = LLMService() + provider = SimpleNamespace(api_key="encrypted-key") + monkeypatch.setattr(service, "_get_ai_provider", lambda *_args, **_kwargs: provider) + + encryption_module = importlib.import_module("app.core.encryption") + monkeypatch.setattr(encryption_module, "decrypt_api_key", lambda value: "openai-api-key") + + captured = {} + + def _fake_completion(**kwargs): + captured.update(kwargs) + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content="ok"), + finish_reason="stop", + ) + ], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + monkeypatch.setattr(llm_module.litellm, "completion", _fake_completion) + + service.generate_response( + messages=[{"role": "user", "content": "hello"}], + llm_provider=ModelProvider.OPENAI, + llm_model="gpt-5.6", + organization_id=uuid4(), + db=_mock_org_db(), + temperature=0.7, + task_defaults={"temperature": 0.7}, + ) + + assert captured["model"] == "openai/gpt-5.6" + assert "temperature" not in captured diff --git a/tests/test_services/test_efficientai_otel.py b/tests/test_services/test_efficientai_otel.py new file mode 100644 index 00000000..069076e4 --- /dev/null +++ b/tests/test_services/test_efficientai_otel.py @@ -0,0 +1,71 @@ +"""Unit tests for EfficientAI tracing bootstrap.""" + +from types import SimpleNamespace + +import pytest + +from app.services.tracing import efficientai_otel as otel + + +@pytest.fixture(autouse=True) +def _reset_initialized_flag(): + otel._INITIALIZED = False + yield + otel._INITIALIZED = False + + +def test_provider_uses_zero_sample_rate(monkeypatch): + created = {} + + class FakeTracerProvider: + def __init__(self, resource=None, sampler=None): + self.resource = resource + self.sampler = sampler + + monkeypatch.setattr(otel, "OTEL_AVAILABLE", True) + monkeypatch.setattr(otel, "TracerProvider", FakeTracerProvider) + monkeypatch.setattr(otel, "Resource", SimpleNamespace(create=lambda attrs: attrs)) + monkeypatch.setattr(otel, "ParentBased", lambda root: ("parent", root)) + monkeypatch.setattr(otel, "TraceIdRatioBased", lambda rate: ("ratio", rate)) + monkeypatch.setattr( + otel, + "trace", + SimpleNamespace( + get_tracer_provider=lambda: object(), + set_tracer_provider=lambda provider: created.setdefault("provider", provider), + ), + ) + monkeypatch.setattr(otel.settings, "OBSERVABILITY_TRACING_SAMPLE_RATE", 0.0) + + provider = otel._get_or_create_provider("efficientai-test") + + assert isinstance(provider, FakeTracerProvider) + assert provider.sampler == ("parent", ("ratio", 0.0)) + assert created["provider"] is provider + + +def test_provider_uses_full_sample_rate(monkeypatch): + class FakeTracerProvider: + def __init__(self, resource=None, sampler=None): + self.resource = resource + self.sampler = sampler + + monkeypatch.setattr(otel, "OTEL_AVAILABLE", True) + monkeypatch.setattr(otel, "TracerProvider", FakeTracerProvider) + monkeypatch.setattr(otel, "Resource", SimpleNamespace(create=lambda attrs: attrs)) + monkeypatch.setattr(otel, "ParentBased", lambda root: ("parent", root)) + monkeypatch.setattr(otel, "TraceIdRatioBased", lambda rate: ("ratio", rate)) + monkeypatch.setattr( + otel, + "trace", + SimpleNamespace( + get_tracer_provider=lambda: object(), + set_tracer_provider=lambda provider: None, + ), + ) + monkeypatch.setattr(otel.settings, "OBSERVABILITY_TRACING_SAMPLE_RATE", 1.0) + + provider = otel._get_or_create_provider("efficientai-test") + + assert isinstance(provider, FakeTracerProvider) + assert provider.sampler == ("parent", ("ratio", 1.0)) diff --git a/tests/test_services/test_observability/test_call_ingest.py b/tests/test_services/test_observability/test_call_ingest.py new file mode 100644 index 00000000..52053fe8 --- /dev/null +++ b/tests/test_services/test_observability/test_call_ingest.py @@ -0,0 +1,97 @@ +"""Tests for observability call ingest helpers.""" + +from uuid import uuid4 + +from app.models.database import Agent, CallRecordingSource +from app.services.observability.call_ingest import persist_playground_voice_call + + +def _make_test_agent(db_session, org_id, default_workspace): + agent = Agent( + id=uuid4(), + agent_id="123456", + organization_id=org_id, + workspace_id=default_workspace.id, + name="Observability Test Agent", + phone_number="+1234567890", + language="en", + description="Agent description", + call_type="outbound", + call_medium="phone_call", + ) + db_session.add(agent) + db_session.commit() + db_session.refresh(agent) + return agent + + +def test_persist_playground_voice_call_merges_live_transcript(db_session, org_id, default_workspace): + agent = _make_test_agent(db_session, org_id, default_workspace) + result_id = "882211" + existing = persist_playground_voice_call( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + agent_id=agent.id, + result_id=result_id, + call_metadata={"duration": 12.5}, + provider_platform="efficientai", + ) + assert existing is not None + existing.call_data = { + **(existing.call_data or {}), + "startedAt": "2026-08-07T09:18:00.000Z", + "live_transcript": [ + {"role": "user", "content": "Hello there"}, + {"role": "agent", "content": "Hi, how can I help?"}, + ], + "messages": [ + {"role": "user", "content": "Hello there"}, + {"role": "bot", "content": "Hi, how can I help?"}, + ], + } + db_session.commit() + + updated = persist_playground_voice_call( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + agent_id=agent.id, + result_id=result_id, + call_metadata={ + "duration": 60.0, + "trace_id": "abc123trace", + "speaker_segments": [ + {"speaker": "user", "text": "Hello there", "start": 0.0, "end": 1.0}, + ], + }, + provider_platform="efficientai", + ) + + assert updated is not None + assert updated.agent_id == agent.id + assert updated.trace_id == "abc123trace" + assert updated.call_event == "call_ended" + assert updated.source == CallRecordingSource.PLAYGROUND + + call_data = updated.call_data + assert len(call_data["messages"]) == 2 + assert call_data["messages"][0]["content"] == "Hello there" + assert call_data["messages"][1]["content"] == "Hi, how can I help?" + assert call_data.get("endedAt") + assert call_data.get("startedAt") == "2026-08-07T09:18:00.000Z" + + +def test_persist_playground_voice_call_sets_agent_on_create(db_session, org_id, default_workspace): + agent = _make_test_agent(db_session, org_id, default_workspace) + recording = persist_playground_voice_call( + db_session, + organization_id=org_id, + workspace_id=default_workspace.id, + agent_id=agent.id, + result_id=str(uuid4().int % 900000 + 100000), + call_metadata={"transcription": "user spoke", "trace_id": "trace-xyz"}, + ) + assert recording is not None + assert recording.agent_id == agent.id + assert recording.trace_id == "trace-xyz" diff --git a/tests/test_services/test_observability/test_elevenlabs_trace.py b/tests/test_services/test_observability/test_elevenlabs_trace.py new file mode 100644 index 00000000..7f9d4612 --- /dev/null +++ b/tests/test_services/test_observability/test_elevenlabs_trace.py @@ -0,0 +1,45 @@ +import json +from pathlib import Path + +from app.services.observability.elevenlabs_trace import ( + enrich_with_turn_metrics, + extract_trace_id, + normalize_elevenlabs_otlp, +) + +FIXTURE_DIR = Path(__file__).resolve().parents[2] / "fixtures" / "elevenlabs" + + +def test_extract_trace_id_from_otlp_fixture(): + payload = json.loads((FIXTURE_DIR / "conv_otel.json").read_text()) + trace_id = extract_trace_id(payload["otlp_traces"]) + assert trace_id == "0af7651916cd43dd8448eb211c80319c" + + +def test_normalize_elevenlabs_otlp_preserves_span_names_and_namespace(): + payload = json.loads((FIXTURE_DIR / "conv_otel.json").read_text()) + normalized = normalize_elevenlabs_otlp( + payload["otlp_traces"], + conversation_id=payload["conversation_id"], + fallback_trace_id="fallback-trace", + ) + + assert normalized["trace_source"] == "elevenlabs" + assert normalized["trace_id"] == "0af7651916cd43dd8448eb211c80319c" + assert normalized["spans"] + assert all(span["name"].startswith("elevenlabs.") for span in normalized["spans"]) + assert all(span["attributes"]["trace.provider"] == "elevenlabs" for span in normalized["spans"]) + + +def test_enrich_with_turn_metrics_adds_metric_spans(): + otel = json.loads((FIXTURE_DIR / "conv_otel.json").read_text()) + conv = json.loads((FIXTURE_DIR / "conv.json").read_text()) + normalized = normalize_elevenlabs_otlp( + otel["otlp_traces"], + conversation_id=otel["conversation_id"], + ) + enriched = enrich_with_turn_metrics(normalized, conv["transcript"]) + names = [span["name"] for span in enriched["spans"]] + assert "elevenlabs.metric.asr" in names + assert "elevenlabs.metric.llm" in names + assert "elevenlabs.metric.tts" in names diff --git a/tests/test_services/test_observability/test_live_event_emitter.py b/tests/test_services/test_observability/test_live_event_emitter.py new file mode 100644 index 00000000..2ef72286 --- /dev/null +++ b/tests/test_services/test_observability/test_live_event_emitter.py @@ -0,0 +1,27 @@ +from uuid import uuid4 + +from app.services.observability.live_event_emitter import LiveObservabilityEmitter + + +def test_live_observability_emitter_posts_turns_and_end( + db_session, seed_org, default_workspace, monkeypatch +): + monkeypatch.setattr("app.services.observability.live_event_emitter.settings.OBSERVABILITY_LIVE_INGEST_ENABLED", True) + org_id = seed_org.id + workspace_id = default_workspace.id + emitter = LiveObservabilityEmitter( + organization_id=org_id, + workspace_id=workspace_id, + provider_call_id=f"pipecat-test-{uuid4().hex[:8]}", + provider_platform="pipecat", + db_factory=lambda: db_session, + ) + + call_short_id = emitter.start_call() + assert call_short_id + + emitter.emit_turn("user", "Hello?") + emitter.emit_turn("assistant", "Hi!", latency={"llm_ms": 300, "tts_ms": 120}) + emitter.end_call(duration_seconds=12.0, trace_id="trace-abc") + + assert emitter.call_short_id == call_short_id diff --git a/tests/test_services/test_observability/test_live_ingest.py b/tests/test_services/test_observability/test_live_ingest.py new file mode 100644 index 00000000..9a800226 --- /dev/null +++ b/tests/test_services/test_observability/test_live_ingest.py @@ -0,0 +1,41 @@ +"""Tests for incremental live observability ingest merge helpers.""" + +from app.services.observability.live_ingest import merge_live_event_call_data + + +def test_merge_call_ended_persists_recording_url(): + merged = merge_live_event_call_data( + existing_call_data={"live_transcript": [{"role": "user", "content": "hi"}]}, + event={ + "event_type": "call.ended", + "event_ts": "2026-08-18T09:00:00Z", + "seq": 3, + "platform": "pipecat", + "payload": { + "endedAt": "2026-08-18T09:00:05Z", + "recording_url": "https://cdn.example/rec.wav", + "duration_seconds": 5.2, + }, + }, + max_out_of_order_seq=5, + ) + assert merged["recording_url"] == "https://cdn.example/rec.wav" + assert merged["duration_seconds"] == 5.2 + assert merged["status"] == "ended" + + +def test_merge_turn_events_build_live_transcript(): + merged = merge_live_event_call_data( + existing_call_data={}, + event={ + "event_type": "turn.assistant", + "event_ts": "2026-08-18T09:00:01Z", + "seq": 2, + "platform": "livekit", + "payload": {"content": "Hello there", "latency": {"llm_ms": 400}}, + }, + max_out_of_order_seq=5, + ) + assert len(merged["live_transcript"]) == 1 + assert merged["live_transcript"][0]["content"] == "Hello there" + assert merged["messages"][0]["role"] == "assistant" diff --git a/tests/test_services/test_observability/test_live_trace.py b/tests/test_services/test_observability/test_live_trace.py new file mode 100644 index 00000000..91c4b30e --- /dev/null +++ b/tests/test_services/test_observability/test_live_trace.py @@ -0,0 +1,48 @@ +from app.services.observability.live_trace import build_live_synthetic_trace + + +def test_build_live_synthetic_trace_builds_turn_and_metric_spans(): + call_data = { + "trace_id": "abc123trace456", + "startedAt": "2026-08-17T06:04:27.000Z", + "endedAt": "2026-08-17T06:04:48.000Z", + "live_transcript": [ + { + "role": "user", + "content": "Hello, can you hear me?", + "event_ts": "2026-08-17T06:04:30.641055+00:00", + }, + { + "role": "assistant", + "content": "Yes, I can hear you clearly.", + "event_ts": "2026-08-17T06:04:33.708357+00:00", + "latency": {"llm_ms": 380, "tts_ms": 210}, + }, + ], + } + + payload = build_live_synthetic_trace( + call_data, + provider_call_id="pipecat-live-1", + provider_platform="pipecat", + ) + + assert payload is not None + assert payload["trace_source"] == "pipecat_live_synthetic" + assert payload["trace_id"] == "abc123trace456" + names = [span["name"] for span in payload["spans"]] + assert "conversation" in names + assert names.count("turn") == 2 + assert "llm" in names + assert "tts" in names + + llm_turn_span = next( + span + for span in payload["spans"] + if span["name"] == "llm" and span["attributes"].get("metric.scope") == "turn_reported" + ) + assert llm_turn_span["duration_ms"] == 380 + + +def test_build_live_synthetic_trace_returns_none_without_transcript(): + assert build_live_synthetic_trace({}, provider_call_id="x", provider_platform="pipecat") is None diff --git a/tests/test_services/test_observability/test_provider_call_enrichment.py b/tests/test_services/test_observability/test_provider_call_enrichment.py new file mode 100644 index 00000000..3ad7fbd4 --- /dev/null +++ b/tests/test_services/test_observability/test_provider_call_enrichment.py @@ -0,0 +1,31 @@ +from app.services.observability.provider_call_enrichment import ( + is_sparse_provider_call_data, + looks_like_retell_call_data, + resolve_observability_provider_platform, +) + + +def test_looks_like_retell_call_data(): + assert looks_like_retell_call_data({"call_id": "call_123", "transcript_object": [{"role": "user", "content": "hi"}]}) + assert not looks_like_retell_call_data({"id": "abc", "messages": [{"role": "user", "content": "hi"}]}) + + +def test_is_sparse_retell_call_data_when_metrics_missing(): + payload = { + "transcript_object": [{"role": "user", "content": "hello"}], + "call_analysis": {"call_summary": "done"}, + } + assert is_sparse_provider_call_data(payload, "retell") is True + + payload["call_cost"] = {"combined_cost": 0.01} + payload["latency"] = {"e2e": {"p50": 900}} + assert is_sparse_provider_call_data(payload, "retell") is False + + +def test_resolve_observability_provider_platform_from_call_shape(): + class _Recording: + provider_platform = "external" + call_data = {"call_id": "call_123", "transcript_object": [{"role": "agent", "content": "hi"}]} + agent_id = None + + assert resolve_observability_provider_platform(_Recording(), _Recording.call_data) == "retell" diff --git a/tests/test_services/test_observability/test_recording_archive.py b/tests/test_services/test_observability/test_recording_archive.py new file mode 100644 index 00000000..f5cfe184 --- /dev/null +++ b/tests/test_services/test_observability/test_recording_archive.py @@ -0,0 +1,40 @@ +from unittest.mock import patch +from uuid import UUID + +from app.services.observability.recording_archive import ( + archive_observability_recording_to_s3, + resolve_observability_recording_url, +) + + +def test_resolve_observability_recording_url_prefers_retell_recording_url(): + call_data = {"recording_url": "https://cdn.retell.ai/recording.wav"} + assert resolve_observability_recording_url(call_data, "retell") == "https://cdn.retell.ai/recording.wav" + + +def test_resolve_observability_recording_url_falls_back_to_retell_multichannel(): + call_data = {"recording_multi_channel_url": "https://cdn.retell.ai/multi.wav"} + assert resolve_observability_recording_url(call_data, "retell") == "https://cdn.retell.ai/multi.wav" + + +@patch("app.services.observability.recording_archive.s3_service") +@patch("app.services.observability.recording_archive._download_recording_bytes") +def test_archive_observability_recording_to_s3_sets_key(mock_download, mock_s3_service): + mock_s3_service.is_enabled.return_value = True + mock_s3_service.prefix = "efficientai/" + mock_download.return_value = (b"audio-bytes", "audio/mpeg") + + call_data = { + "call_status": "ended", + "recording_url": "https://cdn.retell.ai/recording.mp3", + } + archived = archive_observability_recording_to_s3( + call_data=call_data, + provider_platform="retell", + organization_id=UUID("00000000-0000-0000-0000-000000000001"), + call_short_id="123456", + ) + + assert archived["recording_s3_key"].startswith("efficientai/organizations/") + assert archived["recording_source"] == "provider_archive" + mock_s3_service.upload_file_by_key.assert_called_once() diff --git a/tests/test_services/test_observability/test_recording_url_safety.py b/tests/test_services/test_observability/test_recording_url_safety.py new file mode 100644 index 00000000..97fd5c3a --- /dev/null +++ b/tests/test_services/test_observability/test_recording_url_safety.py @@ -0,0 +1,51 @@ +"""Tests for observability recording URL SSRF guards.""" + +import socket +from unittest.mock import patch + +import pytest + +from app.services.observability.recording_url_safety import ( + assert_elevenlabs_recording_url, + build_elevenlabs_conversation_audio_url, +) +from app.services.telephony.exotel_client import ExotelInvalidContentError + + +@pytest.fixture(autouse=True) +def _mock_public_dns(): + with patch.object( + socket, + "getaddrinfo", + return_value=[(None, None, None, None, ("52.0.0.1", 0))], + ): + yield + + +def test_build_elevenlabs_conversation_audio_url(): + url = build_elevenlabs_conversation_audio_url("conv_9001k1zph3fkeh5s8xg9z90swaqa") + assert url == ( + "https://api.elevenlabs.io/v1/convai/conversations/" + "conv_9001k1zph3fkeh5s8xg9z90swaqa/audio" + ) + + +def test_build_elevenlabs_conversation_audio_url_rejects_path_injection(): + with pytest.raises(ExotelInvalidContentError, match="unexpected characters"): + build_elevenlabs_conversation_audio_url("conv_abc/../evil") + + +def test_assert_elevenlabs_recording_url_allows_provider_endpoint(): + assert_elevenlabs_recording_url( + "https://api.elevenlabs.io/v1/convai/conversations/conv_123/audio" + ) + + +def test_assert_elevenlabs_recording_url_rejects_non_elevenlabs_host(): + with pytest.raises(ExotelInvalidContentError, match="not allowlisted"): + assert_elevenlabs_recording_url("https://evil.example/recording.mp3") + + +def test_assert_elevenlabs_recording_url_rejects_non_audio_path(): + with pytest.raises(ExotelInvalidContentError, match="not an allowed conversation audio endpoint"): + assert_elevenlabs_recording_url("https://api.elevenlabs.io/v1/convai/agents/agent_123") diff --git a/tests/test_services/test_observability/test_retell_trace.py b/tests/test_services/test_observability/test_retell_trace.py new file mode 100644 index 00000000..c1c0f34f --- /dev/null +++ b/tests/test_services/test_observability/test_retell_trace.py @@ -0,0 +1,61 @@ +from app.services.observability.retell_trace import build_retell_synthetic_trace + + +def test_build_retell_synthetic_trace_builds_turn_and_metric_spans(): + call_data = { + "call_id": "call_retell_123", + "call_status": "ended", + "start_timestamp": 1_714_423_232_000, + "end_timestamp": 1_714_423_257_000, + "duration_ms": 25000, + "transcript_object": [ + { + "role": "user", + "content": "Hello", + "words": [{"word": "Hello", "start": 0.5, "end": 1.0}], + }, + { + "role": "agent", + "content": "Hi there", + "words": [{"word": "Hi", "start": 1.4, "end": 1.6}, {"word": "there", "start": 1.6, "end": 2.0}], + }, + ], + "latency": { + "asr": {"p50": 180, "p90": 240}, + "llm": {"p50": 420, "p90": 510}, + "tts": {"p50": 260, "p90": 320}, + "e2e": {"p50": 980, "p90": 1200}, + }, + } + + payload = build_retell_synthetic_trace(call_data, provider_call_id="call_retell_123") + + assert payload is not None + assert payload["trace_source"] == "retell_synthetic" + assert payload["trace_id"] == "retell-call_retell_123" + names = [span["name"] for span in payload["spans"]] + assert "conversation" in names + assert "turn" in names + assert "stt" in names + assert "llm" in names + assert "tts" in names + + user_turn = next( + span for span in payload["spans"] if span["name"] == "turn" and span["attributes"].get("turn.role") == "user" + ) + assert user_turn["attributes"]["turn.user_transcript"] == "Hello" + assert user_turn["start_time"] == 1_714_423_232_000 + 500 + + +def test_build_retell_synthetic_trace_parses_plain_text_transcript(): + call_data = { + "call_id": "call_retell_text", + "transcript": "User: Need help\nAgent: Sure thing", + "latency": {"llm": {"p50": 300}}, + } + + payload = build_retell_synthetic_trace(call_data, provider_call_id="call_retell_text") + + assert payload is not None + turn_spans = [span for span in payload["spans"] if span["name"] == "turn"] + assert len(turn_spans) == 2 diff --git a/tests/test_services/test_observability/test_trace_archive.py b/tests/test_services/test_observability/test_trace_archive.py new file mode 100644 index 00000000..25af801e --- /dev/null +++ b/tests/test_services/test_observability/test_trace_archive.py @@ -0,0 +1,63 @@ +from unittest.mock import patch +from uuid import UUID + +from app.services.observability.trace_archive import load_provider_trace, persist_provider_trace + + +@patch("app.services.observability.trace_archive.s3_service") +def test_persist_provider_trace_inline(mock_s3_service): + mock_s3_service.is_enabled.return_value = False + updated = persist_provider_trace( + call_data={"status": "ended"}, + provider_platform="retell", + organization_id=UUID("00000000-0000-0000-0000-000000000001"), + call_short_id="abc123", + trace_payload={ + "trace_id": "retell-call_123", + "root_span_id": "root", + "spans": [{"span_id": "root", "name": "conversation"}], + "trace_source": "retell_synthetic", + }, + source="retell_synthetic", + ) + + provider_trace = updated["provider_trace"] + assert provider_trace["storage"] == "inline" + assert provider_trace["trace_source"] == "retell_synthetic" + assert updated["trace_id"] == "retell-call_123" + assert isinstance(provider_trace.get("normalized_trace"), dict) + + +@patch("app.services.observability.trace_archive.s3_service") +def test_persist_provider_trace_archives_large_payload(mock_s3_service): + mock_s3_service.is_enabled.return_value = True + mock_s3_service.prefix = "efficientai/" + mock_s3_service.upload_file_by_key.return_value = "key" + mock_s3_service.download_file_by_key.return_value = ( + b'{"trace_payload":{"trace_id":"el-1","root_span_id":"root","spans":[{"span_id":"root","name":"conversation"}],' + b'"trace_source":"elevenlabs"}}' + ) + + updated = persist_provider_trace( + call_data={"status": "done"}, + provider_platform="elevenlabs", + organization_id=UUID("00000000-0000-0000-0000-000000000001"), + call_short_id="el123", + trace_payload={ + "trace_id": "el-1", + "root_span_id": "root", + "spans": [{"span_id": "root", "name": "conversation"}], + "trace_source": "elevenlabs", + }, + source="elevenlabs_post_call_webhook", + raw_payload={"resourceSpans": [{"x": "y" * 5000}]}, + inline_limit_bytes=64, + ) + + provider_trace = updated["provider_trace"] + assert provider_trace["storage"] == "s3" + assert isinstance(provider_trace.get("trace_s3_key"), str) + + loaded = load_provider_trace(updated) + assert loaded is not None + assert loaded["trace_id"] == "el-1" diff --git a/tests/test_services/test_observability/test_vapi_trace.py b/tests/test_services/test_observability/test_vapi_trace.py new file mode 100644 index 00000000..6da808c1 --- /dev/null +++ b/tests/test_services/test_observability/test_vapi_trace.py @@ -0,0 +1,35 @@ +from app.services.observability.vapi_trace import build_vapi_synthetic_trace + + +def test_build_vapi_synthetic_trace_builds_turn_and_metric_spans(): + call_data = { + "id": "call_vapi_123", + "status": "ended", + "startedAt": "2026-08-07T09:00:00.000Z", + "endedAt": "2026-08-07T09:00:10.000Z", + "messages": [ + {"role": "user", "message": "hello", "secondsFromStart": 0.5, "duration": 900}, + {"role": "assistant", "message": "hi there", "secondsFromStart": 1.6, "duration": 1200}, + ], + "artifact": { + "performanceMetrics": { + "modelLatencyAverage": 320, + "voiceLatencyAverage": 480, + "transcriberLatencyAverage": 210, + "endpointingLatencyAverage": 140, + "turnLatencyAverage": 2200, + } + }, + } + + payload = build_vapi_synthetic_trace(call_data, provider_call_id="call_vapi_123") + + assert payload is not None + assert payload["trace_source"] == "vapi_synthetic" + assert payload["trace_id"] == "vapi-call_vapi_123" + names = [span["name"] for span in payload["spans"]] + assert "conversation" in names + assert "turn" in names + assert "stt" in names + assert "llm" in names + assert "tts" in names diff --git a/tests/test_services/test_telephony/test_live_recording.py b/tests/test_services/test_telephony/test_live_recording.py new file mode 100644 index 00000000..7c4079bb --- /dev/null +++ b/tests/test_services/test_telephony/test_live_recording.py @@ -0,0 +1,54 @@ +"""Tests for live telephony recording merge helpers.""" + +from __future__ import annotations + +import struct +import wave +from pathlib import Path + +import numpy as np + +from app.services.telephony.live_recording import merge_live_tracks_mono, read_growing_wav_mono + + +def _write_wav(path: Path, samples: np.ndarray, sample_rate: int = 16000) -> None: + with wave.open(str(path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(samples.astype(np.int16).tobytes()) + + +def test_read_growing_wav_mono_reads_partial_file(tmp_path: Path) -> None: + samples = np.array([1000, -1000, 2000, -2000], dtype=np.int16) + path = tmp_path / "partial.wav" + _write_wav(path, samples) + + # Simulate a growing file by truncating after header + partial PCM. + raw = path.read_bytes() + partial = raw[:44 + 4] + partial_path = tmp_path / "growing.wav" + partial_path.write_bytes(partial) + + pcm, sample_rate = read_growing_wav_mono(str(partial_path)) + assert sample_rate == 16000 + assert len(pcm) == 2 + + +def test_merge_live_tracks_mono_mixes_user_and_bot(tmp_path: Path) -> None: + user_path = tmp_path / "user.wav" + bot_path = tmp_path / "bot.wav" + _write_wav(user_path, np.array([1000, 0, 1000, 0], dtype=np.int16)) + _write_wav(bot_path, np.array([0, 1000, 0, 1000], dtype=np.int16)) + + wav_bytes, duration_sec, sample_rate = merge_live_tracks_mono(str(user_path), str(bot_path)) + assert sample_rate == 16000 + assert duration_sec > 0 + assert wav_bytes.startswith(b"RIFF") + + merged_path = tmp_path / "merged-out.wav" + merged_path.write_bytes(wav_bytes) + merged, merged_rate = read_growing_wav_mono(str(merged_path)) + assert merged_rate == 16000 + assert len(merged) == 4 + assert merged[0] == 500 diff --git a/tests/test_services/test_voice_providers/test_provider_helpers.py b/tests/test_services/test_voice_providers/test_provider_helpers.py index b0bd4085..03d13bc4 100644 --- a/tests/test_services/test_voice_providers/test_provider_helpers.py +++ b/tests/test_services/test_voice_providers/test_provider_helpers.py @@ -1,8 +1,14 @@ """Tests for pure helper behavior in provider classes.""" +import json +from pathlib import Path + from app.services.voice_providers.elevenlabs import ElevenLabsVoiceProvider +from app.services.voice_providers.retell import RetellVoiceProvider from app.services.voice_providers.vapi import VapiVoiceProvider +FIXTURE_DIR = Path(__file__).resolve().parents[2] / "fixtures" / "elevenlabs" + def test_strip_code_fences_handles_complete_and_partial_blocks(): provider = ElevenLabsVoiceProvider(api_key="k") @@ -22,3 +28,195 @@ def test_vapi_make_json_serializable_converts_nested_values(): result = provider._make_json_serializable(payload) assert result == payload + + +def test_elevenlabs_list_agents_normalizes_response(monkeypatch): + fixture = json.loads((FIXTURE_DIR / "agents_list.json").read_text()) + + class _Resp: + def raise_for_status(self): + return None + + def json(self): + return fixture + + monkeypatch.setattr( + "app.services.voice_providers.elevenlabs.requests.request", + lambda *_args, **_kwargs: _Resp(), + ) + + provider = ElevenLabsVoiceProvider(api_key="k") + payload = provider.list_agents(page_size=10) + assert payload["has_more"] is False + assert payload["agents"][0]["id"].startswith("agent_") + assert payload["agents"][0]["name"] == "Customer Support Agent" + + +def test_elevenlabs_retrieve_conversation_trace_returns_payload(monkeypatch): + fixture = json.loads((FIXTURE_DIR / "conv_otel.json").read_text()) + + class _Resp: + def raise_for_status(self): + return None + + def json(self): + return fixture + + monkeypatch.setattr( + "app.services.voice_providers.elevenlabs.requests.request", + lambda *_args, **_kwargs: _Resp(), + ) + + provider = ElevenLabsVoiceProvider(api_key="k") + payload = provider.retrieve_conversation_trace("conv_123") + assert payload["conversation_id"] == fixture["conversation_id"] + assert "otlp_traces" in payload + + +def test_elevenlabs_list_conversations_normalizes_response(monkeypatch): + fixture = json.loads((FIXTURE_DIR / "conversations_list.json").read_text()) + + class _Resp: + def raise_for_status(self): + return None + + def json(self): + return fixture + + monkeypatch.setattr( + "app.services.voice_providers.elevenlabs.requests.request", + lambda *_args, **_kwargs: _Resp(), + ) + + provider = ElevenLabsVoiceProvider(api_key="k") + payload = provider.list_conversations(page_size=50, agent_id="agent_abc") + assert payload["has_more"] is False + assert payload["conversations"][0]["conversation_id"] == "conv_12345" + assert payload["conversations"][0]["status"] == "done" + + +def test_vapi_list_agents_normalizes_response(monkeypatch): + fixture = { + "assistants": [ + { + "id": "assist_abc", + "name": "Vapi Support Agent", + "isArchived": False, + "createdAt": "2026-02-02T10:00:00.000Z", + } + ], + "has_more": False, + "next_cursor": None, + } + + class _Resp: + ok = True + status_code = 200 + + def json(self): + return fixture + + monkeypatch.setattr( + "app.services.voice_providers.vapi.requests.get", + lambda *_args, **_kwargs: _Resp(), + ) + + provider = VapiVoiceProvider(api_key="k") + payload = provider.list_agents(page_size=10) + assert payload["has_more"] is False + assert payload["agents"][0]["id"] == "assist_abc" + assert payload["agents"][0]["name"] == "Vapi Support Agent" + + +def test_retell_list_agents_normalizes_response(monkeypatch): + provider = RetellVoiceProvider(api_key="k") + + class _Response: + def model_dump(self): + return { + "items": [ + { + "agent_id": "agent_retell_123", + "agent_name": "Retell Support Agent", + "is_archived": False, + "created_at": "2026-01-02T00:00:00.000Z", + } + ] + } + + class _AgentApi: + @staticmethod + def list(): + return _Response() + + monkeypatch.setattr(provider, "client", type("C", (), {"agent": _AgentApi()})()) + payload = provider.list_agents(page_size=10) + assert payload["agents"][0]["id"] == "agent_retell_123" + assert payload["agents"][0]["name"] == "Retell Support Agent" + + +def test_retell_extract_agent_prompt_prefers_inline_response_engine_prompt(monkeypatch): + provider = RetellVoiceProvider(api_key="k") + + class _AgentApi: + @staticmethod + def retrieve(agent_id): + del agent_id + return { + "agent_id": "agent_retell_123", + "response_engine": {"type": "retell-llm", "general_prompt": "Inline general prompt"}, + } + + monkeypatch.setattr(provider, "client", type("C", (), {"agent": _AgentApi()})()) + assert provider.extract_agent_prompt("agent_retell_123") == "Inline general prompt" + + +def test_retell_extract_agent_prompt_handles_llm_lookup_error_with_fallback(monkeypatch): + provider = RetellVoiceProvider(api_key="k") + + class _AgentApi: + @staticmethod + def retrieve(agent_id): + del agent_id + return { + "agent_id": "agent_retell_123", + "response_engine": { + "type": "retell-llm", + "llm_id": "llm_123", + "system_prompt": "Fallback system prompt", + }, + } + + class _LlmApi: + @staticmethod + def retrieve(llm_id): + raise RuntimeError(f"failed to read llm {llm_id}") + + monkeypatch.setattr(provider, "client", type("C", (), {"agent": _AgentApi(), "llm": _LlmApi()})()) + assert provider.extract_agent_prompt("agent_retell_123") == "Fallback system prompt" + + +def test_retell_extract_agent_prompt_reads_camel_case_llm_prompt(monkeypatch): + provider = RetellVoiceProvider(api_key="k") + + class _AgentApi: + @staticmethod + def retrieve(agent_id): + del agent_id + return { + "agent_id": "agent_retell_123", + "response_engine": {"type": "retell-llm", "llm_id": "llm_123"}, + } + + class _LlmApi: + @staticmethod + def retrieve(llm_id): + del llm_id + return { + "settings": { + "systemPrompt": "Prompt from camelCase settings", + } + } + + monkeypatch.setattr(provider, "client", type("C", (), {"agent": _AgentApi(), "llm": _LlmApi()})()) + assert provider.extract_agent_prompt("agent_retell_123") == "Prompt from camelCase settings" diff --git a/tests/test_workers/test_finalize_telephony_recording.py b/tests/test_workers/test_finalize_telephony_recording.py index b5978686..8e58aa3b 100644 --- a/tests/test_workers/test_finalize_telephony_recording.py +++ b/tests/test_workers/test_finalize_telephony_recording.py @@ -28,6 +28,7 @@ def test_finalize_telephony_recording_task_wires_merge_persist( conversation_turns=[{"speaker": "user", "text": "hi", "start": 0, "end": 1}], transcript_text="user: hi", duration=40.0, + trace_id="0af7651916cd43dd8448eb211c80319c", ) mock_merge.assert_called_once_with( @@ -45,6 +46,7 @@ def test_finalize_telephony_recording_task_wires_merge_persist( transcript_text="user: hi", s3_key="org/eval/audio.wav", duration=42.5, + trace_id="0af7651916cd43dd8448eb211c80319c", ) assert result["status"] == "ok" assert result["s3_key"] == "org/eval/audio.wav" diff --git a/tests/test_workers/test_process_call_import_row.py b/tests/test_workers/test_process_call_import_row.py index 9c598414..fe0fe493 100644 --- a/tests/test_workers/test_process_call_import_row.py +++ b/tests/test_workers/test_process_call_import_row.py @@ -170,6 +170,11 @@ def is_enabled(self): def get_status_message(self): return None if self._enabled else "S3 disabled in tests" + def download_file_by_key(self, key, **_kwargs): + self.downloads = getattr(self, "downloads", []) + self.downloads.append(key) + return b"fake-audio-bytes" + def upload_file_by_key(self, file_content, key, content_type="audio/mpeg"): self.uploads.append({"key": key, "size": len(file_content), "content_type": content_type}) return key diff --git a/tests/test_workers/test_telephony_credential_rate_limit.py b/tests/test_workers/test_telephony_credential_rate_limit.py index b52c673f..f13f0ca9 100644 --- a/tests/test_workers/test_telephony_credential_rate_limit.py +++ b/tests/test_workers/test_telephony_credential_rate_limit.py @@ -42,12 +42,14 @@ def _test_fingerprint(provider: str, auth_id: str, api_endpoint: str) -> str: monkeypatch.setattr(module, "telephony_credential_fingerprint", _test_fingerprint) monkeypatch.setattr(module, "_redis_client", client) - for key in client.scan_iter(match=f"{prefix}*"): + key_pattern = f"{module._KEY_PREFIX}:*:{prefix}*" + + for key in client.scan_iter(match=key_pattern): client.delete(key) yield client - for key in client.scan_iter(match=f"{prefix}*"): + for key in client.scan_iter(match=key_pattern): client.delete(key) module._redis_client = None