diff --git a/.github/workflows/backend-tests-postgres.yml b/.github/workflows/backend-tests-postgres.yml index 6fc18bc4..30be09f4 100644 --- a/.github/workflows/backend-tests-postgres.yml +++ b/.github/workflows/backend-tests-postgres.yml @@ -41,6 +41,8 @@ jobs: REDIS_URL: redis://localhost:6379/0 CELERY_BROKER_URL: redis://localhost:6379/0 CELERY_RESULT_BACKEND: redis://localhost:6379/0 + EFFICIENTAI_PYTEST: "1" + FLEXPRICE_ENABLED: "false" steps: - name: Checkout repository @@ -83,6 +85,8 @@ jobs: SHARD_DATABASE_URL_02: postgresql://postgres:postgres@localhost:5432/efficientai_data_02 SHARDING_INTEGRATION_TEST: "1" REDIS_URL: redis://localhost:6379/0 + EFFICIENTAI_PYTEST: "1" + FLEXPRICE_ENABLED: "false" steps: - name: Checkout repository diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 17094877..f02f45a7 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -2,16 +2,19 @@ Agents API Routes Complete CRUD operations for test agents """ -from fastapi import APIRouter, Depends, HTTPException, status, Query +from uuid import uuid4 + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status, Query from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from typing import List, Optional -from uuid import UUID +from uuid import UUID, uuid4 import random from pydantic import BaseModel from loguru import logger from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key +from app.services.billing.flexprice_service import record_agent_test_setup_generated from app.models.database import ( Agent, ConversationEvaluation, TestAgentConversation, VoiceBundle, AIProvider, Integration, IntegrationPlatform, CallMediumEnum, @@ -254,6 +257,7 @@ def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse] @router.post("/generate-test-prompt", response_model=GenerateTestPromptResponse) async def generate_test_prompt( data: GenerateTestPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -297,6 +301,14 @@ async def generate_test_prompt( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="test_prompt", + model=result.model, + ) return GenerateTestPromptResponse( sections=_test_prompt_section_responses(result.sections), test_agent_prompt=result.test_agent_prompt, @@ -313,6 +325,7 @@ async def generate_test_prompt( @router.post("/generate-scenarios-from-prompt", response_model=GenerateScenariosFromPromptResponse) async def generate_scenarios_from_prompt( data: GenerateScenariosFromPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -357,6 +370,15 @@ async def generate_scenarios_from_prompt( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="scenarios", + model=result.model, + scenario_count=len(result.scenarios), + ) return GenerateScenariosFromPromptResponse( scenarios=_scenario_draft_responses(result.scenarios), provider=result.provider, @@ -372,6 +394,7 @@ async def generate_scenarios_from_prompt( @router.post("/generate-test-setup", response_model=GenerateTestSetupResponse) async def generate_test_setup( data: GenerateTestSetupRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -430,6 +453,15 @@ async def generate_test_setup( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="full_setup", + model=scenario_result.model, + scenario_count=len(scenario_result.scenarios), + ) return GenerateTestSetupResponse( sections=_test_prompt_section_responses(prompt_result.sections), test_agent_prompt=prompt_result.test_agent_prompt, diff --git a/app/api/v1/routes/auth.py b/app/api/v1/routes/auth.py index 9077f120..11c1474d 100644 --- a/app/api/v1/routes/auth.py +++ b/app/api/v1/routes/auth.py @@ -287,6 +287,18 @@ def _extract_bearer(authorization: Optional[str]) -> Optional[str]: return token.strip() +def _revoke_local_password_access_token(bearer: str) -> None: + try: + claims = decode_access_token(bearer) + jti = claims.get("jti") + exp = claims.get("exp") + if jti and exp: + ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) + revoke_access_jti(jti, ttl) + except JWTError: + pass + + def _issue_session_tokens( db: Session, *, @@ -611,15 +623,7 @@ def logout( """Revoke the current session's refresh token and blacklist the access token.""" bearer = _extract_bearer(authorization) if bearer and principal.auth_method == AuthMethod.LOCAL_PASSWORD: - try: - claims = decode_access_token(bearer) - jti = claims.get("jti") - exp = claims.get("exp") - if jti and exp: - ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) - revoke_access_jti(jti, ttl) - except JWTError: - pass + _revoke_local_password_access_token(bearer) if payload and payload.refresh_token: revoke_refresh_token(db, payload.refresh_token) @@ -697,12 +701,14 @@ def refresh_session(payload: RefreshRequest, db: Session = Depends(get_db)) -> T class SwitchOrgRequest(BaseModel): organization_id: str + refresh_token: Optional[str] = None @router.post("/switch-org", response_model=TokenResponse) def switch_organization( payload: SwitchOrgRequest, principal: Principal = Depends(get_principal), + authorization: Optional[str] = Header(None, alias="Authorization"), db: Session = Depends(get_db), ) -> TokenResponse: """ @@ -766,6 +772,13 @@ def switch_organization( detail="User is no longer active.", ) + if principal.auth_method == AuthMethod.LOCAL_PASSWORD: + bearer = _extract_bearer(authorization) + if bearer: + _revoke_local_password_access_token(bearer) + if payload.refresh_token: + revoke_refresh_token(db, payload.refresh_token) + user.last_login_at = datetime.now(timezone.utc) db.commit() db.refresh(user) diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 18749f6d..54188b08 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -1281,6 +1281,20 @@ def _name_for_source(source: str) -> Optional[str]: transcribe_overwrite=payload.transcribe_overwrite, ) + from app.services.billing.flexprice_service import ( + record_call_import_evaluation_started, + ) + + for evaluation in created_evaluations: + record_call_import_evaluation_started( + organization_id, + evaluation.id, + workspace_id=evaluation.workspace_id, + call_import_id=call_import.id, + total_rows=int(evaluation.total_rows or 0), + metric_count=len(leaf_metric_ids), + ) + for evaluation in created_evaluations: db.refresh(evaluation) @@ -3980,6 +3994,18 @@ async def generate_call_import_evaluation_pdf_report( detail="Failed to store PDF report due to a concurrent duplicate request.", ) from None db.refresh(pdf_report) + from app.services.billing.flexprice_service import ( + record_call_import_pdf_report_generated, + ) + + record_call_import_pdf_report_generated( + organization_id, + pdf_report.id, + workspace_id=pdf_report.workspace_id, + evaluation_id=evaluation.id, + call_import_id=call_import_id, + report_type=pdf_report.report_type, + ) return _pdf_report_response_from_row(pdf_report) @@ -6926,6 +6952,16 @@ def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: # in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to # avoid a worker import cycle from the routes module. DISCOVERED_METRICS_KEY = "__discovered_metrics__" +METRIC_SCORES_META_KEYS = frozenset({DISCOVERED_METRICS_KEY, "_billing"}) + + +def _is_metric_scores_meta_key(key: str) -> bool: + normalized = str(key or "").strip().lower() + if not normalized: + return True + if normalized in {item.lower() for item in METRIC_SCORES_META_KEYS}: + return True + return normalized.endswith("__discovered") # Allowed values for an LLM-suggested top-level metric type. Kept in # sync with ``DiscoveredMetricSuggestedType`` in @@ -8674,7 +8710,8 @@ def _reset_eval_row_for_retry( eval_row.metric_scores = { key: value for key, value in existing.items() - if str(key).lower() not in target_keys + if _is_metric_scores_meta_key(str(key)) + or str(key).lower() not in target_keys } else: eval_row.metric_scores = {} @@ -9181,12 +9218,15 @@ async def retry_call_import_evaluation( payload.metric_ids if payload else None ) if metric_ids is not None: + metric_ids = [ + mid for mid in metric_ids if not _is_metric_scores_meta_key(str(mid)) + ] if not metric_ids: raise HTTPException( status_code=400, detail=( - "metric_ids must be a non-empty list. Omit the " - "field to re-run all metrics." + "metric_ids must be a non-empty list of metric UUIDs. " + "Omit the field to re-run all metrics." ), ) diff --git a/app/api/v1/routes/chat.py b/app/api/v1/routes/chat.py index 6fc026f8..6f023654 100644 --- a/app/api/v1/routes/chat.py +++ b/app/api/v1/routes/chat.py @@ -28,6 +28,7 @@ class ChatRequest(BaseModel): temperature: Optional[float] = 0.7 max_tokens: Optional[int] = None llm_config: Optional[Dict[str, Any]] = None + usage_purpose: Optional[str] = "scenario_description" class ChatResponse(BaseModel): @@ -80,6 +81,7 @@ async def chat_completion( uuid4(), workspace_id=workspace_id, model=result.get("model", request.model), + purpose=request.usage_purpose or "scenario_description", ) return ChatResponse( diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index 962207c6..093ea249 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -864,15 +864,17 @@ def re_evaluate_result( detail="Cannot re-evaluate: this result has no transcription. It must be transcribed first." ) - if not result.evaluator_id: + if not result.evaluator_id and not result.agent_id: raise HTTPException( status_code=400, - detail="Cannot re-evaluate: this result is not linked to an evaluator." + detail="Cannot re-evaluate: this result is not linked to an agent or evaluator." ) - evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() - if not evaluator: - raise HTTPException(status_code=404, detail="Linked evaluator no longer exists") + evaluator = None + if result.evaluator_id: + evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() + if not evaluator: + raise HTTPException(status_code=404, detail="Linked evaluator no longer exists") # ------------------------------------------------------------------ # If no audio in S3 yet, try to download from the voice provider diff --git a/app/api/v1/routes/metric_studio.py b/app/api/v1/routes/metric_studio.py index aeb155f3..10b407ca 100644 --- a/app/api/v1/routes/metric_studio.py +++ b/app/api/v1/routes/metric_studio.py @@ -25,6 +25,7 @@ MetricStudioRunRetryRequest, ) from app.services.metric_studio.metric_selection import expand_studio_metric_selection +from app.services.metric_studio.run_rollup import rollup_metric_studio_run from app.services.metric_studio.source_resolver import resolve_source router = APIRouter(prefix="/metric-studio", tags=["metric-studio"]) @@ -117,26 +118,7 @@ def _serialize_result( def _rollup_run_status(db: Session, run: MetricStudioRun) -> None: - results = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run.id) - .all() - ) - completed = sum(1 for r in results if r.status == "completed") - failed = sum(1 for r in results if r.status == "failed") - pending = sum(1 for r in results if r.status in {"pending", "running"}) - run.completed_items = completed - run.failed_items = failed - if pending: - run.status = "running" - elif failed and completed: - run.status = "partial" - elif failed: - run.status = "failed" - else: - run.status = "completed" - run.finished_at = datetime.now(timezone.utc) - db.flush() + rollup_metric_studio_run(db, run, emit_flexprice=True, commit=False) @router.post( @@ -297,6 +279,8 @@ def get_metric_studio_run( ) if not run: raise HTTPException(status_code=404, detail="Studio run not found.") + rollup_metric_studio_run(db, run, emit_flexprice=True, commit=True) + db.refresh(run) return _serialize_run(run) diff --git a/app/api/v1/routes/observability.py b/app/api/v1/routes/observability.py index 4c26a549..611e7866 100644 --- a/app/api/v1/routes/observability.py +++ b/app/api/v1/routes/observability.py @@ -5,15 +5,11 @@ from typing import Any, Dict, List, Optional, Union from uuid import UUID -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, ConfigDict from sqlalchemy.orm import Session from app.dependencies import get_api_key, get_db, get_organization_id, get_workspace_id -from app.services.billing.flexprice_service import ( - record_observability_call_evaluated, - record_observability_call_ingested, -) from app.models.database import ( Agent, APIKey, CallRecording, CallRecordingStatus, CallRecordingSource, Evaluator, EvaluatorResult, EvaluatorResultStatus, Scenario, Workspace, @@ -243,13 +239,6 @@ def _upsert_call_recording( response = _serialize_call_recording(call_recording, include_data=True, agent=agent_obj) response["action"] = action - if action == "created": - record_observability_call_ingested( - organization_id, - call_recording.call_short_id, - workspace_id=workspace_id, - provider=provider_platform, - ) return response @@ -765,7 +754,6 @@ def _messages_to_speaker_segments(messages: List[Dict[str, Any]]) -> List[Dict[s async def evaluate_call( call_short_id: str, payload: EvaluateCallPayload, - background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -877,13 +865,6 @@ async def evaluate_call( except Exception: pass - background_tasks.add_task( - record_observability_call_evaluated, - organization_id, - call_short_id, - workspace_id=workspace_id, - ) - return { "evaluator_result_id": str(evaluator_result.id), "result_id": evaluator_result.result_id, diff --git a/app/api/v1/routes/personas.py b/app/api/v1/routes/personas.py index de07c715..131c7e14 100644 --- a/app/api/v1/routes/personas.py +++ b/app/api/v1/routes/personas.py @@ -3,12 +3,12 @@ CRUD for TTS provider-tied voice personas, voice-options catalog, and custom voice management (ungated). """ -from fastapi import APIRouter, Depends, HTTPException, status, Body, Query +from fastapi import APIRouter, Depends, HTTPException, status, Body, Query, BackgroundTasks from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any -from uuid import UUID +from uuid import UUID, uuid4 from pydantic import BaseModel from loguru import logger @@ -427,6 +427,7 @@ async def get_agent_prompt_sources( ) async def generate_persona_prompt( data: GeneratePersonaPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -442,19 +443,38 @@ async def generate_persona_prompt( provider_enum, model_str = _get_llm_provider_and_model( organization_id, db, data.provider, data.model, data.credential_id ) + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_persona_generation, + ) + try: - result = generate_persona_prompt_from_agent( - agent, - source=data.source, - persona_name=data.persona_name, - persona_gender=data.persona_gender, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, + with llm_usage_context( + usage_context_for_persona_generation(agent, workspace_id=workspace_id) + ): + result = generate_persona_prompt_from_agent( + agent, + source=data.source, + persona_name=data.persona_name, + persona_gender=data.persona_gender, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + from app.services.billing.flexprice_service import record_persona_prompt_generated + + background_tasks.add_task( + record_persona_prompt_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + agent_id=agent.id, + model=result.model, + source=result.source_used, ) return GeneratePersonaPromptResponse( persona_prompt=result.persona_prompt, diff --git a/app/api/v1/routes/platform_admin.py b/app/api/v1/routes/platform_admin.py index f29630ff..433b1e3c 100644 --- a/app/api/v1/routes/platform_admin.py +++ b/app/api/v1/routes/platform_admin.py @@ -6,7 +6,8 @@ from typing import List, Optional from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi import APIRouter, Depends, Header, HTTPException, Query, status +from jose import JWTError from pydantic import BaseModel, EmailStr, Field from sqlalchemy import func from sqlalchemy.orm import Session @@ -16,6 +17,7 @@ create_platform_access_token, get_platform_admin, platform_admin_feature_enabled, + revoke_platform_access_token, ) from app.core.auth.refresh_tokens import revoke_all_user_refresh_tokens from app.core.password import hash_password, validate_password_strength, verify_password @@ -184,6 +186,26 @@ def platform_me( return PlatformAdminSummary(id=str(principal.platform_admin_id), email=principal.email) +def _extract_bearer(authorization: Optional[str]) -> Optional[str]: + if not authorization: + return None + scheme, _, token = authorization.partition(" ") + if scheme.lower() != "bearer" or not token.strip(): + return None + return token.strip() + + +@router.post("/auth/logout") +def platform_logout( + authorization: Optional[str] = Header(None, alias="Authorization"), + principal: PlatformAdminPrincipal = Depends(get_platform_admin), +) -> dict: + bearer = _extract_bearer(authorization) + if bearer: + revoke_platform_access_token(bearer) + return {"success": True, "admin_id": str(principal.platform_admin_id)} + + @router.get("/organizations", response_model=OrganizationListResponse) def list_organizations( offset: int = Query(0, ge=0), diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 31b1a3c2..1681ecbd 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -74,6 +74,7 @@ def poll_call_metrics( """ import time from app.database import SessionLocal + from app.services.playground.post_call_processing import merge_playground_call_data from app.services.voice_providers import get_voice_provider db = SessionLocal() @@ -109,7 +110,14 @@ def poll_call_metrics( # For other providers, implement similar method continue - # Update the call recording with metrics + # Update the call recording with metrics (preserve usage-recorded flag) + prev_data = ( + call_recording.call_data + if isinstance(call_recording.call_data, dict) + else {} + ) + if isinstance(call_metrics, dict): + call_metrics = merge_playground_call_data(prev_data, call_metrics) call_recording.call_data = call_metrics call_recording.status = CallRecordingStatus.UPDATED db.commit() @@ -138,6 +146,20 @@ def poll_call_metrics( # After polling is complete, create EvaluatorResult and trigger evaluation if call_complete and call_metrics and call_recording.agent_id: try: + from app.services.playground.post_call_processing import ( + claim_playground_evaluator_result_slot, + record_playground_post_call_usage_once, + ) + + should_create_evaluator, call_metrics = record_playground_post_call_usage_once( + db, + call_recording_id, + provider_platform=provider_platform, + call_metrics=call_metrics if isinstance(call_metrics, dict) else {}, + ) + if not should_create_evaluator: + return + logger.info(f"[Poll Call Metrics] Call complete, creating EvaluatorResult for call {provider_call_id}") # Extract transcript and speaker segments from call_data @@ -222,18 +244,21 @@ def poll_call_metrics( except Exception as audio_err: logger.warning(f"[Poll Call Metrics] Audio download/upload failed: {audio_err}") - # Generate unique result ID + locked_recording = claim_playground_evaluator_result_slot( + db, + call_recording_id, + provider_call_id=provider_call_id, + ) + if not locked_recording: + return + result_id = generate_unique_result_id(db) - - # Create EvaluatorResult. Background-task path: inherit the - # workspace from the call recording rather than the header - # (this task runs without a request context). evaluator_result = EvaluatorResult( result_id=result_id, - organization_id=call_recording.organization_id, - workspace_id=call_recording.workspace_id, + organization_id=locked_recording.organization_id, + workspace_id=locked_recording.workspace_id, evaluator_id=None, - agent_id=call_recording.agent_id, + agent_id=locked_recording.agent_id, persona_id=None, scenario_id=None, name=result_name, @@ -246,13 +271,11 @@ def poll_call_metrics( call_data=call_metrics, ) db.add(evaluator_result) + db.flush() + locked_recording.evaluator_result_id = evaluator_result.id db.commit() db.refresh(evaluator_result) - # Link the EvaluatorResult to CallRecording - call_recording.evaluator_result_id = evaluator_result.id - db.commit() - logger.info(f"[Poll Call Metrics] Created EvaluatorResult {result_id} for call {provider_call_id}") # Trigger Celery task to process evaluator result (run metrics evaluation) @@ -321,13 +344,16 @@ async def update_call_recording( if integration: try: decrypted_api_key = decrypt_api_key(integration.api_key) - background_tasks.add_task( - poll_call_metrics, - call_recording.id, - call_recording.provider_call_id, - call_recording.provider_platform, - decrypted_api_key - ) + platform_key = (call_recording.provider_platform or "").lower() + # Vapi polls on call-end refresh only; update fires mid-call too. + if platform_key != "vapi": + background_tasks.add_task( + poll_call_metrics, + call_recording.id, + call_recording.provider_call_id, + call_recording.provider_platform, + decrypted_api_key + ) except: pass @@ -343,6 +369,7 @@ class WebCallCreate(BaseModel): metadata: Optional[Dict[str, Any]] = None retell_llm_dynamic_variables: Optional[Dict[str, Any]] = None custom_sip_headers: Optional[Dict[str, str]] = None + ui_surface: Optional[str] = None @router.post("/web-call", response_model=Dict[str, Any]) @@ -477,13 +504,16 @@ async def create_web_call( provider_call_id = web_call_response.get("call_id") call_short_id = generate_unique_call_short_id(db) + stored_call_data = dict(web_call_response) + if web_call_data.ui_surface: + stored_call_data["ui_surface"] = web_call_data.ui_surface call_recording = CallRecording( organization_id=organization_id, workspace_id=workspace_id, call_short_id=call_short_id, status=CallRecordingStatus.PENDING, source=CallRecordingSource.PLAYGROUND, - call_data=web_call_response, # Store initial response + call_data=stored_call_data, provider_call_id=provider_call_id, provider_platform=integration.platform, agent_id=agent.id @@ -504,7 +534,7 @@ async def create_web_call( # Note: We need to pass the decrypted API key, but we should be careful with security # For now, we'll pass it to the background task # In production, you might want to store it temporarily or use a different approach - if provider_call_id: + if provider_call_id and plat_lower != "vapi": background_tasks.add_task( poll_call_metrics, call_recording.id, @@ -960,7 +990,10 @@ async def refresh_call_recording( status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found" ) - + + if call_recording.evaluator_result_id: + return {"message": "Call recording already processed"} + try: decrypted_api_key = decrypt_api_key(integration.api_key) except Exception as e: @@ -1119,8 +1152,13 @@ def _download_audio_from_payload(payload: Dict[str, Any]): if hasattr(provider, "retrieve_call_metrics"): refreshed_call_data = provider.retrieve_call_metrics(call_recording.provider_call_id) if isinstance(refreshed_call_data, dict) and refreshed_call_data: - call_data = refreshed_call_data - call_recording.call_data = refreshed_call_data + prev_data = ( + call_recording.call_data + if isinstance(call_recording.call_data, dict) + else {} + ) + call_data = merge_playground_call_data(prev_data, refreshed_call_data) + call_recording.call_data = call_data db.commit() logger.info(f"[Re-evaluate] Refreshed provider call data for call {call_recording.provider_call_id}") audio_bytes, resp = _download_audio_from_payload(call_data) diff --git a/app/api/v1/routes/prompt_partials.py b/app/api/v1/routes/prompt_partials.py index f65395c0..3380e7b9 100644 --- a/app/api/v1/routes/prompt_partials.py +++ b/app/api/v1/routes/prompt_partials.py @@ -2,12 +2,12 @@ Prompt Partials API Routes CRUD operations with version history for reusable prompt templates. """ -from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi import APIRouter, Depends, HTTPException, status, Query, BackgroundTasks from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any -from uuid import UUID +from uuid import UUID, uuid4 from pydantic import BaseModel from loguru import logger @@ -187,12 +187,15 @@ def _apply_prompt_partial_kind_filter( @router.post("/generate") async def generate_prompt_with_ai( data: GeneratePromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): """Generate a new prompt using AI from a description.""" from app.services.ai.llm_service import llm_service + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted if not data.description.strip(): raise HTTPException(400, "Description is required") @@ -225,6 +228,15 @@ async def generate_prompt_with_ai( task_defaults={"temperature": 0.7, "max_tokens": 4000}, credential_id=data.credential_id, ) + request_id = uuid4() + background_tasks.add_task( + record_prompt_partial_ai_assisted, + organization_id, + request_id, + workspace_id=workspace_id, + mode="generate", + model=model_str, + ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: logger.error(f"[PromptPartials] AI generation failed: {repr(e)}") @@ -234,12 +246,15 @@ async def generate_prompt_with_ai( @router.post("/improve") async def improve_prompt_with_ai( data: ImprovePromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): """Improve/reformat existing prompt content using AI.""" from app.services.ai.llm_service import llm_service + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted if not data.content.strip(): raise HTTPException(400, "Content is required") @@ -268,6 +283,15 @@ async def improve_prompt_with_ai( task_defaults={"temperature": 0.3, "max_tokens": 4000}, credential_id=data.credential_id, ) + request_id = uuid4() + background_tasks.add_task( + record_prompt_partial_ai_assisted, + organization_id, + request_id, + workspace_id=workspace_id, + mode="improve", + model=model_str, + ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: logger.error(f"[PromptPartials] AI improve failed: {repr(e)}") diff --git a/app/api/v1/routes/test_agents.py b/app/api/v1/routes/test_agents.py index f6b6ea1b..6ce1f7a9 100644 --- a/app/api/v1/routes/test_agents.py +++ b/app/api/v1/routes/test_agents.py @@ -16,10 +16,7 @@ TestAgentConversationUpdate, TestAgentConversationResponse ) -from app.services.billing.flexprice_service import ( - record_test_agent_conversation_ended, - record_test_agent_conversation_started, -) +from app.services.billing.flexprice_service import record_test_agent_conversation_ended from app.services.testing.test_agent_service import test_agent_service router = APIRouter(prefix="/test-agents", tags=["test-agents"]) @@ -102,12 +99,6 @@ async def start_conversation( organization_id=organization_id, db=db ) - background_tasks.add_task( - record_test_agent_conversation_started, - organization_id, - conversation_id, - workspace_id=workspace_id, - ) return conversation except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index e6d553b5..9657c763 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -5,7 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, Request, status from sqlalchemy.orm import Session -from uuid import UUID +from uuid import UUID, uuid4 from typing import Dict, Any, Optional, List from loguru import logger @@ -103,6 +103,7 @@ async def websocket_endpoint( agent_id = websocket.query_params.get("agent_id") persona_id = websocket.query_params.get("persona_id") scenario_id = websocket.query_params.get("scenario_id") + ui_surface = websocket.query_params.get("ui_surface") # Fetch agent and voice bundle once for routing and instructions agent = None @@ -478,8 +479,12 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: # Run the bot with the appropriate pipeline call_metadata = None + session_uuid: Optional[UUID] = None from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs + if agent_id and workspace_id and not test_agent_bridge_mode: + session_uuid = uuid4() + agent_silence_hangup_secs = resolve_agent_silence_hangup_secs(agent) try: if use_voice_bundle_pipeline: @@ -608,9 +613,16 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: # evaluator_id is optional - can be None if no persona/scenario if result_id and agent_id: try: - from app.models.database import EvaluatorResult, EvaluatorResultStatus + from app.models.database import ( + CallRecording, + CallRecordingSource, + EvaluatorResult, + EvaluatorResultStatus, + ) + from app.models.enums import CallRecordingStatus + from app.utils.call_recordings import generate_unique_call_short_id from app.workers.celery_app import process_evaluator_result_task - + # Determine name for the result if scenario_name and scenario_name != "Test Call": result_name = scenario_name @@ -618,9 +630,24 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: result_name = f"Test Call - {agent.name}" else: result_name = "Test Call" - + logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}") - + + call_short_id = generate_unique_call_short_id(db) + speaker_segments = call_metadata.get("speaker_segments") or [] + if not isinstance(speaker_segments, list): + speaker_segments = [] + playground_call_data = { + "source": "voice_bundle", + "result_id": result_id, + "transcript": call_metadata.get("transcription"), + "speaker_segments": speaker_segments, + "recording_s3_key": call_metadata.get("s3_key"), + "duration_seconds": call_metadata.get("duration"), + } + if ui_surface: + playground_call_data["ui_surface"] = ui_surface + # Create evaluator result with QUEUED status # persona_id and scenario_id can be None for test calls without persona/scenario evaluator_result = EvaluatorResult( @@ -636,11 +663,28 @@ def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: status=EvaluatorResultStatus.QUEUED.value, # Use .value to get the string audio_s3_key=call_metadata.get("s3_key"), transcription=call_metadata.get("transcription"), - speaker_segments=call_metadata.get("speaker_segments"), + speaker_segments=speaker_segments or None, ) db.add(evaluator_result) + db.flush() + + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=call_short_id, + status=CallRecordingStatus.UPDATED, + source=CallRecordingSource.PLAYGROUND, + call_data=playground_call_data, + provider_call_id=f"voice_bundle_{result_id}", + provider_platform="voice_bundle", + agent_id=UUID(agent_id), + evaluator_result_id=evaluator_result.id, + ) + db.add(call_recording) db.commit() db.refresh(evaluator_result) + + # Playground scoring bills via playground.evaluation_completed. logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}") @@ -814,6 +858,7 @@ def _extract_bearer(value: Optional[str]) -> Optional[str]: agent_id = request.query_params.get("agent_id") persona_id = request.query_params.get("persona_id") scenario_id = request.query_params.get("scenario_id") + ui_surface = request.query_params.get("ui_surface") # Determine which AI Provider to use based on agent configuration ai_provider = None @@ -959,6 +1004,7 @@ def check_provider(provider_enum): agent_id=agent_id, persona_id=persona_id, scenario_id=scenario_id, + ui_surface=ui_surface, fallback_host=request.headers.get("host", f"localhost:{settings.PORT}"), fallback_scheme=( request.headers.get("x-forwarded-proto") diff --git a/app/config.py b/app/config.py index 8242409c..95c98134 100644 --- a/app/config.py +++ b/app/config.py @@ -125,18 +125,43 @@ class Settings(BaseSettings): FRONTEND_DIR: str = "./frontend/dist" FRONTEND_BASE_URL: str = "" - # Content Security Policy (Report-Only by default; set CSP_REPORT_ONLY=false to enforce) + # Content Security Policy (enforcing by default; set CSP_REPORT_ONLY=true for local report-only mode) CSP_ENABLED: bool = True - CSP_REPORT_ONLY: bool = True + CSP_REPORT_ONLY: bool = False + # Browser voice SDKs (Vapi/Daily, Retell/LiveKit, ElevenLabs convai) and their telemetry. + _CSP_VOICE_CONNECT_SRC: str = ( + "https://api.vapi.ai " + "https://*.vapi.ai " + "https://*.daily.co " + "wss://*.daily.co " + "wss://*.livekit.cloud " + "https://api.elevenlabs.io " + "wss://api.elevenlabs.io " + "https://api.retellai.com " + "wss://api.retellai.com " + "https://*.ingest.sentry.io " + "https://*.ingest.us.sentry.io" + ) + _CSP_FRAME_SRC: str = ( + "https://*.daily.co " + "https://*.s3.amazonaws.com " + "https://*.amazonaws.com " + "https://*.cloudfront.net " + "https://storage.googleapis.com " + "https://*.blob.core.windows.net" + ) + # Vapi → Daily.co call-machine bundle requires eval + blob worklets for audio + _CSP_DAILY_SCRIPT_SRC: str = "'unsafe-eval' blob: https://c.daily.co https://*.daily.co" CSP_POLICY: str = ( "default-src 'self'; " - "script-src 'self'; " + f"script-src 'self' {_CSP_DAILY_SCRIPT_SRC}; " "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " "font-src 'self' https://fonts.gstatic.com; " "img-src 'self' data: blob: https:; " - "connect-src 'self' wss: ws:; " + f"connect-src 'self' wss: ws: {_CSP_VOICE_CONNECT_SRC}; " "media-src 'self' blob: https:; " - "frame-src 'self' blob:; " + f"frame-src 'self' blob: {_CSP_FRAME_SRC}; " + "worker-src 'self' blob:; " "object-src 'none'; " "base-uri 'self'; " "form-action 'self'; " @@ -225,6 +250,10 @@ class Settings(BaseSettings): FLEXPRICE_ENABLED: bool = False FLEXPRICE_API_KEY: Optional[str] = None FLEXPRICE_API_HOST: str = "https://us.api.flexprice.io/v1" + FLEXPRICE_AUTO_SUBSCRIBE: bool = False + FLEXPRICE_DEFAULT_PLAN_ID: Optional[str] = None + FLEXPRICE_DEFAULT_CURRENCY: str = "usd" + FLEXPRICE_DEFAULT_BILLING_PERIOD: str = "MONTHLY" # LLM gateway (optional platform-wide proxy for batch LLM calls). LLM_GATEWAY_ENABLED: bool = False LLM_GATEWAY_TYPE: str = "bifrost" # bifrost | litellm_proxy @@ -859,6 +888,15 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None if "trusted_ips" in operational_config: settings.OPERATIONAL_TRUSTED_IPS = operational_config["trusted_ips"] + if "security" in config_data: + security_config = config_data["security"] + if "csp_enabled" in security_config: + settings.CSP_ENABLED = bool(security_config["csp_enabled"]) + if "csp_report_only" in security_config: + settings.CSP_REPORT_ONLY = bool(security_config["csp_report_only"]) + if security_config.get("csp_policy"): + settings.CSP_POLICY = security_config["csp_policy"] + if "flexprice" in config_data: flexprice_config = config_data["flexprice"] if "enabled" in flexprice_config: @@ -867,6 +905,19 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None settings.FLEXPRICE_API_KEY = flexprice_config["api_key"] if flexprice_config.get("api_host"): settings.FLEXPRICE_API_HOST = flexprice_config["api_host"] + if "auto_subscribe" in flexprice_config: + settings.FLEXPRICE_AUTO_SUBSCRIBE = bool(flexprice_config["auto_subscribe"]) + if flexprice_config.get("default_plan_id"): + settings.FLEXPRICE_DEFAULT_PLAN_ID = flexprice_config["default_plan_id"] + if flexprice_config.get("default_currency"): + settings.FLEXPRICE_DEFAULT_CURRENCY = flexprice_config["default_currency"] + if flexprice_config.get("default_billing_period"): + settings.FLEXPRICE_DEFAULT_BILLING_PERIOD = flexprice_config["default_billing_period"] + if ( + os.environ.get("EFFICIENTAI_PYTEST") == "1" + and os.environ.get("FLEXPRICE_TEST_ALLOW") != "1" + ): + settings.FLEXPRICE_ENABLED = False # Update Celery URLs if they weren't explicitly set if not settings.CELERY_BROKER_URL: diff --git a/app/core/auth/platform_admin.py b/app/core/auth/platform_admin.py index 920c0164..6e468385 100644 --- a/app/core/auth/platform_admin.py +++ b/app/core/auth/platform_admin.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session from app.config import settings +from app.core.auth.token_revocation import is_access_jti_revoked, revoke_access_jti from app.database import get_db from app.models.database import PlatformAdmin @@ -58,6 +59,18 @@ def decode_platform_access_token(token: str) -> Dict[str, Any]: ) +def revoke_platform_access_token(token: str) -> None: + try: + claims = decode_platform_access_token(token) + jti = claims.get("jti") + exp = claims.get("exp") + if jti and exp: + ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) + revoke_access_jti(jti, ttl) + except JWTError: + pass + + def _extract_bearer(authorization: Optional[str]) -> Optional[str]: if not authorization: return None @@ -107,6 +120,13 @@ def get_platform_admin( detail="Invalid platform admin token scope.", ) + jti = claims.get("jti") + if jti and is_access_jti_revoked(jti): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token has been revoked.", + ) + try: admin_id = UUID(claims["sub"]) except (KeyError, ValueError) as exc: diff --git a/app/core/operational_access_middleware.py b/app/core/operational_access_middleware.py index 7bf4d8d3..7a0db428 100644 --- a/app/core/operational_access_middleware.py +++ b/app/core/operational_access_middleware.py @@ -1,4 +1,8 @@ -"""Restrict /metrics from the public internet (/health stays open for load balancers).""" +"""Restrict /metrics from the public internet (/health stays open for load balancers). + +Not Spring Boot Actuator: this FastAPI app exposes /health (LB probes) and /metrics +(Prometheus scrape). /metrics is gated by trusted IPs or OPERATIONAL_PUBLIC only. +""" from __future__ import annotations @@ -6,14 +10,11 @@ import logging from typing import Iterable -from fastapi import HTTPException from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import JSONResponse, Response from app.config import settings -from app.core.auth.dependency import _resolve -from app.database import SessionLocal logger = logging.getLogger(__name__) @@ -75,22 +76,6 @@ def _resolved_trusted_ip(request: Request) -> str | None: return hops[-1] -def _has_authenticated_caller(request: Request) -> bool: - db = SessionLocal() - try: - principal = _resolve( - request.headers.get("authorization"), - request.headers.get("x-api-key"), - request.headers.get("x-efficientai-api-key"), - db, - ) - return principal is not None - except HTTPException: - return False - finally: - db.close() - - def is_operational_access_allowed(request: Request) -> bool: """Return True when the caller may access a protected operational endpoint.""" if settings.OPERATIONAL_PUBLIC: @@ -100,9 +85,6 @@ def is_operational_access_allowed(request: Request) -> bool: if resolved_ip and _ip_in_trusted(resolved_ip, settings.OPERATIONAL_TRUSTED_IPS): return True - if _has_authenticated_caller(request): - return True - return False diff --git a/app/services/billing/flexprice_service.py b/app/services/billing/flexprice_service.py index 80cb9bdd..a9471f56 100644 --- a/app/services/billing/flexprice_service.py +++ b/app/services/billing/flexprice_service.py @@ -4,6 +4,41 @@ Every event uses ``external_customer_id=str(organization.id)`` and a stable ``event_id`` for idempotency. ``properties`` should include ``workspace_id`` and ``feature`` (license key) when the surface is gated. + +Ingest **only when value is delivered** (completed), never on ``*_started`` / +``*_created`` / ``*_requested``. Event ``properties`` include billable fields +(``workspace_id``, ``feature``, ``quantity``, ``billable_minutes``) plus audit +IDs (``evaluation_id``, ``audio_seconds``, ``ui_surface``, etc.) for support — +audit fields are not used for Flexprice SUM/COUNT aggregation. Set ``ui_surface`` +only when multiple UI paths share one event (e.g. ``agents_talk`` vs +``agent_playground``); never wire meters to it. + +Billable events (wire plan usage charges to these meters only): + +- call_imports: ``call_import.batch_created`` (``quantity`` = rows imported), + ``call_import.evaluation_completed`` (``quantity`` = newly completed rows), + ``call_import.recording_minutes_billed`` (``quantity`` = ``billable_minutes``), + ``call_import.pdf_report_generated`` (``quantity`` = 1) +- agent_playground: ``playground.evaluation_completed`` (``quantity`` = ``billable_minutes`` + from call duration) **or** ``test_agent.conversation_ended`` (same minute rollup for + standalone test-agent sessions without playground eval) — never both for the same session +- voice_playground: ``tts.sample_synthesized``, ``tts.report_completed``, + ``blind_test.response_submitted`` +- evaluators: ``evaluator.run_completed`` (``quantity`` = 1) and + ``evaluator.recording_minutes_billed`` when the run has audio (``billable_minutes``) +- gepa_optimization: ``prompt_optimization.run_completed`` (``quantity`` = candidates) +- judge_alignment: ``judge_alignment.run_completed`` (``quantity`` = samples scored) +- metrics_ai_assist: ``metrics.ai_assist`` +- metric_studio: ``metric_studio.run_completed`` (``quantity`` = completed items) +- scenario_ai: ``scenario.ai_text_generated`` +- prompt_partials: ``prompt_partial.ai_assisted`` (``mode``: generate | improve | flowchart | flowchart_map) +- call_imports (add-ons): ``call_import.user_insights_generated``, + ``call_import.prompt_improvements_generated`` +- agent_playground (AI helpers): ``persona.prompt_generated``, ``agent.test_setup_generated`` + +Not ingested: ``*_started``, ``*_requested``, ``*_created``, ``observability.*``, +``playground.call_evaluated``, ``test_agent.conversation_started``, +``metric_studio.item_evaluated``, etc. """ from __future__ import annotations @@ -18,8 +53,19 @@ EVENT_SOURCE = "efficientai" FEATURE_CALL_IMPORTS = "call_imports" +FEATURE_AGENT_PLAYGROUND = "agent_playground" FEATURE_VOICE_PLAYGROUND = "voice_playground" FEATURE_GEPA = "gepa_optimization" +FEATURE_EVALUATORS = "evaluators" +FEATURE_JUDGE_ALIGNMENT = "judge_alignment" +FEATURE_METRICS_AI_ASSIST = "metrics_ai_assist" +FEATURE_METRIC_STUDIO = "metric_studio" +FEATURE_SCENARIO_AI = "scenario_ai" +FEATURE_PROMPT_PARTIALS = "prompt_partials" + +# Audit-only ui_surface values (never wire Flexprice meters to these). +UI_SURFACE_AGENTS_TALK = "agents_talk" +UI_SURFACE_AGENT_PLAYGROUND = "agent_playground" # Log once when metering is inactive so AWS/worker misconfig is obvious. _disabled_skip_logged = False @@ -32,16 +78,18 @@ TTS_REPORT_REQUESTED = "tts.report_requested" TTS_REPORT_COMPLETED = "tts.report_completed" CALL_IMPORT_BATCH_CREATED = "call_import.batch_created" -CALL_IMPORT_ROW_IMPORTED = "call_import.row_imported" CALL_IMPORT_EVALUATION_STARTED = "call_import.evaluation_started" CALL_IMPORT_EVALUATION_COMPLETED = "call_import.evaluation_completed" -CALL_IMPORT_EVALUATION_ROW_COMPLETED = "call_import.evaluation_row_completed" +CALL_IMPORT_RECORDING_MINUTES_BILLED = "call_import.recording_minutes_billed" +CALL_IMPORT_AUDIO_MINUTES_BILLED = CALL_IMPORT_RECORDING_MINUTES_BILLED +CALL_IMPORT_PDF_REPORT_GENERATED = "call_import.pdf_report_generated" PLAYGROUND_WEB_CALL_STARTED = "playground.web_call_started" PLAYGROUND_WEBSOCKET_SESSION_STARTED = "playground.websocket_session_started" PLAYGROUND_CALL_EVALUATED = "playground.call_evaluated" PLAYGROUND_EVALUATION_COMPLETED = "playground.evaluation_completed" EVALUATOR_RUN_REQUESTED = "evaluator.run_requested" EVALUATOR_RUN_COMPLETED = "evaluator.run_completed" +EVALUATOR_RECORDING_MINUTES_BILLED = "evaluator.recording_minutes_billed" EVALUATION_CREATED = "evaluation.created" EVALUATION_COMPLETED = "evaluation.completed" PROMPT_OPTIMIZATION_RUN_STARTED = "prompt_optimization.run_started" @@ -52,8 +100,28 @@ OBSERVABILITY_CALL_EVALUATED = "observability.call_evaluated" TEST_AGENT_CONVERSATION_STARTED = "test_agent.conversation_started" TEST_AGENT_CONVERSATION_ENDED = "test_agent.conversation_ended" -METRICS_LLM_ASSIST = "metrics.llm_assist" -CHAT_COMPLETION = "chat.completion" +METRICS_AI_ASSIST = "metrics.ai_assist" +METRIC_STUDIO_ITEM_EVALUATED = "metric_studio.item_evaluated" +METRIC_STUDIO_RUN_COMPLETED = "metric_studio.run_completed" +SCENARIO_AI_TEXT_GENERATED = "scenario.ai_text_generated" +PROMPT_PARTIAL_AI_ASSISTED = "prompt_partial.ai_assisted" +CALL_IMPORT_USER_INSIGHTS_GENERATED = "call_import.user_insights_generated" +CALL_IMPORT_PROMPT_IMPROVEMENTS_GENERATED = "call_import.prompt_improvements_generated" +PERSONA_PROMPT_GENERATED = "persona.prompt_generated" +AGENT_TEST_SETUP_GENERATED = "agent.test_setup_generated" +# Deprecated meters — never ingest (bill on completion events instead). +DEPRECATED_EVENT_NAMES = frozenset( + { + PLAYGROUND_CALL_EVALUATED, + PLAYGROUND_WEB_CALL_STARTED, + PLAYGROUND_WEBSOCKET_SESSION_STARTED, + TEST_AGENT_CONVERSATION_STARTED, + OBSERVABILITY_CALL_EVALUATED, + } +) +# Legacy aliases (Flexprice meters may still exist under old names) +METRICS_LLM_ASSIST = METRICS_AI_ASSIST +CHAT_COMPLETION = SCENARIO_AI_TEXT_GENERATED def _verbose_logging() -> bool: @@ -61,6 +129,14 @@ def _verbose_logging() -> bool: return os.getenv("FLEXPRICE_VERBOSE", "").lower() in {"1", "true", "yes"} +def _pytest_blocks_external_billing() -> bool: + """Block real Flexprice I/O during pytest unless explicitly opted in.""" + return ( + os.environ.get("EFFICIENTAI_PYTEST") == "1" + and os.environ.get("FLEXPRICE_TEST_ALLOW") != "1" + ) + + def _mask_api_key(api_key: Optional[str]) -> str: if not api_key: return "(missing)" @@ -121,6 +197,8 @@ def log_startup_status(*, component: str = "app") -> None: def _verify_connectivity() -> Optional[str]: """Best-effort reachability probe; returns error text or None when OK.""" + if _pytest_blocks_external_billing(): + return None try: import httpx @@ -176,6 +254,64 @@ def _coerce_properties(properties: Optional[dict[str, Any]]) -> dict[str, str]: return out +def _billable_minutes(duration_seconds: Optional[float]) -> int: + if duration_seconds is None: + return 1 + seconds = float(duration_seconds) + if seconds <= 0: + return 1 + import math + + return max(1, int(math.ceil(seconds / 60.0))) + + +def _billing_properties( + workspace_id: UUID, + feature: str, + *, + quantity: Optional[Union[int, float]] = None, + billable_minutes: Optional[int] = None, +) -> dict[str, Any]: + props: dict[str, Any] = {"workspace_id": workspace_id, "feature": feature} + if quantity is not None: + props["quantity"] = quantity + if billable_minutes is not None: + props["billable_minutes"] = billable_minutes + return props + + +def _event_properties( + workspace_id: UUID, + feature: str, + *, + quantity: Optional[Union[int, float]] = None, + billable_minutes: Optional[int] = None, + ui_surface: Optional[str] = None, + **audit: Any, +) -> dict[str, Any]: + """Billable fields plus optional audit metadata for support traceability.""" + props = _billing_properties( + workspace_id, + feature, + quantity=quantity, + billable_minutes=billable_minutes, + ) + surface = (str(ui_surface).strip() if ui_surface is not None else "") or None + if not surface and isinstance(audit.get("ui_surface"), str): + surface = audit["ui_surface"].strip() or None + if surface: + props["ui_surface"] = surface + for key, value in audit.items(): + if key == "ui_surface": + continue + if value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + props[key] = value + return props + + def record_event( event_name: str, organization_id: UUID, @@ -190,6 +326,9 @@ def record_event( """ global _disabled_skip_logged + if _pytest_blocks_external_billing(): + return False + inactive_reason = disabled_reason() if inactive_reason: if not _disabled_skip_logged: @@ -208,6 +347,16 @@ def record_event( ) return False + if event_name in DEPRECATED_EVENT_NAMES: + if _verbose_logging(): + logger.info( + "Flexprice SKIP deprecated {} org={} event_id={}", + event_name, + organization_id, + event_id, + ) + return False + coerced = _coerce_properties(properties) quantity = coerced.get("quantity") @@ -254,6 +403,9 @@ def ensure_customer( email: Optional[str] = None, ) -> None: """Register an organization as a Flexprice customer. No-op when disabled.""" + if _pytest_blocks_external_billing(): + return + inactive_reason = disabled_reason() if inactive_reason: if _verbose_logging(): @@ -296,6 +448,89 @@ def ensure_customer( ) +def _subscription_inactive_reason() -> Optional[str]: + """Why auto-subscribe is off, or None when ensure_subscription may run.""" + inactive = disabled_reason() + if inactive: + return inactive + if not settings.FLEXPRICE_AUTO_SUBSCRIBE: + return "flexprice.auto_subscribe is false (or FLEXPRICE_AUTO_SUBSCRIBE unset)" + if not (settings.FLEXPRICE_DEFAULT_PLAN_ID or "").strip(): + return "flexprice.default_plan_id is unset (or FLEXPRICE_DEFAULT_PLAN_ID env missing)" + return None + + +def _has_active_subscription(client, *, organization_id: UUID, plan_id: str) -> bool: + response = client.subscriptions.query_subscription( + external_customer_id=str(organization_id), + plan_id=plan_id, + limit=1, + ) + items = getattr(response, "items", None) or [] + return len(items) > 0 + + +def ensure_subscription(organization_id: UUID) -> None: + """Assign the default SaaS plan when auto-subscribe is enabled. Never raises.""" + if _pytest_blocks_external_billing(): + return + + inactive_reason = _subscription_inactive_reason() + if inactive_reason: + if _verbose_logging(): + logger.info( + "Flexprice SKIP ensure_subscription org={} ({})", + organization_id, + inactive_reason, + ) + return + + plan_id = settings.FLEXPRICE_DEFAULT_PLAN_ID.strip() + try: + from flexprice import Flexprice + + with Flexprice( + server_url=settings.FLEXPRICE_API_HOST, + api_key_auth=settings.FLEXPRICE_API_KEY, + ) as client: + if _has_active_subscription(client, organization_id=organization_id, plan_id=plan_id): + logger.debug( + "Flexprice ensure_subscription already active org={} plan_id={}", + organization_id, + plan_id, + ) + return + + created = client.subscriptions.create_subscription( + billing_period=settings.FLEXPRICE_DEFAULT_BILLING_PERIOD, + currency=settings.FLEXPRICE_DEFAULT_CURRENCY, + plan_id=plan_id, + external_customer_id=str(organization_id), + subscription_status="active", + ) + logger.info( + "Flexprice ensure_subscription ok org={} plan_id={} subscription_id={}", + organization_id, + plan_id, + getattr(created, "id", None), + ) + except Exception as exc: + if _is_customer_already_exists(exc) or "already exist" in str(exc).lower(): + logger.debug( + "Flexprice ensure_subscription already exists org={} plan_id={}", + organization_id, + plan_id, + ) + return + logger.warning( + "Flexprice ensure_subscription FAILED org={} plan_id={} host={} error={}", + organization_id, + plan_id, + settings.FLEXPRICE_API_HOST, + exc, + ) + + # --- Voice playground --- @@ -306,17 +541,7 @@ def record_blind_test_share_created( workspace_id: UUID, comparison_id: UUID, ) -> None: - record_event( - BLIND_TEST_SHARE_CREATED, - organization_id, - share_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "share_id": share_id, - "comparison_id": comparison_id, - }, - ) + """Not ingested — bill on blind_test.response_submitted instead.""" def record_blind_test_response_submitted( @@ -331,13 +556,13 @@ def record_blind_test_response_submitted( BLIND_TEST_RESPONSE_SUBMITTED, organization_id, response_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "share_id": share_id, - "response_count": response_count, - "quantity": response_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=max(1, response_count), + share_id=share_id, + response_id=response_id, + ), ) @@ -348,17 +573,7 @@ def record_tts_generation_started( workspace_id: UUID, sample_count: int, ) -> None: - record_event( - TTS_GENERATION_STARTED, - organization_id, - comparison_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "sample_count": sample_count, - }, - ) + """Not ingested — bill on tts.sample_synthesized per completed sample.""" def record_tts_sample_synthesized( @@ -375,16 +590,16 @@ def record_tts_sample_synthesized( TTS_SAMPLE_SYNTHESIZED, organization_id, sample_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "sample_id": sample_id, - "provider": provider, - "side": side, - "duration_seconds": duration_seconds, - "quantity": 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=1, + comparison_id=comparison_id, + sample_id=sample_id, + provider=provider, + side=side, + duration_seconds=duration_seconds, + ), ) @@ -395,17 +610,7 @@ def record_tts_report_requested( workspace_id: UUID, comparison_id: UUID, ) -> None: - record_event( - TTS_REPORT_REQUESTED, - organization_id, - report_job_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "report_job_id": report_job_id, - }, - ) + """Not ingested — bill on tts.report_completed.""" def record_tts_report_completed( @@ -419,16 +624,17 @@ def record_tts_report_completed( TTS_REPORT_COMPLETED, organization_id, report_job_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "report_job_id": report_job_id, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=1, + comparison_id=comparison_id, + report_job_id=report_job_id, + ), ) -# --- Call imports --- +# --- Call imports (tracking complete: batch, eval lifecycle, audio minutes, PDF) --- def record_call_import_batch_created( @@ -444,21 +650,33 @@ def record_call_import_batch_created( CALL_IMPORT_BATCH_CREATED, organization_id, call_import_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_CALL_IMPORTS, - "call_import_id": call_import_id, - "total_rows": total_rows, - "quantity": total_rows, - "source": source, - "provider": provider, - }, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=max(0, total_rows), + call_import_id=call_import_id, + source=source, + provider=provider, + total_rows=total_rows, + ), ) # --- Call imports (evaluations) --- +def record_call_import_evaluation_started( + organization_id: UUID, + evaluation_id: UUID, + *, + workspace_id: UUID, + call_import_id: UUID, + total_rows: int, + metric_count: int = 0, +) -> None: + """Not ingested — bill on call_import.evaluation_completed.""" + + def record_call_import_evaluation_completed( organization_id: UUID, evaluation_id: UUID, @@ -467,23 +685,103 @@ def record_call_import_evaluation_completed( call_import_id: UUID, rows_billed: int, completed_total: int, + total_rows: int = 0, metric_count: int = 0, ) -> bool: - """Bill one pass of an evaluation run for newly completed rows.""" + """Bill one finished evaluation pass for newly completed rows (not per row).""" + if rows_billed <= 0: + return False return record_event( CALL_IMPORT_EVALUATION_COMPLETED, organization_id, f"{evaluation_id}:{completed_total}", - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_CALL_IMPORTS, - "call_import_id": call_import_id, - "evaluation_id": evaluation_id, - "rows_billed": rows_billed, - "completed_total": completed_total, - "metric_count": metric_count, - "quantity": rows_billed, - }, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=rows_billed, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + completed_total=completed_total, + total_rows=total_rows, + metric_count=metric_count, + rows_billed=rows_billed, + ), + ) + + +def record_call_import_recording_minutes_billed( + organization_id: UUID, + evaluation_row_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, +) -> bool: + """Bill recording duration for one successfully evaluated call-import row.""" + if billable_minutes <= 0: + return False + return record_event( + CALL_IMPORT_RECORDING_MINUTES_BILLED, + organization_id, + evaluation_row_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=billable_minutes, + billable_minutes=billable_minutes, + evaluation_row_id=evaluation_row_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + ), + ) + + +def record_call_import_audio_minutes_billed( + organization_id: UUID, + evaluation_row_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, +) -> bool: + return record_call_import_recording_minutes_billed( + organization_id, + evaluation_row_id, + workspace_id=workspace_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + + +def record_call_import_pdf_report_generated( + organization_id: UUID, + pdf_report_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + report_type: str, +) -> None: + record_event( + CALL_IMPORT_PDF_REPORT_GENERATED, + organization_id, + pdf_report_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + pdf_report_id=pdf_report_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + report_type=report_type, + ), ) @@ -497,16 +795,7 @@ def record_playground_web_call_started( workspace_id: UUID, agent_id: UUID, ) -> None: - record_event( - PLAYGROUND_WEB_CALL_STARTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "agent_id": agent_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_websocket_session_started( @@ -515,15 +804,7 @@ def record_playground_websocket_session_started( *, workspace_id: UUID, ) -> None: - record_event( - PLAYGROUND_WEBSOCKET_SESSION_STARTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_call_evaluated( @@ -535,18 +816,7 @@ def record_playground_call_evaluated( call_short_id: str, metric_count: int, ) -> None: - record_event( - PLAYGROUND_CALL_EVALUATED, - organization_id, - evaluation_attempt_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "evaluator_result_id": evaluator_result_id, - "evaluation_attempt_id": evaluation_attempt_id, - "metric_count": metric_count, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_evaluation_completed( @@ -558,19 +828,25 @@ def record_playground_evaluation_completed( call_short_id: str, duration_seconds: Optional[float] = None, metric_count: int = 0, + ui_surface: Optional[str] = None, ) -> None: + """Bill playground voice/web call scoring. Pass ``ui_surface`` when known.""" + minutes = _billable_minutes(duration_seconds) record_event( PLAYGROUND_EVALUATION_COMPLETED, organization_id, evaluation_attempt_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "evaluator_result_id": evaluator_result_id, - "evaluation_attempt_id": evaluation_attempt_id, - "duration_seconds": duration_seconds, - "metric_count": metric_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=minutes, + billable_minutes=minutes, + evaluator_result_id=evaluator_result_id, + call_short_id=call_short_id, + duration_seconds=duration_seconds, + metric_count=metric_count, + ui_surface=ui_surface, + ), ) @@ -584,15 +860,8 @@ def record_evaluator_run_requested( workspace_id: UUID, quantity: int, ) -> None: - record_event( - EVALUATOR_RUN_REQUESTED, - organization_id, - request_id, - properties={ - "workspace_id": workspace_id, - "quantity": quantity, - }, - ) + """Not ingested — bill on evaluator.run_completed when scoring finishes.""" + del organization_id, request_id, workspace_id, quantity def record_evaluator_run_completed( @@ -600,19 +869,52 @@ def record_evaluator_run_completed( result_id: str, *, workspace_id: UUID, - evaluator_id: UUID, + evaluator_id: Optional[UUID] = None, + evaluator_result_id: Optional[UUID] = None, call_count: int = 1, ) -> None: + del call_count record_event( EVALUATOR_RUN_COMPLETED, organization_id, result_id, - properties={ - "workspace_id": workspace_id, - "evaluator_id": evaluator_id, - "result_id": result_id, - "call_count": call_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=1, + result_id=result_id, + evaluator_id=evaluator_id, + evaluator_result_id=evaluator_result_id, + ), + ) + + +def record_evaluator_recording_minutes_billed( + organization_id: UUID, + evaluator_result_id: UUID, + *, + workspace_id: UUID, + duration_seconds: Optional[float] = None, +) -> bool: + """Bill audio duration for a completed evaluator run that includes a recording.""" + minutes = _billable_minutes(duration_seconds) + if minutes <= 0: + return False + return record_event( + EVALUATOR_RECORDING_MINUTES_BILLED, + organization_id, + evaluator_result_id, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=minutes, + billable_minutes=minutes, + evaluator_result_id=evaluator_result_id, + duration_seconds=duration_seconds, + audio_seconds=int(round(float(duration_seconds))) + if duration_seconds is not None + else None, + ), ) @@ -627,16 +929,8 @@ def record_evaluation_created( audio_id: UUID, metrics_requested: int, ) -> None: - record_event( - EVALUATION_CREATED, - organization_id, - evaluation_id, - properties={ - "workspace_id": workspace_id, - "audio_id": audio_id, - "metrics_requested": metrics_requested, - }, - ) + """Not ingested — bill on evaluation.completed.""" + del organization_id, evaluation_id, workspace_id, audio_id, metrics_requested def record_evaluation_completed( @@ -649,7 +943,12 @@ def record_evaluation_completed( EVALUATION_COMPLETED, organization_id, evaluation_id, - properties={"workspace_id": workspace_id}, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=1, + evaluation_id=evaluation_id, + ), ) @@ -664,18 +963,8 @@ def record_prompt_optimization_run_started( agent_id: UUID, max_metric_calls: Optional[int] = None, ) -> None: - record_event( - PROMPT_OPTIMIZATION_RUN_STARTED, - organization_id, - run_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_GEPA, - "run_id": run_id, - "agent_id": agent_id, - "max_metric_calls": max_metric_calls, - }, - ) + """Not ingested — bill on prompt_optimization.run_completed.""" + del organization_id, run_id, workspace_id, agent_id, max_metric_calls def record_prompt_optimization_run_completed( @@ -686,17 +975,19 @@ def record_prompt_optimization_run_completed( agent_id: UUID, candidates_count: int = 0, ) -> None: + billed = max(1, candidates_count) record_event( PROMPT_OPTIMIZATION_RUN_COMPLETED, organization_id, run_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_GEPA, - "run_id": run_id, - "agent_id": agent_id, - "candidates_count": candidates_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_GEPA, + quantity=billed, + run_id=run_id, + agent_id=agent_id, + candidates_count=candidates_count, + ), ) @@ -711,17 +1002,8 @@ def record_judge_alignment_run_started( dataset_id: UUID, sample_count: int, ) -> None: - record_event( - JUDGE_ALIGNMENT_RUN_STARTED, - organization_id, - run_id, - properties={ - "workspace_id": workspace_id, - "run_id": run_id, - "dataset_id": dataset_id, - "sample_count": sample_count, - }, - ) + """Not ingested — bill on judge_alignment.run_completed.""" + del organization_id, run_id, workspace_id, dataset_id, sample_count def record_judge_alignment_run_completed( @@ -732,16 +1014,21 @@ def record_judge_alignment_run_completed( dataset_id: UUID, samples_scored: int, ) -> None: + billed = max(0, samples_scored) + if billed <= 0: + return record_event( JUDGE_ALIGNMENT_RUN_COMPLETED, organization_id, run_id, - properties={ - "workspace_id": workspace_id, - "run_id": run_id, - "dataset_id": dataset_id, - "samples_scored": samples_scored, - }, + properties=_event_properties( + workspace_id, + FEATURE_JUDGE_ALIGNMENT, + quantity=billed, + run_id=run_id, + dataset_id=dataset_id, + samples_scored=samples_scored, + ), ) @@ -755,16 +1042,8 @@ def record_observability_call_ingested( workspace_id: UUID, provider: Optional[str] = None, ) -> None: - record_event( - OBSERVABILITY_CALL_INGESTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "provider": provider, - }, - ) + """Not ingested — bill on observability.call_evaluated.""" + del organization_id, call_short_id, workspace_id, provider def record_observability_call_evaluated( @@ -773,15 +1052,8 @@ def record_observability_call_evaluated( *, workspace_id: UUID, ) -> None: - record_event( - OBSERVABILITY_CALL_EVALUATED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — observability is not a billable product surface.""" + del organization_id, call_short_id, workspace_id # --- Test agents --- @@ -789,44 +1061,72 @@ def record_observability_call_evaluated( def record_test_agent_conversation_started( organization_id: UUID, - conversation_id: UUID, + conversation_id: Union[str, UUID], *, workspace_id: UUID, + result_id: Optional[str] = None, + agent_id: Optional[UUID] = None, + call_short_id: Optional[str] = None, ) -> None: - record_event( - TEST_AGENT_CONVERSATION_STARTED, - organization_id, - conversation_id, - properties={ - "workspace_id": workspace_id, - "conversation_id": conversation_id, - }, - ) + """Not ingested — bill on test_agent.conversation_ended.""" + del organization_id, conversation_id, workspace_id, result_id, agent_id, call_short_id def record_test_agent_conversation_ended( organization_id: UUID, - conversation_id: UUID, + conversation_id: Union[str, UUID], *, workspace_id: UUID, duration_seconds: Optional[float] = None, turn_count: int = 0, + result_id: Optional[str] = None, + agent_id: Optional[UUID] = None, + call_short_id: Optional[str] = None, ) -> None: + minutes = _billable_minutes(duration_seconds) record_event( TEST_AGENT_CONVERSATION_ENDED, organization_id, conversation_id, - properties={ - "workspace_id": workspace_id, - "conversation_id": conversation_id, - "duration_seconds": duration_seconds, - "turn_count": turn_count, - "quantity": duration_seconds or 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=minutes, + billable_minutes=minutes, + conversation_id=conversation_id, + duration_seconds=duration_seconds, + turn_count=turn_count, + result_id=result_id, + agent_id=agent_id, + call_short_id=call_short_id, + ), ) -# --- LLM assist --- +# --- Metrics AI assist (metric builder) --- + + +def record_metrics_ai_assist( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + mode: str, +) -> None: + if workspace_id is None: + return + record_event( + METRICS_AI_ASSIST, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_METRICS_AI_ASSIST, + quantity=1, + request_id=request_id, + mode=mode, + ), + ) def record_metrics_llm_assist( @@ -836,14 +1136,96 @@ def record_metrics_llm_assist( workspace_id: Optional[UUID], mode: str, ) -> None: + record_metrics_ai_assist( + organization_id, + request_id, + workspace_id=workspace_id, + mode=mode, + ) + + +# --- Metric Studio --- + + +def record_metric_studio_item_evaluated( + organization_id: UUID, + result_row_id: UUID, + *, + workspace_id: UUID, + run_id: UUID, + source_kind: str, + source_ref: str, + metric_count: int = 0, +) -> None: + """Not ingested — bill on metric_studio.run_completed.""" + del ( + organization_id, + result_row_id, + workspace_id, + run_id, + source_kind, + source_ref, + metric_count, + ) + + +def record_metric_studio_run_completed( + organization_id: UUID, + run_id: UUID, + *, + workspace_id: UUID, + run_status: str, + total_items: int, + completed_items: int, + failed_items: int, +) -> None: + del run_status + billed = max(0, completed_items) + if billed <= 0: + return record_event( - METRICS_LLM_ASSIST, + METRIC_STUDIO_RUN_COMPLETED, + organization_id, + run_id, + properties=_event_properties( + workspace_id, + FEATURE_METRIC_STUDIO, + quantity=billed, + run_id=run_id, + total_items=total_items, + completed_items=completed_items, + failed_items=failed_items, + ), + ) + + +# --- Scenario / assistant AI text --- + + +def record_scenario_ai_text_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + model: Optional[str] = None, + purpose: str = "scenario_description", + scenario_count: Optional[int] = None, +) -> None: + if workspace_id is None: + return + record_event( + SCENARIO_AI_TEXT_GENERATED, organization_id, request_id, - properties={ - "workspace_id": workspace_id, - "mode": mode, - }, + properties=_event_properties( + workspace_id, + FEATURE_SCENARIO_AI, + quantity=1, + request_id=request_id, + model=model, + purpose=purpose, + scenario_count=scenario_count, + ), ) @@ -853,14 +1235,152 @@ def record_chat_completion( *, workspace_id: Optional[UUID], model: Optional[str] = None, + purpose: str = "scenario_description", + scenario_count: Optional[int] = None, +) -> None: + record_scenario_ai_text_generated( + organization_id, + request_id, + workspace_id=workspace_id, + model=model, + purpose=purpose, + scenario_count=scenario_count, + ) + + +# --- Prompt partials AI assist --- + + +def record_prompt_partial_ai_assisted( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + mode: str, + partial_id: Optional[UUID] = None, + model: Optional[str] = None, +) -> None: + if workspace_id is None: + return + record_event( + PROMPT_PARTIAL_AI_ASSISTED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_PROMPT_PARTIALS, + quantity=1, + request_id=request_id, + mode=mode, + partial_id=partial_id, + model=model, + ), + ) + + +# --- Call import AI add-ons --- + + +def record_call_import_user_insights_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + evaluation_id: UUID, +) -> None: + if workspace_id is None: + return + record_event( + CALL_IMPORT_USER_INSIGHTS_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + request_id=request_id, + evaluation_id=evaluation_id, + ), + ) + + +def record_call_import_prompt_improvements_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + evaluation_id: UUID, + imported_agent_id: Optional[UUID] = None, +) -> None: + if workspace_id is None: + return + record_event( + CALL_IMPORT_PROMPT_IMPROVEMENTS_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + request_id=request_id, + evaluation_id=evaluation_id, + imported_agent_id=imported_agent_id, + ), + ) + + +# --- Agent / persona AI helpers --- + + +def record_persona_prompt_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + agent_id: UUID, + model: Optional[str] = None, + source: Optional[str] = None, ) -> None: + if workspace_id is None: + return + record_event( + PERSONA_PROMPT_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=1, + request_id=request_id, + agent_id=agent_id, + model=model, + source=source, + ), + ) + + +def record_agent_test_setup_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + purpose: str, + model: Optional[str] = None, + scenario_count: Optional[int] = None, +) -> None: + if workspace_id is None: + return record_event( - CHAT_COMPLETION, + AGENT_TEST_SETUP_GENERATED, organization_id, request_id, - properties={ - "workspace_id": workspace_id, - "model": model, - "quantity": 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=1, + request_id=request_id, + purpose=purpose, + model=model, + scenario_count=scenario_count, + ), ) diff --git a/app/services/media_urls.py b/app/services/media_urls.py index 6ef55adf..80cc674b 100644 --- a/app/services/media_urls.py +++ b/app/services/media_urls.py @@ -67,6 +67,7 @@ def build_voice_agent_ws_url( agent_id: Optional[str] = None, persona_id: Optional[str] = None, scenario_id: Optional[str] = None, + ui_surface: Optional[str] = None, fallback_host: Optional[str] = None, fallback_scheme: str = "http", ) -> str: @@ -86,5 +87,7 @@ def build_voice_agent_ws_url( query += f"&persona_id={quote(persona_id)}" if scenario_id: query += f"&scenario_id={quote(scenario_id)}" + if ui_surface: + query += f"&ui_surface={quote(ui_surface, safe='')}" return f"{base}{settings.API_V1_PREFIX}/voice-agent/ws?{query}" diff --git a/app/services/metric_studio/run_rollup.py b/app/services/metric_studio/run_rollup.py new file mode 100644 index 00000000..aee9015b --- /dev/null +++ b/app/services/metric_studio/run_rollup.py @@ -0,0 +1,61 @@ +"""Shared Metrics Studio run rollup + Flexprice emission.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy.orm import Session + +from app.models.database import MetricStudioRun, MetricStudioRunResult + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def rollup_metric_studio_run( + db: Session, + run: MetricStudioRun, + *, + emit_flexprice: bool = True, + commit: bool = True, +) -> None: + results = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.run_id == run.id) + .all() + ) + completed = sum(1 for r in results if r.status == "completed") + failed = sum(1 for r in results if r.status == "failed") + pending = sum(1 for r in results if r.status in {"pending", "running"}) + run.completed_items = completed + run.failed_items = failed + if pending: + run.status = "running" + elif failed and completed: + run.status = "partial" + run.finished_at = run.finished_at or _now_utc() + elif failed: + run.status = "failed" + run.finished_at = run.finished_at or _now_utc() + else: + run.status = "completed" + run.finished_at = run.finished_at or _now_utc() + + if commit: + db.commit() + else: + db.flush() + + if emit_flexprice and pending == 0 and run.finished_at is not None: + from app.services.billing.flexprice_service import record_metric_studio_run_completed + + record_metric_studio_run_completed( + run.organization_id, + run.id, + workspace_id=run.workspace_id, + run_status=run.status, + total_items=int(run.total_items or 0), + completed_items=completed, + failed_items=failed, + ) diff --git a/app/services/organization_provisioning.py b/app/services/organization_provisioning.py index a839ca67..847b8b03 100644 --- a/app/services/organization_provisioning.py +++ b/app/services/organization_provisioning.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from app.models.database import Workspace -from app.services.billing.flexprice_service import ensure_customer +from app.services.billing.flexprice_service import ensure_customer, ensure_subscription from app.services.workspace_rbac import ( backfill_org_workspace_memberships, ensure_creator_workspace_admin, @@ -62,3 +62,4 @@ def provision_billing_customer( ) -> None: """Register the org with Flexprice when billing is enabled (no-op otherwise).""" ensure_customer(organization_id, name=name, email=email) + ensure_subscription(organization_id) diff --git a/app/services/playground/post_call_processing.py b/app/services/playground/post_call_processing.py new file mode 100644 index 00000000..05a7f11f --- /dev/null +++ b/app/services/playground/post_call_processing.py @@ -0,0 +1,133 @@ +"""Atomic post-call processing for playground Voice AI poll tasks.""" +from __future__ import annotations + +from typing import Any, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import CallRecording + +PLAYGROUND_CALL_DATA_PRESERVE_KEYS = ("ui_surface", "external_usage_recorded") + + +def merge_playground_call_data( + prev: Optional[dict[str, Any]], + new: dict[str, Any], +) -> dict[str, Any]: + """Keep internal audit fields when provider metrics replace call_data.""" + merged = dict(new) + if isinstance(prev, dict): + for key in PLAYGROUND_CALL_DATA_PRESERVE_KEYS: + if prev.get(key) is not None: + merged[key] = prev[key] + return merged + + +def _lock_call_recording(db: Session, call_recording_id: UUID) -> Optional[CallRecording]: + return ( + db.query(CallRecording) + .filter(CallRecording.id == call_recording_id) + .with_for_update() + .first() + ) + + +def record_playground_post_call_usage_once( + db: Session, + call_recording_id: UUID, + *, + provider_platform: str, + call_metrics: dict[str, Any], +) -> tuple[bool, dict[str, Any]]: + """ + Record external provider usage at most once per call recording. + + Returns (should_create_evaluator, updated_call_metrics). + """ + from app.services.usage.external_agent_usage import ( + apply_playground_provider_usage_from_call_data, + ) + + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return False, call_metrics + + if locked.evaluator_result_id: + db.rollback() + logger.info( + "[Poll Call Metrics] Skipping post-call processing — " + "evaluator result already exists" + ) + return False, call_metrics + + stored_data = locked.call_data if isinstance(locked.call_data, dict) else {} + metrics = dict(call_metrics) if isinstance(call_metrics, dict) else call_metrics + platform_key = str(provider_platform or "").lower() + + if stored_data.get("external_usage_recorded") and isinstance(metrics, dict): + metrics = merge_playground_call_data(stored_data, metrics) + locked.call_data = metrics + db.commit() + elif isinstance(metrics, dict): + try: + apply_playground_provider_usage_from_call_data( + organization_id=locked.organization_id, + workspace_id=locked.workspace_id, + agent_id=locked.agent_id, + provider_platform=platform_key, + call_short_id=locked.call_short_id, + call_data=metrics, + ) + except Exception: + db.rollback() + logger.exception( + "[Poll Call Metrics] Usage counters failed for " + f"call recording {call_recording_id}" + ) + return False, call_metrics + + metrics["external_usage_recorded"] = True + metrics = merge_playground_call_data(stored_data, metrics) + locked.call_data = metrics + try: + db.commit() + except Exception: + db.rollback() + logger.exception( + "[Poll Call Metrics] Failed to persist external_usage_recorded " + f"for call recording {call_recording_id}" + ) + return False, call_metrics + + return True, metrics + + +def claim_playground_evaluator_result_slot( + db: Session, + call_recording_id: UUID, + *, + provider_call_id: str | None = None, +) -> Optional[CallRecording]: + """ + Lock the call recording row and return it when evaluator creation may proceed. + + Caller must create/link EvaluatorResult and commit before the lock is released. + """ + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return None + + if locked.evaluator_result_id: + db.rollback() + call_ref = provider_call_id or locked.provider_call_id or str(call_recording_id) + logger.info( + f"[Poll Call Metrics] Skipping evaluator creation — " + f"another poll already created result for call {call_ref}" + ) + return None + + return locked diff --git a/app/services/testing/llm_to_llm_evaluator_simulation.py b/app/services/testing/llm_to_llm_evaluator_simulation.py new file mode 100644 index 00000000..c14f84ef --- /dev/null +++ b/app/services/testing/llm_to_llm_evaluator_simulation.py @@ -0,0 +1,274 @@ +"""Text-based LLM-to-LLM simulation for evaluator runs without an external voice provider.""" + +from __future__ import annotations + +import re +from typing import Any, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import Agent, Evaluator, EvaluatorResult, Persona, Scenario, VoiceBundle +from app.models.enums import ModelProvider +from app.services.ai.llm_service import llm_service +from app.services.testing.test_agent_simulation_prompt import ( + build_persona_description_for_bridge, + build_test_agent_system_prompt, + get_agent_base_prompt, + resolve_persona_max_turns, +) +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, + usage_context_for_test_agent_simulation, +) + +_GOODBYE_RE = re.compile( + r"\b(goodbye|bye|thanks?\s+you|talk\s+to\s+you\s+later|have\s+a\s+(?:good|great)\s+(?:day|one))\b", + re.IGNORECASE, +) + + +def _build_agent_system_prompt(agent: Agent) -> str: + agent_name = (agent.name or "Voice AI Agent").strip() + base = get_agent_base_prompt(agent) + return ( + f"You are {agent_name}, a voice AI agent on a live phone call.\n\n" + f"Your instructions:\n{base}\n\n" + "Respond naturally in 1-3 sentences as on a phone call. " + "Respond ONLY with what you would say — no stage directions." + ) + + +def _should_end_conversation(text: str, *, turn_index: int, min_turns: int = 2) -> bool: + if turn_index < min_turns: + return False + return bool(_GOODBYE_RE.search(text or "")) + + +def _caller_messages(system_prompt: str, transcript: list[dict[str, str]]) -> list[dict[str, str]]: + messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] + for entry in transcript: + speaker = entry.get("speaker") + text = (entry.get("text") or "").strip() + if not text: + continue + if speaker == "Speaker 1": + messages.append({"role": "assistant", "content": text}) + else: + messages.append({"role": "user", "content": text}) + if len(messages) == 1: + messages.append( + { + "role": "user", + "content": "The call has just connected. Start the conversation.", + } + ) + return messages + + +def _agent_messages(system_prompt: str, transcript: list[dict[str, str]]) -> list[dict[str, str]]: + messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] + for entry in transcript: + speaker = entry.get("speaker") + text = (entry.get("text") or "").strip() + if not text: + continue + if speaker == "Speaker 2": + messages.append({"role": "assistant", "content": text}) + else: + messages.append({"role": "user", "content": text}) + return messages + + +def _resolve_voice_bundle_llm( + db: Session, + *, + voice_bundle: VoiceBundle, + organization_id: UUID, +) -> tuple[ModelProvider, str, Optional[dict], Optional[UUID]]: + raw_provider = voice_bundle.llm_provider + if raw_provider is None: + raise ValueError("Voice bundle is missing llm_provider") + provider = ( + raw_provider + if isinstance(raw_provider, ModelProvider) + else ModelProvider(str(raw_provider).lower()) + ) + model = (voice_bundle.llm_model or "").strip() + if not model: + raise ValueError("Voice bundle is missing llm_model") + llm_config = voice_bundle.llm_config if isinstance(voice_bundle.llm_config, dict) else None + credential_id = getattr(voice_bundle, "llm_credential_id", None) + return provider, model, llm_config, credential_id + + +def _generate_turn( + *, + messages: list[dict[str, str]], + llm_provider: ModelProvider, + llm_model: str, + organization_id: UUID, + db: Session, + llm_config: Optional[dict], + credential_id: Optional[UUID], +) -> str: + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + task_defaults={"temperature": 0.7, "max_tokens": 300}, + credential_id=credential_id, + ) + text = (result.get("text") or "").strip() + if not text: + raise ValueError("LLM returned an empty simulation response") + return text + + +def run_llm_to_llm_evaluator_simulation( + *, + evaluator: Evaluator, + result: EvaluatorResult, + agent: Agent, + persona: Persona, + scenario: Scenario, + organization_id: UUID, + db: Session, +) -> dict[str, Any]: + """Run a text simulation and populate the evaluator result transcript.""" + if not agent.voice_bundle_id: + raise ValueError("Agent does not have a voice bundle configured") + + voice_bundle = ( + db.query(VoiceBundle) + .filter( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id, + ) + .first() + ) + if not voice_bundle: + raise ValueError(f"Voice bundle {agent.voice_bundle_id} not found") + + llm_provider, llm_model, llm_config, credential_id = _resolve_voice_bundle_llm( + db, + voice_bundle=voice_bundle, + organization_id=organization_id, + ) + + max_turns = resolve_persona_max_turns(persona) + persona_description = build_persona_description_for_bridge(persona) + caller_system = build_test_agent_system_prompt( + agent, + persona, + scenario, + persona_description=persona_description, + max_turns=max_turns, + ) + agent_system = _build_agent_system_prompt(agent) + + caller_ctx = usage_context_for_test_agent_simulation( + organization_id=organization_id, + workspace_id=evaluator.workspace_id, + agent_id=agent.id, + evaluator_id=evaluator.id, + persona_id=persona.id, + scenario_id=scenario.id, + evaluator_result_id=result.id, + provider_platform="internal", + ) + agent_ctx = LLMUsageContext( + organization_id=organization_id, + workspace_id=evaluator.workspace_id, + product_section=LLMUsageProductSection.EVALUATORS, + resource_id=evaluator.id, + resource_type="evaluator", + extra={ + "agent_id": str(agent.id), + "evaluator_id": str(evaluator.id), + "persona_id": str(persona.id), + "scenario_id": str(scenario.id), + "evaluator_result_id": str(result.id), + "synthetic_testing": "pre_prod", + "simulation_leg": "production_agent", + "provider_platform": "internal", + }, + ) + + transcript: list[dict[str, str]] = [] + first_message = f"Hello, this is {persona.name} calling." + transcript.append({"speaker": "Speaker 1", "text": first_message}) + + exchanges = 0 + while exchanges < max_turns: + with llm_usage_context(agent_ctx): + agent_text = _generate_turn( + messages=_agent_messages(agent_system, transcript), + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + credential_id=credential_id, + ) + transcript.append({"speaker": "Speaker 2", "text": agent_text}) + exchanges += 1 + if _should_end_conversation(agent_text, turn_index=exchanges): + break + + with llm_usage_context(caller_ctx): + caller_text = _generate_turn( + messages=_caller_messages(caller_system, transcript), + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + credential_id=credential_id, + ) + transcript.append({"speaker": "Speaker 1", "text": caller_text}) + exchanges += 1 + if _should_end_conversation(caller_text, turn_index=exchanges): + break + + transcription = "\n".join( + f"{entry['speaker']}: {entry['text']}" for entry in transcript if entry.get("text") + ) + speaker_segments = [ + { + "speaker": entry["speaker"], + "text": entry["text"], + "start": float(idx), + "end": float(idx) + 1.0, + } + for idx, entry in enumerate(transcript) + ] + + result.transcription = transcription + result.speaker_segments = speaker_segments + result.provider_platform = "internal" + result.call_data = { + "source": "llm_to_llm_simulation", + "simulation": "llm_to_llm", + "exchanges": exchanges, + "messages": transcript, + } + result.duration_seconds = float(max(1, len(transcript))) + + logger.info( + "[LLM simulation] Completed evaluator {} result {} with {} transcript lines", + evaluator.evaluator_id, + result.result_id, + len(transcript), + ) + return { + "transcript_lines": len(transcript), + "exchanges": exchanges, + "provider_platform": "internal", + } diff --git a/app/services/testing/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py index b13b78b1..50b19699 100644 --- a/app/services/testing/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -615,6 +615,12 @@ def resolve_api_key_for_provider( test_agent_config = TestAgentConfig( organization_id=organization_id, workspace_id=getattr(agent, "workspace_id", None), + agent_id=agent.id, + evaluator_id=evaluator_id, + persona_id=persona.id, + scenario_id=scenario.id, + evaluator_result_id=evaluator_result_id, + db=db, agent_name=agent.name or "Voice AI Agent", agent_description=agent.description or "A voice AI assistant", test_agent_simulation_prompt=simulation_prompt, diff --git a/app/services/testing/test_agent_service.py b/app/services/testing/test_agent_service.py index 84348190..c7d09b37 100644 --- a/app/services/testing/test_agent_service.py +++ b/app/services/testing/test_agent_service.py @@ -24,6 +24,7 @@ from app.services.ai.transcription_service import transcription_service from app.services.ai.llm_service import llm_service from app.services.ai.tts_service import tts_service +from app.services.storage.s3_service import s3_service from app.services.testing.test_agent_simulation_prompt import build_test_agent_system_prompt from sqlalchemy.orm import Session @@ -237,6 +238,20 @@ def process_audio_chunk( if not all([agent, persona, scenario]): raise ValueError("Missing agent, persona, or scenario") + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_test_agent_simulation, + ) + + usage_ctx = usage_context_for_test_agent_simulation( + organization_id=organization_id, + workspace_id=conversation.workspace_id, + agent_id=agent.id, + persona_id=persona.id, + scenario_id=scenario.id, + conversation_id=conversation.id, + ) + # Calculate timestamp if chunk_timestamp is None: if conversation.started_at: @@ -260,19 +275,45 @@ def process_audio_chunk( "error": f"Failed to convert audio format: {str(e)}", } - # Save audio chunk temporarily for transcription + with llm_usage_context(usage_ctx): + return self._process_audio_chunk_with_context( + conversation=conversation, + wav_audio_bytes=wav_audio_bytes, + voice_bundle=voice_bundle, + agent=agent, + persona=persona, + scenario=scenario, + organization_id=organization_id, + db=db, + chunk_timestamp=chunk_timestamp, + ) + + def _process_audio_chunk_with_context( + self, + *, + conversation: TestAgentConversation, + wav_audio_bytes: bytes, + voice_bundle: VoiceBundle, + agent: Agent, + persona: Persona, + scenario: Scenario, + organization_id: UUID, + db: Session, + chunk_timestamp: float, + ) -> Dict[str, Any]: temp_file_path = None try: - # Save converted WAV to temp file for transcription with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file: temp_file.write(wav_audio_bytes) temp_file_path = temp_file.name - # Upload to S3 temporarily for transcription service (it needs S3 key) chunk_file_id = uuid.uuid4() - chunk_s3_key = s3_service.upload_file(file_content=wav_audio_bytes, file_id=chunk_file_id, file_format="wav") + chunk_s3_key = s3_service.upload_file( + file_content=wav_audio_bytes, + file_id=chunk_file_id, + file_format="wav", + ) - # Transcribe using STT transcription_result = transcription_service.transcribe( audio_file_key=chunk_s3_key, stt_provider=voice_bundle.stt_provider, @@ -292,24 +333,24 @@ def process_audio_chunk( "error": "No speech detected in audio chunk", } - # Add voice agent turn to conversation conversation_turns = conversation.live_transcription or [] - conversation_turns.append({"speaker": "voice_agent", "text": voice_agent_text, "timestamp": chunk_timestamp}) + conversation_turns.append( + { + "speaker": "voice_agent", + "text": voice_agent_text, + "timestamp": chunk_timestamp, + } + ) - # Build conversation history for LLM messages = [] - - # System prompt system_prompt = self._build_system_prompt(agent, persona, scenario, db) messages.append({"role": "system", "content": system_prompt}) - # Add conversation history (last 10 turns for context) recent_turns = conversation_turns[-10:] for turn in recent_turns: role = "user" if turn["speaker"] == "voice_agent" else "assistant" messages.append({"role": role, "content": turn["text"]}) - # Generate response using LLM llm_result = llm_service.generate_response( messages=messages, llm_provider=voice_bundle.llm_provider, @@ -331,7 +372,6 @@ def process_audio_chunk( "error": "LLM did not generate a response", } - # Convert response to speech using TTS from app.services.voice_agent.resolve_tts_voice import resolve_effective_tts_voice_id tts_voice = resolve_effective_tts_voice_id( @@ -348,34 +388,44 @@ def process_audio_chunk( config=voice_bundle.tts_config, ) - # Upload response audio to S3 (temporarily, for reference) response_file_id = uuid.uuid4() - response_s3_key = s3_service.upload_file(file_content=response_audio_bytes, file_id=response_file_id, file_format="mp3") + s3_service.upload_file( + file_content=response_audio_bytes, + file_id=response_file_id, + file_format="mp3", + ) - # Add test agent turn to conversation conversation_turns.append( { "speaker": "test_agent", "text": test_agent_text, - "timestamp": chunk_timestamp + transcription_result.get("processing_time", 0) + llm_result.get("processing_time", 0), + "timestamp": chunk_timestamp + + transcription_result.get("processing_time", 0) + + llm_result.get("processing_time", 0), } ) - # Update conversation conversation.live_transcription = conversation_turns - conversation.full_transcript = "\n".join([f"{turn['speaker']}: {turn['text']}" for turn in conversation_turns]) + conversation.full_transcript = "\n".join( + [f"{turn['speaker']}: {turn['text']}" for turn in conversation_turns] + ) db.commit() return { "response_audio": response_audio_bytes, - "transcription": {"voice_agent": voice_agent_text, "test_agent": test_agent_text}, + "transcription": { + "voice_agent": voice_agent_text, + "test_agent": test_agent_text, + }, "metadata": { - "processing_times": {"stt": transcription_result.get("processing_time", 0), "llm": llm_result.get("processing_time", 0)} + "processing_times": { + "stt": transcription_result.get("processing_time", 0), + "llm": llm_result.get("processing_time", 0), + } }, } finally: - # Clean up temp file if temp_file_path and os.path.exists(temp_file_path): try: os.unlink(temp_file_path) diff --git a/app/services/usage/bucket_context.py b/app/services/usage/bucket_context.py index 64187a39..ef813493 100644 --- a/app/services/usage/bucket_context.py +++ b/app/services/usage/bucket_context.py @@ -26,6 +26,18 @@ } ) +# Per-request identifiers — useful in LLMUsageContext.extra but must not +# create a new Redis/PG bucket per call. +_HIGH_CARDINALITY_EXTRA_KEYS = frozenset( + { + "evaluator_result_id", + "result_short_id", + "metric_studio_result_id", + "conversation_id", + "call_short_id", + } +) + # Never store roll-up counters in JSONB — they stay as BIGINT columns for SUM(). _FORBIDDEN_CONTEXT_KEYS = frozenset( { @@ -58,7 +70,7 @@ def build_bucket_context( for key, value in extra.items(): if value is None or key in ctx: continue - if key in _FORBIDDEN_CONTEXT_KEYS: + if key in _FORBIDDEN_CONTEXT_KEYS or key in _HIGH_CARDINALITY_EXTRA_KEYS: continue ctx[key] = str(value) return ctx diff --git a/app/services/usage/context.py b/app/services/usage/context.py index 8bd895e1..ade2cfde 100644 --- a/app/services/usage/context.py +++ b/app/services/usage/context.py @@ -206,31 +206,88 @@ def usage_context_for_agent( def usage_context_for_evaluator_result(result: Any) -> LLMUsageContext: - """Usage context for processing an evaluator result (Vapi / playground runs).""" - if result.agent_id: - return LLMUsageContext( - organization_id=result.organization_id, - workspace_id=result.workspace_id, - product_section=LLMUsageProductSection.AGENTS, - resource_id=result.agent_id, - resource_type="agent", - extra={"agent_id": str(result.agent_id)}, - ) - extra: dict[str, str] = {"evaluator_result_id": str(result.id)} + """Usage context for post-run processing of synthetic evaluator results.""" + extra: dict[str, str] = { + "evaluator_result_id": str(result.id), + "synthetic_testing": "pre_prod", + } if getattr(result, "result_id", None): extra["result_short_id"] = str(result.result_id) + if result.agent_id: + extra["agent_id"] = str(result.agent_id) if result.evaluator_id: extra["evaluator_id"] = str(result.evaluator_id) + if getattr(result, "persona_id", None): + extra["persona_id"] = str(result.persona_id) + if getattr(result, "scenario_id", None): + extra["scenario_id"] = str(result.scenario_id) + platform = getattr(result, "provider_platform", None) + if platform: + extra["provider_platform"] = str(platform).lower() + + resource_id = result.evaluator_id or result.id + resource_type = "evaluator" if result.evaluator_id else "evaluator_result" + section = ( + LLMUsageProductSection.EVALUATORS + if result.evaluator_id + else LLMUsageProductSection.PLAYGROUND + ) return LLMUsageContext( organization_id=result.organization_id, workspace_id=result.workspace_id, - product_section=LLMUsageProductSection.EVALUATORS, - resource_id=result.id, - resource_type="evaluator_result", + product_section=section, + resource_id=resource_id, + resource_type=resource_type, extra=extra, ) +def usage_context_for_metric_studio_run( + run: Any, + *, + source_kind: Optional[str] = None, + source_ref: Optional[str] = None, + result_row_id: Optional[UUID] = None, +) -> LLMUsageContext: + """Usage context for Metrics Studio batch scoring.""" + extra: dict[str, str] = {"metric_studio_run_id": str(run.id)} + if source_kind: + extra["source_kind"] = source_kind + if source_ref: + extra["source_ref"] = source_ref + if result_row_id: + extra["metric_studio_result_id"] = str(result_row_id) + if source_kind == "evaluator_result": + extra["synthetic_testing"] = "pre_prod" + return LLMUsageContext( + organization_id=run.organization_id, + workspace_id=run.workspace_id, + product_section=LLMUsageProductSection.METRICS, + resource_id=run.id, + resource_type="metric_studio_run", + extra=extra, + ) + + +def usage_context_for_persona_generation( + agent: Any, + *, + workspace_id: Optional[UUID] = None, +) -> LLMUsageContext: + """Usage context for LLM-generated persona caller prompts.""" + return LLMUsageContext( + organization_id=agent.organization_id, + workspace_id=workspace_id or agent.workspace_id, + product_section=LLMUsageProductSection.PERSONAS, + resource_id=agent.id, + resource_type="agent", + extra={ + "agent_id": str(agent.id), + "synthetic_testing": "pre_prod", + }, + ) + + def usage_context_for_prompt_optimization_run(run: Any) -> LLMUsageContext: """Usage context for a GEPA prompt optimization run.""" cfg = run.config if isinstance(run.config, dict) else {} @@ -284,3 +341,67 @@ def usage_context_for_prompt_partial(partial: Any) -> LLMUsageContext: resource_type="prompt_partial", extra={"prompt_partial_id": str(partial.id)}, ) + + +def usage_context_for_playground_voice_call( + *, + organization_id: UUID, + workspace_id: Optional[UUID], + agent_id: Optional[UUID], + provider_platform: Optional[str], + call_short_id: Optional[str] = None, +) -> LLMUsageContext: + """Usage context for live playground Voice AI Agent provider sessions.""" + extra: dict[str, str] = {"synthetic_testing": "pre_prod"} + if agent_id: + extra["agent_id"] = str(agent_id) + if provider_platform: + extra["provider_platform"] = str(provider_platform).lower() + if call_short_id: + extra["call_short_id"] = call_short_id + return LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.PLAYGROUND, + resource_id=agent_id, + resource_type="agent" if agent_id else None, + extra=extra, + ) + + +def usage_context_for_test_agent_simulation( + *, + organization_id: UUID, + workspace_id: Optional[UUID] = None, + agent_id: Optional[UUID] = None, + evaluator_id: Optional[UUID] = None, + persona_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, + evaluator_result_id: Optional[UUID] = None, + conversation_id: Optional[UUID] = None, + provider_platform: Optional[str] = None, +) -> LLMUsageContext: + """Usage context for synthetic LLM-to-LLM simulation (caller / test-agent leg).""" + extra: dict[str, str] = {"simulation": "llm_to_llm"} + if agent_id: + extra["agent_id"] = str(agent_id) + if evaluator_id: + extra["evaluator_id"] = str(evaluator_id) + if persona_id: + extra["persona_id"] = str(persona_id) + if scenario_id: + extra["scenario_id"] = str(scenario_id) + if evaluator_result_id: + extra["evaluator_result_id"] = str(evaluator_result_id) + if conversation_id: + extra["conversation_id"] = str(conversation_id) + if provider_platform: + extra["provider_platform"] = str(provider_platform) + return LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.TEST_AGENT, + resource_id=agent_id, + resource_type="agent" if agent_id else None, + extra=extra, + ) diff --git a/app/services/usage/external_agent_usage.py b/app/services/usage/external_agent_usage.py new file mode 100644 index 00000000..146d1029 --- /dev/null +++ b/app/services/usage/external_agent_usage.py @@ -0,0 +1,363 @@ +"""Extract production-agent LLM usage from external voice provider call payloads.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Optional +from uuid import UUID + +from loguru import logger + +from app.services.usage.context import LLMUsageContext +from app.services.usage.llm_usage import record_llm_usage, record_stt_usage, record_tts_usage +from app.services.usage.normalize import UsageSnapshot, usage_snapshot_is_billable + + +@dataclass(frozen=True) +class ExternalAgentUsageExtraction: + model: str + llm: Optional[UsageSnapshot] = None + stt_audio_seconds: int = 0 + tts_characters: int = 0 + + +class ExternalUsageRecordingError(RuntimeError): + """Raised when billable external provider usage could not be persisted.""" + + +def _as_int(value: Any) -> int: + try: + return max(0, int(value or 0)) + except (TypeError, ValueError): + return 0 + + +def _pick(*values: Any) -> Any: + for value in values: + if value is not None: + return value + return None + + +def _vapi_cost_breakdown(call_data: dict[str, Any]) -> dict[str, Any]: + raw = call_data.get("costBreakdown") or call_data.get("cost_breakdown") or {} + return raw if isinstance(raw, dict) else {} + + +def _retell_llm_snapshot(call_data: dict[str, Any]) -> Optional[UsageSnapshot]: + llm_usage = call_data.get("llm_token_usage") or {} + if not isinstance(llm_usage, dict): + llm_usage = {} + + prompt = completion = 0 + direct_fields = ( + ("prompt_tokens", "prompt"), + ("input_tokens", "prompt"), + ("llm_prompt_tokens", "prompt"), + ("completion_tokens", "completion"), + ("output_tokens", "completion"), + ("llm_completion_tokens", "completion"), + ) + for key, bucket in direct_fields: + value = _as_int(llm_usage.get(key)) + if value <= 0: + continue + if bucket == "prompt": + prompt += value + else: + completion += value + + call_cost = call_data.get("call_cost") or {} + product_costs = call_cost.get("product_costs") if isinstance(call_cost, dict) else None + if isinstance(product_costs, list): + for item in product_costs: + if not isinstance(item, dict): + continue + product = str(item.get("product") or "").lower() + if "llm" not in product and "gpt" not in product and "model" not in product: + continue + prompt += _as_int(_pick(item.get("prompt_tokens"), item.get("input_tokens"))) + completion += _as_int( + _pick(item.get("completion_tokens"), item.get("output_tokens")) + ) + + latency = call_data.get("latency") or {} + if isinstance(latency, dict): + llm_latency = latency.get("llm") + if isinstance(llm_latency, dict): + prompt += _as_int(_pick(llm_latency.get("prompt_tokens"), llm_latency.get("input_tokens"))) + completion += _as_int( + _pick(llm_latency.get("completion_tokens"), llm_latency.get("output_tokens")) + ) + + if prompt > 0 or completion > 0: + return UsageSnapshot(prompt_tokens=prompt, completion_tokens=completion) + + values = llm_usage.get("values") + total_tokens = sum(_as_int(v) for v in values) if isinstance(values, list) else 0 + if total_tokens <= 0: + return None + + prompt = int(round(total_tokens * 0.7)) + completion = max(0, total_tokens - prompt) + return UsageSnapshot(prompt_tokens=prompt, completion_tokens=completion) + + +def extract_external_agent_usage( + call_data: dict[str, Any] | None, + *, + platform: str, +) -> Optional[ExternalAgentUsageExtraction]: + if not isinstance(call_data, dict) or not call_data: + return None + + platform_key = (platform or "").strip().lower() + if platform_key == "vapi": + cb = _vapi_cost_breakdown(call_data) + snapshot = UsageSnapshot( + prompt_tokens=_as_int( + _pick(cb.get("llmPromptTokens"), cb.get("llm_prompt_tokens")) + ), + completion_tokens=_as_int( + _pick(cb.get("llmCompletionTokens"), cb.get("llm_completion_tokens")) + ), + cache_read_tokens=_as_int( + _pick( + cb.get("llmCachedPromptTokens"), + cb.get("llm_cached_prompt_tokens"), + ) + ), + ) + model = ( + _pick( + call_data.get("model"), + call_data.get("assistant", {}).get("model") + if isinstance(call_data.get("assistant"), dict) + else None, + ) + or "vapi-agent" + ) + duration = _as_int( + _pick( + call_data.get("durationSeconds"), + call_data.get("duration_seconds"), + ) + ) + tts_chars = _as_int( + _pick(cb.get("ttsCharacters"), cb.get("tts_characters")) + ) + if not usage_snapshot_is_billable(snapshot) and duration <= 0 and tts_chars <= 0: + return None + return ExternalAgentUsageExtraction( + model=str(model), + llm=snapshot if usage_snapshot_is_billable(snapshot) else None, + stt_audio_seconds=duration, + tts_characters=tts_chars, + ) + + if platform_key == "retell": + call_cost = call_data.get("call_cost") or {} + duration_ms = call_data.get("duration_ms") + duration = _as_int( + _pick( + call_cost.get("total_duration_seconds") + if isinstance(call_cost, dict) + else None, + int(duration_ms) / 1000 if duration_ms else None, + call_data.get("duration_seconds"), + call_data.get("duration"), + ) + ) + model = _pick(call_data.get("llm_model"), call_data.get("model")) or "retell-agent" + snapshot = _retell_llm_snapshot(call_data) + if not usage_snapshot_is_billable(snapshot) and duration <= 0: + return None + return ExternalAgentUsageExtraction( + model=str(model), + llm=snapshot if usage_snapshot_is_billable(snapshot) else None, + stt_audio_seconds=duration, + ) + + if platform_key == "elevenlabs": + metadata = call_data.get("metadata") or {} + if not isinstance(metadata, dict): + metadata = {} + duration = _as_int( + _pick( + metadata.get("call_duration_secs"), + call_data.get("duration_seconds"), + call_data.get("duration"), + ) + ) + if duration <= 0: + return None + return ExternalAgentUsageExtraction( + model="elevenlabs-agent", + stt_audio_seconds=duration, + ) + + if platform_key == "smallest": + duration = _as_int( + _pick( + call_data.get("duration_seconds"), + call_data.get("duration"), + ) + ) + if duration <= 0: + return None + return ExternalAgentUsageExtraction( + model="smallest-agent", + stt_audio_seconds=duration, + ) + + return None + + +def apply_playground_provider_usage_from_call_data( + *, + organization_id: UUID, + workspace_id: Optional[UUID], + agent_id: Optional[UUID], + provider_platform: str, + call_short_id: Optional[str], + call_data: dict[str, Any], +) -> None: + """Apply provider usage counters from call_data (no dedup guard).""" + from types import SimpleNamespace + + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_playground_voice_call, + ) + + metrics = dict(call_data or {}) + usage_ctx = usage_context_for_playground_voice_call( + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + provider_platform=provider_platform, + call_short_id=call_short_id, + ) + stub = SimpleNamespace( + organization_id=organization_id, + provider_platform=provider_platform, + call_data=metrics, + result_id=call_short_id or "playground", + ) + with llm_usage_context(usage_ctx): + if not record_external_agent_usage(stub, usage_ctx=usage_ctx): + raise ExternalUsageRecordingError( + f"failed to persist external provider usage for call " + f"{call_short_id or 'playground'}" + ) + + +def record_playground_provider_usage_from_call_data( + *, + organization_id: UUID, + workspace_id: Optional[UUID], + agent_id: Optional[UUID], + provider_platform: str, + call_short_id: Optional[str], + call_data: dict[str, Any], +) -> dict[str, Any]: + """Record provider usage for a completed playground Voice AI call.""" + metrics = dict(call_data or {}) + if metrics.get("external_usage_recorded"): + return metrics + + apply_playground_provider_usage_from_call_data( + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=agent_id, + provider_platform=provider_platform, + call_short_id=call_short_id, + call_data=metrics, + ) + metrics["external_usage_recorded"] = True + return metrics + + +def record_external_agent_usage( + result: Any, + *, + usage_ctx: LLMUsageContext, +) -> bool: + """Record production-agent usage from stored provider call_data. + + Returns True when usage was persisted or there was nothing billable to record. + Returns False when billable usage could not be stored. + """ + call_data = getattr(result, "call_data", None) + platform = getattr(result, "provider_platform", None) or "" + extraction = extract_external_agent_usage( + call_data if isinstance(call_data, dict) else None, + platform=str(platform), + ) + if extraction is None: + return True + + org_id = getattr(result, "organization_id", None) + if org_id is None: + logger.warning( + "external agent usage record skipped: missing organization_id for result {}", + getattr(result, "result_id", result), + ) + return False + + ctx = LLMUsageContext( + organization_id=usage_ctx.organization_id, + workspace_id=usage_ctx.workspace_id, + product_section=usage_ctx.product_section, + resource_id=usage_ctx.resource_id, + resource_type=usage_ctx.resource_type, + extra={ + **(usage_ctx.extra or {}), + "synthetic_testing": "pre_prod", + "simulation_leg": "production_agent", + "provider_platform": str(platform).lower(), + }, + ) + + outcomes: list[bool] = [] + if extraction.llm and usage_snapshot_is_billable(extraction.llm): + outcomes.append( + record_llm_usage( + extraction.model, + extraction.llm, + organization_id=org_id, + ctx=ctx, + ) + ) + if extraction.stt_audio_seconds > 0: + outcomes.append( + record_stt_usage( + f"{platform}-stt", + audio_seconds=extraction.stt_audio_seconds, + organization_id=org_id, + ctx=ctx, + count_call=False, + ) + ) + if extraction.tts_characters > 0: + outcomes.append( + record_tts_usage( + f"{platform}-tts", + characters=extraction.tts_characters, + organization_id=org_id, + ctx=ctx, + ) + ) + + if not outcomes: + return True + + if all(outcomes): + return True + + logger.warning( + "external agent usage record incomplete for result {} (platform={})", + getattr(result, "result_id", result), + platform, + ) + return False diff --git a/app/services/usage/llm_usage.py b/app/services/usage/llm_usage.py index bbeac232..38c9455f 100644 --- a/app/services/usage/llm_usage.py +++ b/app/services/usage/llm_usage.py @@ -373,13 +373,13 @@ def _buffer_to_postgres( organization_id: UUID, bucket: Dict[str, Any], deltas: Dict[str, int], -) -> None: +) -> bool: """Durable fallback when Redis is unavailable.""" try: from app.database import SessionLocal except Exception as exc: logger.warning("usage postgres fallback unavailable: {}", exc) - return + return False usage_date = bucket["usage_date"] if isinstance(usage_date, str): @@ -443,9 +443,11 @@ def _buffer_to_postgres( params, ) db.commit() + return True except Exception as exc: db.rollback() logger.warning("usage postgres fallback insert failed: {}", exc) + return False finally: db.close() @@ -455,7 +457,7 @@ def _incr_pending( prefix: str, deltas: Dict[str, int], bucket: Dict[str, Any], -) -> None: +) -> bool: hash_key = _pending_hash_key(organization_id) try: client = _client() @@ -466,9 +468,10 @@ def _incr_pending( pipe.sadd("usage:pending:orgs", str(organization_id)) pipe.expire(hash_key, _PENDING_TTL_SECONDS) pipe.execute() + return True except redis.RedisError as exc: logger.warning("usage redis counter failed, buffering to postgres: {}", exc) - _buffer_to_postgres(organization_id, bucket, deltas) + return _buffer_to_postgres(organization_id, bucket, deltas) def _context_for_record( @@ -491,18 +494,22 @@ def record_llm_usage( organization_id: Optional[UUID] = None, ctx: Optional[LLMUsageContext] = None, usage_date: Optional[date] = None, -) -> None: - """Increment counters for one LLM call (best-effort, never raises).""" +) -> bool: + """Increment counters for one LLM call (best-effort, never raises). + + Returns True when usage was persisted or there was nothing billable to record. + Returns False when billable usage could not be stored. + """ if not model: model = "unknown" context = _context_for_record(organization_id=organization_id, ctx=ctx) if context is None: logger.warning("llm usage record skipped: missing organization_id") - return + return False deltas = _deltas_from_usage(usage) if not _has_billable_usage_deltas(deltas): - return + return True day = usage_date or datetime.now(timezone.utc).date() bucket = _bucket_from_context( @@ -512,7 +519,7 @@ def record_llm_usage( usage_kind=USAGE_KIND_LLM, ) prefix = _bucket_prefix(**bucket) - _incr_pending(context.organization_id, prefix, deltas, bucket) + return _incr_pending(context.organization_id, prefix, deltas, bucket) def record_stt_usage( @@ -523,18 +530,18 @@ def record_stt_usage( ctx: Optional[LLMUsageContext] = None, usage_date: Optional[date] = None, count_call: bool = True, -) -> None: +) -> bool: """Increment counters for one STT call (audio seconds + optional call_count).""" if not model: model = "unknown" context = _context_for_record(organization_id=organization_id, ctx=ctx) if context is None: logger.warning("stt usage record skipped: missing organization_id") - return + return False seconds = int(max(0, math.ceil(float(audio_seconds or 0)))) if seconds <= 0: - return + return True deltas = _deltas_from_stt(seconds, count_call=count_call) day = usage_date or datetime.now(timezone.utc).date() bucket = _bucket_from_context( @@ -544,7 +551,7 @@ def record_stt_usage( usage_kind=USAGE_KIND_STT, ) prefix = _bucket_prefix(**bucket) - _incr_pending(context.organization_id, prefix, deltas, bucket) + return _incr_pending(context.organization_id, prefix, deltas, bucket) def record_call_usage( @@ -554,7 +561,7 @@ def record_call_usage( ctx: Optional[LLMUsageContext] = None, usage_date: Optional[date] = None, audio_seconds: int = 0, -) -> None: +) -> bool: """Record one completed call session (call_count + optional duration). Best-effort, never raises. Uses the same Redis-buffered path as other @@ -565,7 +572,7 @@ def record_call_usage( context = _context_for_record(organization_id=organization_id, ctx=ctx) if context is None: logger.warning("call usage record skipped: missing organization_id") - return + return False seconds = max(0, int(audio_seconds or 0)) deltas = { @@ -586,7 +593,7 @@ def record_call_usage( usage_kind=USAGE_KIND_LLM, ) prefix = _bucket_prefix(**bucket) - _incr_pending(context.organization_id, prefix, deltas, bucket) + return _incr_pending(context.organization_id, prefix, deltas, bucket) def record_tts_usage( @@ -596,18 +603,18 @@ def record_tts_usage( organization_id: Optional[UUID] = None, ctx: Optional[LLMUsageContext] = None, usage_date: Optional[date] = None, -) -> None: +) -> bool: """Increment counters for one TTS call (characters + call_count).""" if not model: model = "unknown" context = _context_for_record(organization_id=organization_id, ctx=ctx) if context is None: logger.warning("tts usage record skipped: missing organization_id") - return + return False chars = max(0, int(characters or 0)) if chars <= 0: - return + return True deltas = _deltas_from_tts(chars) day = usage_date or datetime.now(timezone.utc).date() bucket = _bucket_from_context( @@ -617,7 +624,7 @@ def record_tts_usage( usage_kind=USAGE_KIND_TTS, ) prefix = _bucket_prefix(**bucket) - _incr_pending(context.organization_id, prefix, deltas, bucket) + return _incr_pending(context.organization_id, prefix, deltas, bucket) def probe_audio_seconds(audio_file_path: str) -> int: diff --git a/app/services/webrtc_bridge/test_agent_processor.py b/app/services/webrtc_bridge/test_agent_processor.py index 990fa6bd..a4d39946 100644 --- a/app/services/webrtc_bridge/test_agent_processor.py +++ b/app/services/webrtc_bridge/test_agent_processor.py @@ -9,8 +9,8 @@ import asyncio import io import os +from dataclasses import dataclass from typing import Optional, Callable, Awaitable, List, Dict, Any, Union -from dataclasses import dataclass, field from uuid import UUID from loguru import logger @@ -93,6 +93,13 @@ class TestAgentConfig: organization_id: Optional[Union[UUID, str]] = None workspace_id: Optional[Union[UUID, str]] = None + agent_id: Optional[Union[UUID, str]] = None + evaluator_id: Optional[Union[UUID, str]] = None + persona_id: Optional[Union[UUID, str]] = None + scenario_id: Optional[Union[UUID, str]] = None + evaluator_result_id: Optional[Union[UUID, str]] = None + conversation_id: Optional[Union[UUID, str]] = None + db: Any = None class TestAgentProcessor: @@ -344,37 +351,116 @@ async def _do_process_transcript(self, transcript: str) -> Optional[bytes]: # Fire-and-forget so we don't block the caller asyncio.create_task(self._do_process_transcript(pending)) + def _build_simulation_context(self): + from app.services.usage.context import usage_context_for_test_agent_simulation + + if not self.config.organization_id: + return None + return usage_context_for_test_agent_simulation( + organization_id=UUID(str(self.config.organization_id)), + workspace_id=( + UUID(str(self.config.workspace_id)) + if self.config.workspace_id + else None + ), + agent_id=UUID(str(self.config.agent_id)) if self.config.agent_id else None, + evaluator_id=( + UUID(str(self.config.evaluator_id)) if self.config.evaluator_id else None + ), + persona_id=UUID(str(self.config.persona_id)) if self.config.persona_id else None, + scenario_id=UUID(str(self.config.scenario_id)) if self.config.scenario_id else None, + evaluator_result_id=( + UUID(str(self.config.evaluator_result_id)) + if self.config.evaluator_result_id + else None + ), + conversation_id=( + UUID(str(self.config.conversation_id)) + if self.config.conversation_id + else None + ), + ) + + def _sync_llm_call(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: + from app.models.database import ModelProvider + from app.services.ai.llm_service import llm_service + from app.services.usage.context import llm_usage_context + + ctx = self._build_simulation_context() + org_id = UUID(str(self.config.organization_id)) + if ctx is not None: + with llm_usage_context(ctx): + return llm_service.generate_response( + messages=messages, + llm_provider=ModelProvider.OPENAI, + llm_model=self.config.llm_model, + organization_id=org_id, + db=self.config.db, + temperature=( + self.config.llm_temperature + if self.config.llm_temperature is not None + else 0.7 + ), + max_tokens=( + self.config.llm_max_tokens + if self.config.llm_max_tokens is not None + else 150 + ), + ) + return llm_service.generate_response( + messages=messages, + llm_provider=ModelProvider.OPENAI, + llm_model=self.config.llm_model, + organization_id=org_id, + db=self.config.db, + temperature=( + self.config.llm_temperature + if self.config.llm_temperature is not None + else 0.7 + ), + max_tokens=( + self.config.llm_max_tokens + if self.config.llm_max_tokens is not None + else 150 + ), + ) + async def _generate_llm_response(self) -> Optional[str]: """Generate a response using the LLM.""" try: messages = [ {"role": "system", "content": self._system_prompt} ] + self.conversation_history - - if EFFICIENTAI_AVAILABLE and self._llm_service: - # Use EfficientAI LLM service - # Note: This is a simplified version - actual implementation - # would need to handle the frame-based processing - pass - - # Use direct OpenAI API + + if self.config.db and self.config.organization_id: + result = await asyncio.to_thread(self._sync_llm_call, messages) + return (result.get("text") or "").strip() + import openai - client = getattr(self, '_openai_client', None) + client = getattr(self, "_openai_client", None) if not client: api_key = self.config.llm_api_key or os.getenv("OPENAI_API_KEY") client = openai.AsyncOpenAI(api_key=api_key) - + response = await client.chat.completions.create( model=self.config.llm_model, messages=messages, - max_tokens=self.config.llm_max_tokens if self.config.llm_max_tokens is not None else 150, - temperature=self.config.llm_temperature if self.config.llm_temperature is not None else 0.7, + max_tokens=( + self.config.llm_max_tokens + if self.config.llm_max_tokens is not None + else 150 + ), + temperature=( + self.config.llm_temperature + if self.config.llm_temperature is not None + else 0.7 + ), ) self._record_llm_usage(response=response) return response.choices[0].message.content.strip() - + except Exception as e: logger.error(f"[TestAgent] LLM error: {e}") return None @@ -409,77 +495,41 @@ def _tts_settings(self) -> Dict[str, Any]: return dict(self.config.tts_config or {}) def _record_tts_usage(self, *, text: str) -> None: - if not self.config.organization_id: + ctx = self._build_simulation_context() + if ctx is None: return try: - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, - ) + from app.services.usage.context import llm_usage_context from app.services.usage.llm_usage import record_tts_usage model = self.config.tts_model or TTS_DEFAULT_MODELS.get( self.config.tts_provider.lower(), "unknown" ) - org_id = UUID(str(self.config.organization_id)) - ws_id = ( - UUID(str(self.config.workspace_id)) - if self.config.workspace_id - else None - ) - with llm_usage_context( - LLMUsageContext( - organization_id=org_id, - workspace_id=ws_id, - product_section=LLMUsageProductSection.TEST_AGENT, - ) - ): + with llm_usage_context(ctx): record_tts_usage( model, characters=len(text or ""), - organization_id=org_id, + organization_id=ctx.organization_id, ) except Exception as exc: logger.debug("test agent tts usage record skipped: {}", exc) def _record_llm_usage(self, *, response: Any) -> None: - if not self.config.organization_id: + ctx = self._build_simulation_context() + if ctx is None: return try: - from app.services.usage.context import ( - LLMUsageContext, - LLMUsageProductSection, - llm_usage_context, - ) + from app.services.usage.context import llm_usage_context from app.services.usage.llm_usage import record_llm_usage - from app.services.usage.normalize import UsageSnapshot + from app.services.usage.normalize import UsageSnapshot, normalize_llm_usage - usage = getattr(response, "usage", None) - if usage is None: - return - org_id = UUID(str(self.config.organization_id)) - ws_id = ( - UUID(str(self.config.workspace_id)) - if self.config.workspace_id - else None - ) + snapshot = normalize_llm_usage(raw_response=response) model = self.config.llm_model or "unknown" - snapshot = UsageSnapshot( - prompt_tokens=int(getattr(usage, "prompt_tokens", 0) or 0), - completion_tokens=int(getattr(usage, "completion_tokens", 0) or 0), - ) - with llm_usage_context( - LLMUsageContext( - organization_id=org_id, - workspace_id=ws_id, - product_section=LLMUsageProductSection.TEST_AGENT, - ) - ): + with llm_usage_context(ctx): record_llm_usage( model, snapshot, - organization_id=org_id, + organization_id=ctx.organization_id, ) except Exception as exc: logger.debug("test agent llm usage record skipped: {}", exc) diff --git a/app/workers/tasks/agent_flowchart_jobs.py b/app/workers/tasks/agent_flowchart_jobs.py index 02431794..9b52b3c0 100644 --- a/app/workers/tasks/agent_flowchart_jobs.py +++ b/app/workers/tasks/agent_flowchart_jobs.py @@ -2,7 +2,7 @@ from __future__ import annotations -from uuid import UUID +from uuid import UUID, uuid4 from loguru import logger from sqlalchemy.orm.attributes import flag_modified @@ -77,6 +77,16 @@ def generate_agent_flowchart_task( partial.agent_flowchart_status = "completed" flag_modified(partial, "agent_flowchart") db.commit() + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted + + record_prompt_partial_ai_assisted( + partial.organization_id, + uuid4(), + workspace_id=partial.workspace_id, + mode="flowchart", + partial_id=partial.id, + model=model_str, + ) logger.info( "Agent flowchart completed for partial {} ({} nodes)", partial_id, @@ -162,6 +172,15 @@ def map_agent_flowchart_prompt_sections_task( partial.agent_flowchart_status = "completed" flag_modified(partial, "agent_flowchart") db.commit() + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted + + record_prompt_partial_ai_assisted( + partial.organization_id, + uuid4(), + workspace_id=partial.workspace_id, + mode="flowchart_map", + partial_id=partial.id, + ) logger.info("Agent flowchart prompt mapping completed for partial {}", partial_id) except Exception as exc: logger.exception( diff --git a/app/workers/tasks/evaluate_call_import_row_audio.py b/app/workers/tasks/evaluate_call_import_row_audio.py index fb746b59..c80ba6db 100644 --- a/app/workers/tasks/evaluate_call_import_row_audio.py +++ b/app/workers/tasks/evaluate_call_import_row_audio.py @@ -140,6 +140,13 @@ def evaluate_call_import_row_audio_task( result_id = f"call-import-eval:{eval_row.id}" audio_failed = False + if recording_s3_key: + from app.workers.tasks.evaluate_call_import_row_core import ( + resolve_eval_row_audio_seconds, + ) + + resolve_eval_row_audio_seconds(catalog_db, eval_row, source_row) + if audio_metrics and recording_s3_key: from app.workers.tasks.helpers.audio_evaluation import ( evaluate_audio_metrics, diff --git a/app/workers/tasks/evaluate_call_import_row_core.py b/app/workers/tasks/evaluate_call_import_row_core.py index 30148887..872fc1fe 100644 --- a/app/workers/tasks/evaluate_call_import_row_core.py +++ b/app/workers/tasks/evaluate_call_import_row_core.py @@ -2,6 +2,9 @@ from __future__ import annotations +import math +import os +import tempfile from datetime import datetime, timezone from typing import Any, List, Optional from uuid import UUID @@ -21,6 +24,8 @@ EVAL_CANCELLED_BY_USER_ERROR: str = "Evaluation cancelled by user" +_BILLING_META_KEY = "_billing" + _ALL_COLUMNS_BLOCK_MAX_CHARS = 16_000 _ALL_COLUMNS_CELL_MAX_CHARS = 4_000 @@ -507,6 +512,18 @@ def commit_terminal_row_and_rollup( catalog_db: Session | None = None, ) -> None: parent_db = catalog_db if catalog_db is not None and catalog_db is not row_db else row_db + if (eval_row.status or "").lower() == "completed": + source_row = ( + row_db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + maybe_bill_completed_eval_row_flexprice( + row_db, + evaluation, + eval_row, + source_row=source_row, + ) row_db.commit() if parent_db is not row_db: evaluation = ( @@ -574,7 +591,15 @@ def rollup_parent( reconcile_evaluation_counters(db, evaluation) db.refresh(evaluation) + _apply_parent_status_from_counters(evaluation) + + total = int(evaluation.total_rows or 0) completed = int(evaluation.completed_rows or 0) + failed = int(evaluation.failed_rows or 0) + in_progress = total - completed - failed + if in_progress > 0: + return + already_billed = int(getattr(evaluation, "billed_completed_rows", 0) or 0) delta = completed - already_billed if delta > 0: @@ -596,13 +621,12 @@ def rollup_parent( call_import_id=evaluation.call_import_id, rows_billed=delta, completed_total=completed, + total_rows=int(evaluation.total_rows or 0), metric_count=metric_count, ) if billing_accepted: evaluation.billed_completed_rows = completed - _apply_parent_status_from_counters(evaluation) - def parse_restricted_metric_uuids( restricted_metric_ids: Optional[List[str]], @@ -709,3 +733,120 @@ def row_needs_llm_phase( db, evaluation, source_row, metrics ) return bool(transcript_metrics or comparison_metrics) + + +def cache_eval_row_audio_seconds( + eval_row: CallImportEvaluationRow, + audio_seconds: int, +) -> None: + merged = dict(eval_row.metric_scores or {}) + billing = dict(merged.get(_BILLING_META_KEY) or {}) + billing["audio_seconds"] = max(0, int(audio_seconds)) + merged[_BILLING_META_KEY] = billing + eval_row.metric_scores = merged + + +def get_cached_eval_row_audio_seconds(eval_row: CallImportEvaluationRow) -> int: + merged = eval_row.metric_scores if isinstance(eval_row.metric_scores, dict) else {} + billing = merged.get(_BILLING_META_KEY) or {} + if not isinstance(billing, dict): + return 0 + try: + return max(0, int(billing.get("audio_seconds") or 0)) + except (TypeError, ValueError): + return 0 + + +def billable_audio_minutes(audio_seconds: int) -> int: + seconds = max(0, int(audio_seconds or 0)) + if seconds <= 0: + return 0 + return max(1, math.ceil(seconds / 60)) + + +def probe_recording_audio_seconds(recording_s3_key: str) -> int: + from app.services.storage.s3_service import s3_service + from app.services.usage.llm_usage import probe_audio_seconds + + key = (recording_s3_key or "").strip() + if not key: + return 0 + audio_bytes = s3_service.download_file_by_key(key) + if not audio_bytes: + return 0 + tmp_fd, tmp_path = tempfile.mkstemp(suffix=".mp3") + os.close(tmp_fd) + try: + with open(tmp_path, "wb") as handle: + handle.write(audio_bytes) + return probe_audio_seconds(tmp_path) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + + +def resolve_eval_row_audio_seconds( + db: Session, + eval_row: CallImportEvaluationRow, + source_row: CallImportRow | None, +) -> int: + cached = get_cached_eval_row_audio_seconds(eval_row) + if cached > 0: + return cached + if source_row is None: + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + if source_row is None: + return 0 + recording_s3_key = (source_row.recording_s3_key or "").strip() + if not recording_s3_key: + return 0 + seconds = probe_recording_audio_seconds(recording_s3_key) + if seconds > 0: + cache_eval_row_audio_seconds(eval_row, seconds) + return seconds + + +def maybe_bill_completed_eval_row_flexprice( + db: Session, + evaluation: CallImportEvaluation, + eval_row: CallImportEvaluationRow, + *, + source_row: CallImportRow | None = None, +) -> None: + if (eval_row.status or "").lower() != "completed": + return + from app.services.billing.flexprice_service import ( + record_call_import_recording_minutes_billed, + ) + + if source_row is None: + source_row = ( + db.query(CallImportRow) + .filter(CallImportRow.id == eval_row.call_import_row_id) + .first() + ) + if source_row is None: + return + if not (source_row.recording_s3_key or "").strip(): + return + + audio_seconds = resolve_eval_row_audio_seconds(db, eval_row, source_row) + minutes = billable_audio_minutes(audio_seconds) + if minutes <= 0: + return + + record_call_import_recording_minutes_billed( + evaluation.organization_id, + eval_row.id, + workspace_id=eval_row.workspace_id or evaluation.workspace_id, + evaluation_id=evaluation.id, + call_import_id=evaluation.call_import_id, + audio_seconds=audio_seconds, + billable_minutes=minutes, + ) diff --git a/app/workers/tasks/evaluate_studio_run_item.py b/app/workers/tasks/evaluate_studio_run_item.py index 83a4b6e8..2caa7224 100644 --- a/app/workers/tasks/evaluate_studio_run_item.py +++ b/app/workers/tasks/evaluate_studio_run_item.py @@ -17,6 +17,7 @@ MetricStudioRunResult, ) from app.services.metric_studio.metric_selection import load_studio_run_metrics +from app.services.metric_studio.run_rollup import rollup_metric_studio_run from app.services.metric_studio.source_resolver import resolve_source from app.workers.config import celery_app from app.workers.tasks.evaluate_call_import_row_core import ( @@ -40,28 +41,7 @@ def _now_utc() -> datetime: def _rollup_run(db: Session, run: MetricStudioRun) -> None: - results = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run.id) - .all() - ) - completed = sum(1 for r in results if r.status == "completed") - failed = sum(1 for r in results if r.status == "failed") - pending = sum(1 for r in results if r.status in {"pending", "running"}) - run.completed_items = completed - run.failed_items = failed - if pending: - run.status = "running" - elif failed and completed: - run.status = "partial" - run.finished_at = _now_utc() - elif failed: - run.status = "failed" - run.finished_at = _now_utc() - else: - run.status = "completed" - run.finished_at = _now_utc() - db.commit() + rollup_metric_studio_run(db, run, emit_flexprice=True, commit=True) @celery_app.task( @@ -97,6 +77,18 @@ def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: result_row.started_at = result_row.started_at or _now_utc() db.commit() + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_metric_studio_run, + ) + + usage_ctx = usage_context_for_metric_studio_run( + run, + source_kind=result_row.source_kind, + source_ref=result_row.source_ref, + result_row_id=result_row.id, + ) + sample = resolve_source( db, organization_id=run.organization_id, @@ -141,74 +133,75 @@ def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: _rollup_run(db, run) return {"status": "failed"} - ai_providers = ( - db.query(AIProvider) - .filter( - AIProvider.organization_id == run.organization_id, - AIProvider.is_active.is_(True), - ) - .all() - ) - - if audio_metrics and sample.audio_s3_key: - try: - audio_scores = evaluate_audio_metrics( - audio_s3_key=sample.audio_s3_key, - audio_metrics=audio_metrics, - result_id=f"studio:{result_row.id}", + with llm_usage_context(usage_ctx): + ai_providers = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == run.organization_id, + AIProvider.is_active.is_(True), ) - metric_scores.update(audio_scores) - except Exception as audio_err: - logger.error( - f"[MetricStudio {result_row.id}] audio evaluation failed: {audio_err}", - exc_info=True, - ) - metric_scores.update( - handle_audio_evaluation_error(audio_metrics, audio_err) - ) - - if llm_metrics and transcript: - result_id = f"studio:{result_row.id}" - production_text = (sample.transcript or "").strip() - diarised_text = (sample.diarised_transcript or "").strip() - comparison_ids = { - str(m.id) - for m in llm_metrics - if getattr(m, "compare_transcripts", False) - } - buckets = build_llm_config_buckets( - db, - llm_metrics, - overrides={}, - run_provider=None, - run_model=None, - run_llm_config=None, - run_credential_id=None, + .all() ) - try: - for _config, groups in buckets.items(): - comparison_pair = None - if bucket_needs_comparison_pair( - groups, - production_transcript=production_text, - diarised_transcript=diarised_text, - comparison_metric_ids=comparison_ids, - ): - comparison_pair = (production_text, diarised_text) - bucket_metrics = flatten_metric_groups(groups) - scores, _ = evaluate_with_llm( - transcription=transcript, - llm_metrics=bucket_metrics, - ai_providers=ai_providers, - organization_id=run.organization_id, - result_id=result_id, - db=db, - comparison_pair=comparison_pair, - metric_groups=groups, + + if audio_metrics and sample.audio_s3_key: + try: + audio_scores = evaluate_audio_metrics( + audio_s3_key=sample.audio_s3_key, + audio_metrics=audio_metrics, + result_id=f"studio:{result_row.id}", + ) + metric_scores.update(audio_scores) + except Exception as audio_err: + logger.error( + f"[MetricStudio {result_row.id}] audio evaluation failed: {audio_err}", + exc_info=True, + ) + metric_scores.update( + handle_audio_evaluation_error(audio_metrics, audio_err) ) - metric_scores.update(scores) - except Exception as llm_err: - metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) + + if llm_metrics and transcript: + result_id = f"studio:{result_row.id}" + production_text = (sample.transcript or "").strip() + diarised_text = (sample.diarised_transcript or "").strip() + comparison_ids = { + str(m.id) + for m in llm_metrics + if getattr(m, "compare_transcripts", False) + } + buckets = build_llm_config_buckets( + db, + llm_metrics, + overrides={}, + run_provider=None, + run_model=None, + run_llm_config=None, + run_credential_id=None, + ) + try: + for _config, groups in buckets.items(): + comparison_pair = None + if bucket_needs_comparison_pair( + groups, + production_transcript=production_text, + diarised_transcript=diarised_text, + comparison_metric_ids=comparison_ids, + ): + comparison_pair = (production_text, diarised_text) + bucket_metrics = flatten_metric_groups(groups) + scores, _ = evaluate_with_llm( + transcription=transcript, + llm_metrics=bucket_metrics, + ai_providers=ai_providers, + organization_id=run.organization_id, + result_id=result_id, + db=db, + comparison_pair=comparison_pair, + metric_groups=groups, + ) + metric_scores.update(scores) + except Exception as llm_err: + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) result_row.metric_scores = metric_scores flag_modified(result_row, "metric_scores") @@ -223,6 +216,19 @@ def evaluate_studio_run_item_task(self, result_row_id: str) -> dict[str, Any]: result_row.error_message = None result_row.finished_at = _now_utc() db.commit() + + from app.services.billing.flexprice_service import record_metric_studio_item_evaluated + + record_metric_studio_item_evaluated( + run.organization_id, + result_row.id, + workspace_id=run.workspace_id, + run_id=run.id, + source_kind=result_row.source_kind, + source_ref=result_row.source_ref, + metric_count=len(metric_scores), + ) + _rollup_run(db, run) return {"status": "completed", "scores": len(metric_scores)} except Exception as exc: diff --git a/app/workers/tasks/generate_evaluation_prompt_improvements.py b/app/workers/tasks/generate_evaluation_prompt_improvements.py index fd22b1fb..4c76ef6f 100644 --- a/app/workers/tasks/generate_evaluation_prompt_improvements.py +++ b/app/workers/tasks/generate_evaluation_prompt_improvements.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime, timezone -from uuid import UUID +from uuid import UUID, uuid4 from loguru import logger from sqlalchemy.orm.attributes import flag_modified @@ -125,6 +125,18 @@ def generate_evaluation_prompt_improvements_task( evaluation.prompt_improvements = prompt_improvements_state_to_db(state) flag_modified(evaluation, "prompt_improvements") db.commit() + if state.status == "completed": + from app.services.billing.flexprice_service import ( + record_call_import_prompt_improvements_generated, + ) + + record_call_import_prompt_improvements_generated( + evaluation.organization_id, + uuid4(), + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + imported_agent_id=UUID(imported_agent_id), + ) logger.info( "Prompt improvements completed for evaluation {} ({} suggestions)", evaluation_id, diff --git a/app/workers/tasks/generate_evaluation_user_insights.py b/app/workers/tasks/generate_evaluation_user_insights.py index 71f666df..c9180493 100644 --- a/app/workers/tasks/generate_evaluation_user_insights.py +++ b/app/workers/tasks/generate_evaluation_user_insights.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime, timezone -from uuid import UUID +from uuid import UUID, uuid4 from loguru import logger from sqlalchemy.orm.attributes import flag_modified @@ -100,6 +100,17 @@ def on_progress(completed: int, total: int) -> None: evaluation.user_insights = user_insights_state_to_db(state) flag_modified(evaluation, "user_insights") db.commit() + if state.status == "completed": + from app.services.billing.flexprice_service import ( + record_call_import_user_insights_generated, + ) + + record_call_import_user_insights_generated( + evaluation.organization_id, + uuid4(), + workspace_id=evaluation.workspace_id, + evaluation_id=evaluation.id, + ) except Exception as exc: # noqa: BLE001 logger.exception( "User insights generation failed for evaluation {}: {}", diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index 56a1ac80..e3c68bf5 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -588,6 +588,8 @@ def _resolve_call_duration_seconds(result) -> int: def _record_agent_call_usage(result, *, usage_ctx) -> None: if not _should_record_external_agent_call_usage(result): return + if _external_usage_already_recorded(result): + return try: from app.services.usage.llm_usage import record_call_usage @@ -605,6 +607,30 @@ def _record_agent_call_usage(result, *, usage_ctx) -> None: ) +def _external_usage_already_recorded(result) -> bool: + call_data = getattr(result, "call_data", None) + if not isinstance(call_data, dict): + return False + return bool(call_data.get("external_usage_recorded")) + + +def _record_external_agent_llm_usage(result, *, usage_ctx) -> None: + if not _should_record_external_agent_call_usage(result): + return + if _external_usage_already_recorded(result): + return + try: + from app.services.usage.external_agent_usage import record_external_agent_usage + + record_external_agent_usage(result, usage_ctx=usage_ctx) + except Exception as exc: + logger.debug( + "[EvaluatorResult {}] external agent llm usage record skipped: {}", + result.result_id, + exc, + ) + + @celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) def process_evaluator_result_task(self, result_id: str): """ @@ -732,22 +758,6 @@ def process_evaluator_result_task(self, result_id: str): llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) selected_metric_count = len(llm_metrics) + len(audio_metrics) - call_recording = _playground_call_recording(db, result) - if call_recording: - from app.services.billing.flexprice_service import ( - record_playground_call_evaluated, - ) - - evaluation_attempt_id = f"{result.id}:{self.request.id}" - record_playground_call_evaluated( - result.organization_id, - evaluation_attempt_id, - evaluator_result_id=result.id, - workspace_id=result.workspace_id, - call_short_id=call_recording.call_short_id, - metric_count=selected_metric_count, - ) - evaluation_time = None # Step 3: Audio metrics evaluation @@ -852,14 +862,22 @@ def process_evaluator_result_task(self, result_id: str): result.status = EvaluatorResultStatus.COMPLETED.value result.error_message = None _record_agent_call_usage(result, usage_ctx=usage_ctx) + _record_external_agent_llm_usage(result, usage_ctx=usage_ctx) _commit_evaluator_result(db, result) from app.services.billing.flexprice_service import ( + record_evaluator_recording_minutes_billed, + record_evaluator_run_completed, record_playground_evaluation_completed, ) call_recording = _playground_call_recording(db, result) if call_recording: + call_data = ( + call_recording.call_data + if isinstance(call_recording.call_data, dict) + else {} + ) record_playground_evaluation_completed( result.organization_id, f"{result.id}:{self.request.id}", @@ -868,7 +886,23 @@ def process_evaluator_result_task(self, result_id: str): call_short_id=call_recording.call_short_id, duration_seconds=result.duration_seconds, metric_count=len(metric_scores) or selected_metric_count, + ui_surface=call_data.get("ui_surface"), + ) + else: + record_evaluator_run_completed( + result.organization_id, + result.result_id, + workspace_id=result.workspace_id, + evaluator_id=result.evaluator_id, + evaluator_result_id=result.id, ) + if (result.audio_s3_key or "").strip(): + record_evaluator_recording_minutes_billed( + result.organization_id, + result.id, + workspace_id=result.workspace_id, + duration_seconds=result.duration_seconds, + ) total_time = time.time() - task_start_time logger.info( diff --git a/app/workers/tasks/run_evaluator.py b/app/workers/tasks/run_evaluator.py index a82f0199..3d8fba34 100644 --- a/app/workers/tasks/run_evaluator.py +++ b/app/workers/tasks/run_evaluator.py @@ -71,18 +71,20 @@ def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): result.call_event = "task_started" db.commit() - loop = asyncio.get_event_loop() - if loop.is_closed(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - bridge_result = loop.run_until_complete( - test_agent_bridge_service.bridge_test_agent_to_voice_agent( - evaluator_id=evaluator_uuid, - evaluator_result_id=result_uuid, - organization_id=evaluator.organization_id, - db=db, + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + bridge_result = loop.run_until_complete( + test_agent_bridge_service.bridge_test_agent_to_voice_agent( + evaluator_id=evaluator_uuid, + evaluator_result_id=result_uuid, + organization_id=evaluator.organization_id, + db=db, + ) ) - ) + finally: + loop.close() + asyncio.set_event_loop(None) db.refresh(result) @@ -91,15 +93,6 @@ def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): db.commit() - from app.services.billing.flexprice_service import record_evaluator_run_completed - - record_evaluator_run_completed( - evaluator.organization_id, - result.result_id, - workspace_id=evaluator.workspace_id, - evaluator_id=evaluator.id, - ) - return { "evaluator_id": evaluator_id, "result_id": evaluator_result_id, @@ -119,10 +112,67 @@ def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): raise elif has_voice_bundle: - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = "Standard voice agent flow not yet implemented for evaluator runs" - db.commit() - return {"error": "Standard flow not implemented"} + from app.models.database import Persona, Scenario + from app.services.testing.llm_to_llm_evaluator_simulation import ( + run_llm_to_llm_evaluator_simulation, + ) + + persona = ( + db.query(Persona).filter(Persona.id == evaluator.persona_id).first() + if evaluator.persona_id + else None + ) + scenario = ( + db.query(Scenario).filter(Scenario.id == evaluator.scenario_id).first() + if evaluator.scenario_id + else None + ) + if not persona or not scenario: + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = "Evaluator requires persona and scenario for voice-bundle simulation" + db.commit() + return {"error": "Missing persona or scenario"} + + try: + result.status = EvaluatorResultStatus.CALL_INITIATING.value + result.call_event = "llm_simulation_started" + db.commit() + + run_llm_to_llm_evaluator_simulation( + evaluator=evaluator, + result=result, + agent=agent, + persona=persona, + scenario=scenario, + organization_id=evaluator.organization_id, + db=db, + ) + result.status = EvaluatorResultStatus.QUEUED.value + result.call_event = "llm_simulation_completed" + db.commit() + + from app.workers.celery_app import process_evaluator_result_task + + task = process_evaluator_result_task.delay(str(result.id)) + result.celery_task_id = task.id + db.commit() + + return { + "evaluator_id": evaluator_id, + "result_id": evaluator_result_id, + "status": "simulated", + "provider_platform": "internal", + } + except Exception as sim_error: + logger.error( + f"[RunEvaluator {evaluator.evaluator_id}] LLM simulation error: {sim_error}", + exc_info=True, + ) + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = str(sim_error) + result.call_event = "llm_simulation_error" + db.commit() + raise else: logger.error(f"[RunEvaluator {evaluator.evaluator_id}] Agent missing required configuration") diff --git a/config.docker.yml b/config.docker.yml index 3d169129..b06558b4 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -5,7 +5,7 @@ app: name: "EfficientAI Voice AI Evaluation Platform" version: "0.1.0" - debug: true + debug: false secret_key: "your-secret-key-here-change-in-production" frontend_base_url: "http://localhost:8000" @@ -15,12 +15,17 @@ server: port: 8000 # Operational endpoints (/metrics). /health is always open for load balancers. +# Not Spring Boot Actuator — /metrics is IP-gated via trusted_ips below. # Add VPC CIDRs here if Prometheus scrapes /metrics from inside the VPC. operational: public: false trusted_ips: - "10.0.0.0/8" +security: + csp_enabled: true + csp_report_only: false + # Database Configuration (Docker service name) database: url: "postgresql://efficientai:password@db:5432/efficientai" @@ -99,7 +104,7 @@ auth: local_password: # Lifetime of the Bearer tokens minted at POST /auth/login (in minutes). # Keep this short; clients re-authenticate silently. - token_ttl_minutes: 720 # 12 hours + token_ttl_minutes: 15 # Turn this off in Cloud SaaS to block self-serve signup. allow_signup: true @@ -135,4 +140,10 @@ judge_alignment: # Enterprise License (JWT signed with RS256). Unlocks gated features like # oidc_sso, mfa_enforce, audit_export, voice_playground, gepa_optimization. # license: -# key: "eyJhbGciOi..." \ No newline at end of file +# key: "eyJhbGciOi..." + +# Flexprice usage metering (optional). api_key from FLEXPRICE_API_KEY env. +flexprice: + enabled: true + api_host: "https://api.cloud.flexprice.io/v1" + # api_key: use FLEXPRICE_API_KEY in .env / container env \ No newline at end of file diff --git a/config.yml.example b/config.yml.example index b361321b..bf268c83 100644 --- a/config.yml.example +++ b/config.yml.example @@ -17,12 +17,20 @@ server: port: 8000 # Operational endpoints (/metrics). /health is always open for ALB/kube probes. +# This is FastAPI, not Spring Boot Actuator — scanners may flag /health or /metrics +# under an "Actuator" template. /metrics is IP-gated; /health returns minimal status only. # Include VPC CIDRs in trusted_ips if Prometheus scrapes /metrics from inside the VPC. operational: public: false trusted_ips: - "10.0.0.0/8" +# HTTP security headers (CSP, X-Frame-Options, etc.) +security: + csp_enabled: true + csp_report_only: false # false = enforcing Content-Security-Policy header (required for compliance scans) + # csp_policy: "default-src 'self'; ..." # optional override (defaults include Vapi/Daily, Retell/LiveKit, ElevenLabs) + # Database Configuration database: # Legacy single-DB mode: use one database (e.g. efficientai). Sharding off (default). @@ -251,11 +259,20 @@ judge_alignment: # Flexprice usage-based billing (optional). When disabled or api_key is unset, # no SDK calls are made and the app behaves as today. Cloud SaaS: set enabled # true and provide FLEXPRICE_API_KEY (env var overrides api_key below). +# Pytest always mocks Flexprice locally (EFFICIENTAI_PYTEST=1); never hits this API. +# Setup guide: docs/billing/flexprice-saas-setup.md +# Bootstrap meters: python scripts/setup_flexprice_meters.py +# Plan pricing checklist: python scripts/setup_flexprice_meters.py --plan-guide # flexprice: # enabled: false # api_key: null # api_host: "https://us.api.flexprice.io/v1" -# Env override: FLEXPRICE_ENABLED, FLEXPRICE_API_KEY, FLEXPRICE_API_HOST +# auto_subscribe: false +# default_plan_id: null +# default_currency: "usd" +# default_billing_period: "MONTHLY" +# Env override: FLEXPRICE_ENABLED, FLEXPRICE_API_KEY, FLEXPRICE_API_HOST, +# FLEXPRICE_AUTO_SUBSCRIBE, FLEXPRICE_DEFAULT_PLAN_ID # Debug: FLEXPRICE_VERBOSE=1 logs every skipped event (when inactive or verbose) # Call-import worker concurrency (Redis fair-share for evaluations). diff --git a/docker-compose.yml b/docker-compose.yml index 1d6f0d0b..ee5c01b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -120,6 +120,9 @@ services: command: eai telephony-worker --config /app/config.yml --host 0.0.0.0 --port 8001 worker: + # Local dev: `eai start-all` already runs Celery workers. Skip this service + # when using start-all, or rebuild the image (`docker compose build worker`) + # so billing code matches your checkout — stale images emit deprecated events. # Pre-built image from GitHub Container Registry # Use EFFICIENTAI_VERSION env var to pin to a specific version (e.g., 1.0.0) image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest} diff --git a/docs/billing/flexprice-saas-setup.md b/docs/billing/flexprice-saas-setup.md new file mode 100644 index 00000000..b7f0944f --- /dev/null +++ b/docs/billing/flexprice-saas-setup.md @@ -0,0 +1,130 @@ +# Flexprice SaaS setup (EfficientAI) + +Usage-based billing for cloud customers. Code ingests **completion-only** events; Flexprice meters aggregate usage; plan **usage charges** turn meters into invoice lines. + +## Architecture + +``` +App (completion events) → Flexprice meters (SUM/COUNT) → Plan usage charges → Customer invoice + ↑ + provision_billing_customer on signup (ensure_customer) +``` + +Every billable event includes `workspace_id`, `feature`, and `quantity` (plus `billable_minutes` for voice). Audit IDs (`evaluation_id`, `audio_seconds`, `run_id`, etc.) are included for support traceability and do not affect meter aggregation. + +## 1. Enable metering + +In `config.yml` (or env): + +```yaml +flexprice: + enabled: true + api_host: "https://us.api.flexprice.io/v1" # or api.cloud.flexprice.io +``` + +Set `FLEXPRICE_API_KEY` in the environment (never commit keys). + +Signup and API-key flows call `provision_billing_customer()` so each org exists in Flexprice as `external_customer_id = organization.id`. When `auto_subscribe` is enabled, new orgs also receive a subscription on `default_plan_id` (Platform plan). + +```yaml +flexprice: + enabled: true + api_host: "https://api.cloud.flexprice.io/v1" + auto_subscribe: true + default_plan_id: "plan_01KVT8BTT0HRB419QVCTNHS9RV" + default_currency: "usd" + default_billing_period: "MONTHLY" +``` + +Env overrides: `FLEXPRICE_AUTO_SUBSCRIBE`, `FLEXPRICE_DEFAULT_PLAN_ID`. Leave `auto_subscribe: false` for self-hosted OSS (customer optional, no subscription). + +## 2. Bootstrap meters and features + +```bash +python scripts/setup_flexprice_meters.py +python scripts/setup_flexprice_meters.py --repair-call-imports # if batch_created meters were duplicated +python scripts/setup_flexprice_meters.py --repair-playground # archive call_evaluated + duplicate evaluation_completed meters +python scripts/setup_flexprice_meters.py --repair-features --dry-run # preview stale feature→meter links +python scripts/setup_flexprice_meters.py --repair-features # repoint license features (delete + recreate) +``` + +Flexprice feature **PUT** cannot change `meter_id`. `--repair-features` deletes and recreates each `LICENSE_FEATURES` row on the canonical published meter. **Plan usage charges on meters are not affected.** + +Print the plan pricing checklist (no API): + +```bash +python scripts/setup_flexprice_meters.py --plan-guide +``` + +## 3. Call imports — separate meters (recommended) + +| Plan usage line | Meter | Aggregation | Typical price | +|-----------------|-------|-------------|---------------| +| Call import evaluations | Call Import Evaluations | SUM `quantity` | Per evaluated row | +| Call import audio | Call Import Recording Minutes | SUM `billable_minutes` | Per audio minute | +| Call import PDFs | Call Import PDF Reports | SUM `quantity` | Per report | +| Imported rows (cap) | Call Imports (feature) | SUM `quantity` | **$0** — use for tier included rows | + +Do **not** put a heavy per-row price on both batch import and evaluation — customers would feel double-charged. Batch meter is for entitlements; eval + audio + PDF are paid lines. + +**Audio event:** `call_import.recording_minutes_billed` fires when a row completes evaluation **and has a recording** (not on import alone). + +## 4. Other products (paid usage charges) + +| Product | Meter event | Unit | +|---------|-------------|------| +| Agent playground | `playground.evaluation_completed` | billable minutes | +| Test agent API | `test_agent.conversation_ended` | billable minutes | +| Voice playground | `blind_test.response_submitted`, `tts.sample_synthesized`, `tts.report_completed` | per response / sample / report | +| Evaluators | `evaluator.run_completed` + `evaluator.recording_minutes_billed` (when audio) | per run + per audio minute | +| GEPA | `prompt_optimization.run_completed` | per candidate (SUM quantity) | +| Judge alignment | `judge_alignment.run_completed` | per sample scored (SUM quantity) | +| Metrics AI assist | `metrics.ai_assist` | per request | +| Metric studio | `metric_studio.run_completed` | completed items (SUM quantity) | +| Scenario AI | `scenario.ai_text_generated` | per generation | +| Prompt partials AI | `prompt_partial.ai_assisted` | per request (generate / improve / flowchart / map) | +| Call import user insights | `call_import.user_insights_generated` | per completed generation | +| Call import prompt improvements | `call_import.prompt_improvements_generated` | per completed generation | +| Persona prompt generation | `persona.prompt_generated` | per generation | +| Agent test setup | `agent.test_setup_generated` | per API call (test prompt / scenarios / full setup) | + +Observability is **not billed** — no plan usage charges for observability events. + +## 5. Configure plan in Flexprice UI + +1. Open your SaaS **Plan**. +2. Add **Usage charge** for each paid meter above (pick meter by name, set unit price). +3. Click **Sync Usage Charges** on the plan. +4. Assign plan to customers / refresh subscriptions after meter or price changes. + +After `--repair-call-imports`, re-sync subscriptions if price IDs changed. + +## 6. Smoke tests before launch + +| Action | Expected event | Check quantity | +|--------|----------------|----------------| +| Materialize call import | `call_import.batch_created` | row count | +| Complete eval pass | `call_import.evaluation_completed` | delta rows | +| Eval row with ~90s audio | `call_import.recording_minutes_billed` | 2 minutes | +| Generate PDF | `call_import.pdf_report_generated` | 1 | +| Complete playground eval | `playground.evaluation_completed` | billable minutes | +| Complete evaluator run | `evaluator.run_completed` | 1 | +| Evaluator run with audio | `evaluator.recording_minutes_billed` | billable minutes | +| Complete GEPA run | `prompt_optimization.run_completed` | candidate count | +| Complete judge alignment | `judge_alignment.run_completed` | samples scored | +| Prompt partial AI generate | `prompt_partial.ai_assisted` | 1 (`mode=generate`) | +| Call import user insights completes | `call_import.user_insights_generated` | 1 | +| Persona prompt generate | `persona.prompt_generated` | 1 | +| Agent test setup generate | `agent.test_setup_generated` | 1 | + +Use Flexprice **Price Lookup** with `external_customer_id = organization UUID`. + +## 7. Test isolation + +Pytest sets `EFFICIENTAI_PYTEST=1` and mocks all Flexprice I/O. Full suite never hits the live dashboard. + +## 8. Not yet wired (post-launch) + +- Flexprice entitlements replacing JWT license limits (`app/core/license.py`) +- Backfill existing orgs as Flexprice customers +- Multiple plan tiers at signup (Basic $100, reference codes, Stripe checkout) diff --git a/env.example b/env.example index 40b2f1a5..53a08814 100644 --- a/env.example +++ b/env.example @@ -41,6 +41,10 @@ CORS_ORIGINS=["http://localhost:3000", "http://localhost:8000"] API_KEY_HEADER=X-API-Key RATE_LIMIT_PER_MINUTE=60 +# HTTP security headers (see config.yml.example security section) +CSP_ENABLED=true +CSP_REPORT_ONLY=false + # ----------------------------------------------------------------------------- # Authentication (pluggable auth providers) # ----------------------------------------------------------------------------- diff --git a/frontend/src/components/VoiceAgent.tsx b/frontend/src/components/VoiceAgent.tsx index 07253840..07650b79 100644 --- a/frontend/src/components/VoiceAgent.tsx +++ b/frontend/src/components/VoiceAgent.tsx @@ -22,6 +22,7 @@ interface VoiceAgentProps { agentId?: string customEndpoint?: string customEndpointLabel?: string + billingSurface?: 'agents_talk' | 'agent_playground' onSessionSaved?: () => void compact?: boolean sidebarLayout?: boolean @@ -30,7 +31,7 @@ interface VoiceAgentProps { type TranscriptEntry = { role: 'user' | 'agent'; content: string; timestamp: string } -export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpoint, customEndpointLabel, onSessionSaved, compact = false, sidebarLayout = false, agentDisplayName }: VoiceAgentProps) { +export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpoint, customEndpointLabel, billingSurface, onSessionSaved, compact = false, sidebarLayout = false, agentDisplayName }: VoiceAgentProps) { const { selectedAgent } = useAgentStore() // Use agentId prop if provided, otherwise fall back to selectedAgent from store @@ -419,6 +420,7 @@ export default function VoiceAgent({ personaId, scenarioId, agentId, customEndpo } if (!customEndpoint && personaId) params.append('persona_id', personaId) if (!customEndpoint && scenarioId) params.append('scenario_id', scenarioId) + if (!customEndpoint && billingSurface) params.append('ui_surface', billingSurface) if (!customEndpoint && params.toString()) { endpointUrl += `?${params.toString()}` diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index c432705b..f8b74cc1 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,9 +1,12 @@ import axios, { AxiosInstance } from 'axios' import { + clearAuthSession, getApiErrorDetail, + hasRevocableUserCredentials, isOrganizationAccessDenied, organizationAccessDeniedMessage, redirectToLoginWithMessage, + type UserSessionCredentials, } from './authSession' import type { GenerateScenariosFromPromptParams, @@ -585,6 +588,7 @@ class ApiClient { requestUrl.includes('/auth/login') || requestUrl.includes('/auth/signup') || requestUrl.includes('/auth/refresh') || + requestUrl.includes('/auth/logout') || requestUrl.includes('/auth/config') const detail = getApiErrorDetail(error) @@ -611,10 +615,7 @@ class ApiClient { } } - localStorage.removeItem('apiKey') - localStorage.removeItem('accessToken') - localStorage.removeItem('refreshToken') - localStorage.removeItem('authUser') + clearAuthSession() window.location.href = '/login' } return Promise.reject(error) @@ -832,6 +833,43 @@ class ApiClient { return response.data } + async platformLogout(accessToken?: string | null): Promise<{ success: boolean; admin_id: string }> { + const token = accessToken ?? localStorage.getItem('platformAccessToken') + const headers: Record = { 'Content-Type': 'application/json' } + if (token) { + headers.Authorization = `Bearer ${token}` + } + const response = await axios.post( + `${API_BASE_URL}/api/v1/platform/auth/logout`, + {}, + { headers }, + ) + return response.data + } + + /** + * Voluntary logout: revoke JWT blacklist + refresh token on the server. + * Uses raw axios so credentials survive after local session is cleared. + */ + revokeUserSessionBestEffort(credentials: UserSessionCredentials): void { + if (!hasRevocableUserCredentials(credentials)) { + return + } + const auth = { accessToken: credentials.accessToken, apiKey: credentials.apiKey } + void this.revokeUserSession(credentials.refreshToken, auth) + .catch(() => this.revokeUserSession(credentials.refreshToken, auth)) + .catch(() => {}) + } + + revokePlatformSessionBestEffort(accessToken?: string | null): void { + if (!accessToken) { + return + } + void this.platformLogout(accessToken) + .catch(() => this.platformLogout(accessToken)) + .catch(() => {}) + } + async getPlatformOrganizationStats(): Promise { const response = await axios.get(`${API_BASE_URL}/api/v1/platform/organizations/stats`, { headers: this.platformHeaders(), @@ -942,10 +980,32 @@ class ApiClient { return response.data } - async logout(refreshToken?: string | null): Promise<{ success: boolean; auth_method: string }> { - const response = await this.client.post('/api/v1/auth/logout', { - refresh_token: refreshToken || localStorage.getItem('refreshToken') || undefined, - }) + async logout( + refreshToken?: string | null, + credentials?: { accessToken?: string | null; apiKey?: string | null }, + ): Promise<{ success: boolean; auth_method: string }> { + return this.revokeUserSession(refreshToken, credentials) + } + + private async revokeUserSession( + refreshToken?: string | null, + credentials?: { accessToken?: string | null; apiKey?: string | null }, + ): Promise<{ success: boolean; auth_method: string }> { + const headers: Record = { 'Content-Type': 'application/json' } + const accessToken = credentials?.accessToken ?? localStorage.getItem('accessToken') + const apiKey = credentials?.apiKey ?? localStorage.getItem('apiKey') + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}` + } else if (apiKey) { + headers['X-API-Key'] = apiKey + } + const response = await axios.post( + `${API_BASE_URL}/api/v1/auth/logout`, + { + refresh_token: refreshToken ?? localStorage.getItem('refreshToken') ?? undefined, + }, + { headers }, + ) return response.data } @@ -966,6 +1026,7 @@ class ApiClient { async switchOrganization(organizationId: string): Promise { const response = await this.client.post('/api/v1/auth/switch-org', { organization_id: organizationId, + refresh_token: localStorage.getItem('refreshToken') || undefined, }) return response.data } @@ -3809,6 +3870,7 @@ class ApiClient { metadata?: Record retell_llm_dynamic_variables?: Record custom_sip_headers?: Record + ui_surface?: 'agents_talk' | 'agent_playground' }): Promise<{ call_type: string access_token?: string diff --git a/frontend/src/lib/authSession.ts b/frontend/src/lib/authSession.ts index 4a1e673c..ecfbf6e5 100644 --- a/frontend/src/lib/authSession.ts +++ b/frontend/src/lib/authSession.ts @@ -1,5 +1,11 @@ export const AUTH_REDIRECT_MESSAGE_KEY = 'authRedirectMessage' +export type UserSessionCredentials = { + accessToken?: string | null + refreshToken?: string | null + apiKey?: string | null +} + export function getApiErrorDetail(error: unknown): string | undefined { const detail = (error as { response?: { data?: { detail?: unknown } } })?.response?.data ?.detail @@ -23,6 +29,16 @@ export function clearAuthSession(): void { localStorage.removeItem('activeWorkspaceId') } +/** True when a voluntary logout can still revoke something server-side. */ +export function hasRevocableUserCredentials(credentials: UserSessionCredentials): boolean { + return Boolean(credentials.accessToken || credentials.apiKey || credentials.refreshToken) +} + +export function clearPlatformAdminSession(): void { + localStorage.removeItem('platformAccessToken') + localStorage.removeItem('platformAdminUser') +} + export function redirectToLoginWithMessage(message: string): void { sessionStorage.setItem(AUTH_REDIRECT_MESSAGE_KEY, message) clearAuthSession() diff --git a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx index 1afd845d..121f1839 100644 --- a/frontend/src/pages/agents/components/AgentTalkSidebar.tsx +++ b/frontend/src/pages/agents/components/AgentTalkSidebar.tsx @@ -198,7 +198,7 @@ export default function AgentTalkSidebar({ setIsConnected(false) setActiveSpeaker(null) }) - const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {}, ui_surface: 'agents_talk' }) await client.startCall({ accessToken: webCall.access_token!, callId: webCall.call_id, @@ -242,7 +242,7 @@ export default function AgentTalkSidebar({ }) } }) - const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {}, ui_surface: 'agents_talk' }) callShortIdRef.current = webCall.call_short_id ?? null const vapiCall = await client.start(agent.voice_ai_agent_id!) if (callShortIdRef.current && vapiCall?.id) { @@ -253,7 +253,7 @@ export default function AgentTalkSidebar({ } } } else if (isElevenLabs) { - const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {}, ui_surface: 'agents_talk' }) if (!webCall.signed_url) throw new Error('No signed URL') const conversation = await Conversation.startSession({ signedUrl: webCall.signed_url, @@ -297,7 +297,7 @@ export default function AgentTalkSidebar({ elevenLabsConversationRef.current = conversation } else if (isSmallest) { const { AtomsClient } = await import('atoms-client-sdk') - const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {} }) + const webCall = await apiClient.createWebCall({ agent_id: agent.id, metadata: {}, ui_surface: 'agents_talk' }) if (!webCall.access_token || !webCall.host) throw new Error('Missing Smallest credentials') const client = new AtomsClient() smallestClientRef.current = client @@ -375,7 +375,7 @@ export default function AgentTalkSidebar({ {mode === 'test_agent' ? ( canTalkTest ? (
- +
) : (
diff --git a/frontend/src/pages/auth/Login.tsx b/frontend/src/pages/auth/Login.tsx index 35837549..eea8a92b 100644 --- a/frontend/src/pages/auth/Login.tsx +++ b/frontend/src/pages/auth/Login.tsx @@ -6,6 +6,7 @@ import type { AuthConfigResponse, AuthProviderConfig, LoginOrgOption } from '../ import { buildAuthorizeUrl } from '../../lib/oidc' import { PASSWORD_POLICY_HINT, validatePasswordPolicy } from '../../lib/passwordPolicy' import { consumeAuthRedirectMessage } from '../../lib/authSession' +import { getApiErrorMessage } from '../../lib/apiErrors' import { consumePendingInviteToken, getPendingInviteToken, @@ -219,8 +220,8 @@ export default function Login() { consumePendingInviteToken() setSession(res.access_token, res.user, res.refresh_token) navigate('/') - } catch (err: any) { - setError(err?.response?.data?.detail || 'Sign up failed') + } catch (err: unknown) { + setError(getApiErrorMessage(err, 'Sign up failed')) } finally { setIsLoading(false) } diff --git a/frontend/src/pages/auth/SelectOrganization.tsx b/frontend/src/pages/auth/SelectOrganization.tsx index ac7a3eef..62e4eca6 100644 --- a/frontend/src/pages/auth/SelectOrganization.tsx +++ b/frontend/src/pages/auth/SelectOrganization.tsx @@ -5,6 +5,7 @@ import { Building2, Loader2 } from 'lucide-react' import Logo from '../../components/Logo' import { Card, CardBody } from '@heroui/react' import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' import { useAuthStore } from '../../store/authStore' export default function SelectOrganization() { @@ -32,8 +33,8 @@ export default function SelectOrganization() { try { await switchOrg(orgId) navigate('/', { replace: true }) - } catch (err: any) { - setError(err?.response?.data?.detail || 'Could not enter the selected organization') + } catch (err: unknown) { + setError(getApiErrorMessage(err, 'Could not enter the selected organization')) } finally { setSwitchingTo(null) } diff --git a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx index 55b70c81..be8a2d56 100644 --- a/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx +++ b/frontend/src/pages/callImports/CallImportEvaluationDetail.tsx @@ -111,6 +111,10 @@ import { evaluationBulkOperationLabel, type BulkEvaluationOperation, } from './evaluationBulkOperation' +import { + formatApiErrorDetail, + isReservedMetricScoreKey, +} from './metricScoresMeta' const PIE_COLORS = [ '#6366f1', @@ -1282,12 +1286,16 @@ export default function CallImportEvaluationDetail() { setRerunMetricsOpen(false) }, onError: (err: any) => { + const fallback = 'Failed to re-run the selected metrics.' setRerunError( err?.response?.status === 409 - ? err?.response?.data?.detail || bulkOperationConflictMessage - : err?.response?.data?.detail || + ? formatApiErrorDetail( + err?.response?.data?.detail, + bulkOperationConflictMessage, + ) + : formatApiErrorDetail(err?.response?.data?.detail, fallback) || err?.message || - 'Failed to re-run the selected metrics.', + fallback, ) }, }) @@ -1477,7 +1485,7 @@ export default function CallImportEvaluationDetail() { const scores = row.metric_scores if (!scores || typeof scores !== 'object') continue for (const [metricId, entry] of Object.entries(scores)) { - if (!metricId) continue + if (!metricId || isReservedMetricScoreKey(metricId)) continue if (childrenInGroups.has(metricId)) continue const fallbackName = entry && typeof entry === 'object' && 'metric_name' in entry diff --git a/frontend/src/pages/callImports/metricScoresMeta.ts b/frontend/src/pages/callImports/metricScoresMeta.ts new file mode 100644 index 00000000..a3fdfa3c --- /dev/null +++ b/frontend/src/pages/callImports/metricScoresMeta.ts @@ -0,0 +1,33 @@ +/** Keys stored in ``metric_scores`` that are not real Metric IDs. */ +const RESERVED_METRIC_SCORE_KEYS = new Set(['_billing', '__discovered_metrics__']) + +export function isReservedMetricScoreKey(key: string): boolean { + if (!key) return true + if (RESERVED_METRIC_SCORE_KEYS.has(key)) return true + if (key.endsWith('__discovered')) return true + return false +} + +export function formatApiErrorDetail(detail: unknown, fallback: string): string { + if (typeof detail === 'string' && detail.trim()) return detail + if (Array.isArray(detail)) { + const parts = detail + .map((item) => { + if (item && typeof item === 'object' && 'msg' in item) { + const msg = String((item as { msg?: unknown }).msg ?? '') + const loc = (item as { loc?: unknown }).loc + if (Array.isArray(loc) && loc.length > 0) { + return `${loc.join('.')}: ${msg}` + } + return msg + } + return typeof item === 'string' ? item : null + }) + .filter((part): part is string => !!part && part.trim().length > 0) + if (parts.length > 0) return parts.join('; ') + } + if (detail && typeof detail === 'object' && 'msg' in detail) { + return String((detail as { msg?: unknown }).msg ?? fallback) + } + return fallback +} diff --git a/frontend/src/pages/platform/PlatformLogin.tsx b/frontend/src/pages/platform/PlatformLogin.tsx index d9c97c1c..389e99a2 100644 --- a/frontend/src/pages/platform/PlatformLogin.tsx +++ b/frontend/src/pages/platform/PlatformLogin.tsx @@ -4,6 +4,7 @@ import { AlertCircle, Eye, EyeOff } from 'lucide-react' import { Button, Chip } from '@heroui/react' import Logo from '../../components/Logo' import { apiClient } from '../../lib/api' +import { getApiErrorMessage } from '../../lib/apiErrors' import { usePlatformAdminStore } from '../../store/platformAdminStore' export default function PlatformLogin() { @@ -42,7 +43,7 @@ export default function PlatformLogin() { 'Is the backend running?', ) } else { - setError(detail || 'Sign in failed') + setError(getApiErrorMessage(err, 'Sign in failed')) } } finally { setIsLoading(false) diff --git a/frontend/src/pages/playground/agent/AgentPlayground.tsx b/frontend/src/pages/playground/agent/AgentPlayground.tsx index e790766e..66d195dc 100644 --- a/frontend/src/pages/playground/agent/AgentPlayground.tsx +++ b/frontend/src/pages/playground/agent/AgentPlayground.tsx @@ -294,6 +294,7 @@ export default function AgentPlayground() { metadata: {}, retell_llm_dynamic_variables: {}, custom_sip_headers: {}, + ui_surface: 'agent_playground', }) if (!webCallResponse.call_id || !webCallResponse.access_token) { @@ -331,6 +332,7 @@ export default function AgentPlayground() { const webCallResponse = await apiClient.createWebCall({ agent_id: fullAgent.id, metadata: {}, + ui_surface: 'agent_playground', }) if (webCallResponse.call_short_id) { @@ -435,6 +437,7 @@ export default function AgentPlayground() { const webCallResponse = await apiClient.createWebCall({ agent_id: fullAgent.id, metadata: {}, + ui_surface: 'agent_playground', }) if (webCallResponse.call_short_id) { @@ -545,6 +548,7 @@ export default function AgentPlayground() { const webCallResponse = await apiClient.createWebCall({ agent_id: fullAgent.id, metadata: {}, + ui_surface: 'agent_playground', }) if (webCallResponse.call_short_id) { @@ -1574,7 +1578,7 @@ export default function AgentPlayground() {
- +
diff --git a/frontend/src/store/authStore.ts b/frontend/src/store/authStore.ts index 4db27106..2b37a2b0 100644 --- a/frontend/src/store/authStore.ts +++ b/frontend/src/store/authStore.ts @@ -1,5 +1,6 @@ import { create } from 'zustand' import { apiClient } from '../lib/api' +import { clearAuthSession } from '../lib/authSession' import { useWorkspaceStore } from './workspaceStore' /** @@ -113,17 +114,15 @@ export const useAuthStore = create((set, get) => { }, logout: () => { - const refreshToken = get().refreshToken - apiClient.logout(refreshToken).catch(() => {}) - apiClient.clearApiKey() - apiClient.clearAccessToken() - apiClient.clearRefreshToken() - localStorage.removeItem(STORAGE_API_KEY) - localStorage.removeItem(STORAGE_ACCESS_TOKEN) - localStorage.removeItem(STORAGE_REFRESH_TOKEN) - localStorage.removeItem(STORAGE_USER) + const credentials = { + accessToken: get().accessToken, + refreshToken: get().refreshToken, + apiKey: get().apiKey, + } + clearAuthSession() useWorkspaceStore.getState().clearActiveWorkspaceId() set({ apiKey: null, accessToken: null, refreshToken: null, user: null }) + apiClient.revokeUserSessionBestEffort(credentials) }, validate: async () => { diff --git a/frontend/src/store/platformAdminStore.ts b/frontend/src/store/platformAdminStore.ts index 1b79b15c..92e86a2d 100644 --- a/frontend/src/store/platformAdminStore.ts +++ b/frontend/src/store/platformAdminStore.ts @@ -1,4 +1,6 @@ import { create } from 'zustand' +import { apiClient } from '../lib/api' +import { clearPlatformAdminSession } from '../lib/authSession' type PlatformAdminUser = { id: string @@ -24,7 +26,7 @@ function readStoredAdmin(): PlatformAdminUser | null { } } -export const usePlatformAdminStore = create((set) => { +export const usePlatformAdminStore = create((set, get) => { const storedToken = localStorage.getItem(STORAGE_TOKEN) const storedAdmin = readStoredAdmin() @@ -37,9 +39,10 @@ export const usePlatformAdminStore = create((set) => { set({ accessToken: token, admin }) }, logout: () => { - localStorage.removeItem(STORAGE_TOKEN) - localStorage.removeItem(STORAGE_ADMIN) + const accessToken = get().accessToken + clearPlatformAdminSession() set({ accessToken: null, admin: null }) + apiClient.revokePlatformSessionBestEffort(accessToken) }, } }) diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 4b233ae6..228cd0cb 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -21,7 +21,6 @@ "noFallthroughCasesInSwitch": true, /* Path aliases */ - "baseUrl": ".", "paths": { "@/*": ["./src/*"] } diff --git a/scripts/setup_flexprice_meters.py b/scripts/setup_flexprice_meters.py index 533526c7..0531d2a5 100644 --- a/scripts/setup_flexprice_meters.py +++ b/scripts/setup_flexprice_meters.py @@ -1,8 +1,21 @@ #!/usr/bin/env python3 -"""Create Flexprice features and meters for EfficientAI usage metering catalog.""" +"""Bootstrap Flexprice meters and license features for EfficientAI SaaS billing. + +Completion-only policy: only billable *completed* events get meters here. Track-only +events (*_started, *_requested, *_created) are not catalogued. + +Call imports use multiple meters (separate plan usage charges): + - call_import.batch_created — feature primary; entitlements / included row cap ($0 typical) + - call_import.evaluation_completed — paid per evaluated row + - call_import.recording_minutes_billed — paid per audio minute + - call_import.pdf_report_generated — paid per PDF + +See docs/billing/flexprice-saas-setup.md for plan pricing and customer onboarding. +""" from __future__ import annotations +import argparse import json import sys from pathlib import Path @@ -15,76 +28,353 @@ CONFIG_PATH = Path(__file__).resolve().parent.parent / "config.yml" -# (event_name, display_name, aggregation_type, aggregation_field|None) +CALL_IMPORT_BATCH_EVENT = "call_import.batch_created" +CALL_IMPORT_BATCH_METER_NAME = "Call Imports" +CALL_IMPORT_BATCH_AGG_TYPE = "SUM" +CALL_IMPORT_BATCH_AGG_FIELD = "quantity" + +AGENT_PLAYGROUND_PRIMARY_EVENT = "playground.evaluation_completed" +VOICE_PLAYGROUND_PRIMARY_EVENT = "blind_test.response_submitted" +GEPA_PRIMARY_EVENT = "prompt_optimization.run_completed" +EVALUATOR_RUN_COMPLETED_EVENT = "evaluator.run_completed" +JUDGE_ALIGNMENT_PRIMARY_EVENT = "judge_alignment.run_completed" +METRICS_AI_ASSIST_EVENT = "metrics.ai_assist" +METRIC_STUDIO_PRIMARY_EVENT = "metric_studio.run_completed" +SCENARIO_AI_TEXT_EVENT = "scenario.ai_text_generated" +PROMPT_PARTIAL_AI_ASSISTED_EVENT = "prompt_partial.ai_assisted" +CALL_IMPORT_USER_INSIGHTS_EVENT = "call_import.user_insights_generated" +CALL_IMPORT_PROMPT_IMPROVEMENTS_EVENT = "call_import.prompt_improvements_generated" +PERSONA_PROMPT_GENERATED_EVENT = "persona.prompt_generated" +AGENT_TEST_SETUP_GENERATED_EVENT = "agent.test_setup_generated" + +# Standalone meters (not owned by a license feature primary event). +# Each row: (event_name, display_name, aggregation_type, aggregation_field|None) METERS: list[tuple[str, str, str, str | None]] = [ - # Voice playground - ("blind_test.share_created", "Blind Test Share Created", "COUNT", None), - ("blind_test.response_submitted", "Blind Test Response Submitted", "COUNT", None), - ("tts.generation_started", "TTS Generation Started", "COUNT", None), - ("tts.sample_synthesized", "TTS Sample Synthesized", "SUM", "quantity"), - ("tts.report_requested", "TTS Report Requested", "COUNT", None), - ("tts.report_completed", "TTS Report Completed", "COUNT", None), - # Call imports - ("call_import.batch_created", "Call Import Batch Created", "SUM", "quantity"), - ("call_import.evaluation_completed", "Call Import Evaluation Completed", "SUM", "quantity"), - # Agent playground - ("playground.web_call_started", "Playground Web Call Started", "COUNT", None), - ("playground.websocket_session_started", "Playground Websocket Session Started", "COUNT", None), - ("playground.call_evaluated", "Playground Call Evaluated", "COUNT", None), - ("playground.evaluation_completed", "Playground Evaluation Completed", "COUNT", None), - # Evaluators - ("evaluator.run_requested", "Evaluator Run Requested", "SUM", "quantity"), - ("evaluator.run_completed", "Evaluator Run Completed", "COUNT", None), + # Call imports — paid usage (batch_created meter comes from call_imports feature) + ("call_import.evaluation_completed", "Call Import Evaluations", "SUM", "quantity"), + ( + "call_import.recording_minutes_billed", + "Call Import Recording Minutes", + "SUM", + "billable_minutes", + ), + ("call_import.pdf_report_generated", "Call Import PDF Reports", "SUM", "quantity"), + (CALL_IMPORT_USER_INSIGHTS_EVENT, "Call Import User Insights", "COUNT", None), + ( + CALL_IMPORT_PROMPT_IMPROVEMENTS_EVENT, + "Call Import Prompt Improvements", + "COUNT", + None, + ), + # Evaluators — run + optional audio (run_completed meter from evaluators feature) + ( + "evaluator.recording_minutes_billed", + "Evaluator Recording Minutes", + "SUM", + "billable_minutes", + ), + # Agent playground — test-agent API path (playground eval meter from feature) + ( + "test_agent.conversation_ended", + "Test Agent Conversation Minutes", + "SUM", + "billable_minutes", + ), + # Voice playground — billable completions (feature primary = blind_test responses) + ("tts.sample_synthesized", "TTS Samples Synthesized", "SUM", "quantity"), + ("tts.report_completed", "TTS Reports Completed", "SUM", "quantity"), # Legacy evaluations - ("evaluation.created", "Evaluation Created", "COUNT", None), - ("evaluation.completed", "Evaluation Completed", "COUNT", None), - # Prompt optimization - ("prompt_optimization.run_started", "Prompt Optimization Run Started", "COUNT", None), - ("prompt_optimization.run_completed", "Prompt Optimization Run Completed", "COUNT", None), - # Judge alignment - ("judge_alignment.run_started", "Judge Alignment Run Started", "COUNT", None), - ("judge_alignment.run_completed", "Judge Alignment Run Completed", "COUNT", None), - # Observability - ("observability.call_ingested", "Observability Call Ingested", "COUNT", None), - ("observability.call_evaluated", "Observability Call Evaluated", "COUNT", None), - # Test agents - ("test_agent.conversation_started", "Test Agent Conversation Started", "COUNT", None), - ("test_agent.conversation_ended", "Test Agent Conversation Ended", "SUM", "quantity"), - # LLM assist - ("metrics.llm_assist", "Metrics LLM Assist", "COUNT", None), - ("chat.completion", "Chat Completion", "SUM", "quantity"), + ("evaluation.completed", "Legacy Evaluation Completed", "SUM", "quantity"), + # Agent playground AI helpers + (PERSONA_PROMPT_GENERATED_EVENT, "Persona Prompt Generation", "COUNT", None), + (AGENT_TEST_SETUP_GENERATED_EVENT, "Agent Test Setup Generation", "COUNT", None), +] + +# Ops guide: meters to attach as plan USAGE charges (display order for SaaS plans). +PLAN_BILLABLE_METERS: list[dict[str, Any]] = [ + { + "product": "Call Imports", + "plan_line": "Call import evaluations", + "event_name": "call_import.evaluation_completed", + "meter_name": "Call Import Evaluations", + "aggregation": "SUM quantity", + "charge": True, + "notes": "Main compute line — one unit per newly completed row per eval pass.", + }, + { + "product": "Call Imports", + "plan_line": "Call import audio minutes", + "event_name": "call_import.recording_minutes_billed", + "meter_name": "Call Import Recording Minutes", + "aggregation": "SUM billable_minutes", + "charge": True, + "notes": "Only rows with a recording; ceil(seconds/60), min 1.", + }, + { + "product": "Call Imports", + "plan_line": "Call import PDF reports", + "event_name": "call_import.pdf_report_generated", + "meter_name": "Call Import PDF Reports", + "aggregation": "SUM quantity", + "charge": True, + "notes": "Optional export add-on.", + }, + { + "product": "Call Imports", + "plan_line": "Call import user insights", + "event_name": CALL_IMPORT_USER_INSIGHTS_EVENT, + "meter_name": "Call Import User Insights", + "aggregation": "COUNT", + "charge": True, + "notes": "On-demand map-reduce insights generation per evaluation.", + }, + { + "product": "Call Imports", + "plan_line": "Call import prompt improvements", + "event_name": CALL_IMPORT_PROMPT_IMPROVEMENTS_EVENT, + "meter_name": "Call Import Prompt Improvements", + "aggregation": "COUNT", + "charge": True, + "notes": "Prompt improvement suggestions from evaluation clusters.", + }, + { + "product": "Call Imports", + "plan_line": "Imported rows (entitlement cap)", + "event_name": CALL_IMPORT_BATCH_EVENT, + "meter_name": CALL_IMPORT_BATCH_METER_NAME, + "aggregation": "SUM quantity", + "charge": False, + "notes": "Feature call_imports primary meter — tier included rows; $0 usage charge typical.", + }, + { + "product": "Agent Playground", + "plan_line": "Playground evaluated calls", + "event_name": AGENT_PLAYGROUND_PRIMARY_EVENT, + "meter_name": "Agent Playground", + "aggregation": "SUM billable_minutes", + "charge": True, + "notes": "Billed when playground evaluation completes.", + }, + { + "product": "Agent Playground", + "plan_line": "Test agent conversations", + "event_name": "test_agent.conversation_ended", + "meter_name": "Test Agent Conversation Minutes", + "aggregation": "SUM billable_minutes", + "charge": True, + "notes": "Standalone test-agent API only; not double-billed with playground eval.", + }, + { + "product": "Agent Playground", + "plan_line": "Persona prompt generation", + "event_name": PERSONA_PROMPT_GENERATED_EVENT, + "meter_name": "Persona Prompt Generation", + "aggregation": "COUNT", + "charge": True, + "notes": "AI-generated caller persona prompts from agent prompts.", + }, + { + "product": "Agent Playground", + "plan_line": "Agent test setup generation", + "event_name": AGENT_TEST_SETUP_GENERATED_EVENT, + "meter_name": "Agent Test Setup Generation", + "aggregation": "COUNT", + "charge": True, + "notes": "Test prompt and scenario draft generation for agents.", + }, + { + "product": "Voice Playground", + "plan_line": "Blind test responses", + "event_name": VOICE_PLAYGROUND_PRIMARY_EVENT, + "meter_name": "Voice Playground", + "aggregation": "SUM quantity", + "charge": True, + }, + { + "product": "Voice Playground", + "plan_line": "TTS samples", + "event_name": "tts.sample_synthesized", + "meter_name": "TTS Samples Synthesized", + "aggregation": "SUM quantity", + "charge": True, + }, + { + "product": "Voice Playground", + "plan_line": "TTS reports", + "event_name": "tts.report_completed", + "meter_name": "TTS Reports Completed", + "aggregation": "SUM quantity", + "charge": True, + }, + { + "product": "Evaluators", + "plan_line": "Evaluator runs", + "event_name": EVALUATOR_RUN_COMPLETED_EVENT, + "meter_name": "Evaluators", + "aggregation": "COUNT", + "charge": True, + }, + { + "product": "Evaluators", + "plan_line": "Evaluator audio minutes", + "event_name": "evaluator.recording_minutes_billed", + "meter_name": "Evaluator Recording Minutes", + "aggregation": "SUM billable_minutes", + "charge": True, + "notes": "When the run includes a recording; ceil(seconds/60), min 1.", + }, + { + "product": "GEPA Optimization", + "plan_line": "Prompt optimization candidates", + "event_name": GEPA_PRIMARY_EVENT, + "meter_name": "GEPA Optimization", + "aggregation": "SUM quantity", + "charge": True, + "notes": "Quantity = candidates produced in the completed run.", + }, + { + "product": "Judge Alignment", + "plan_line": "Judge alignment samples", + "event_name": JUDGE_ALIGNMENT_PRIMARY_EVENT, + "meter_name": "Judge Alignment", + "aggregation": "SUM quantity", + "charge": True, + "notes": "Quantity = samples scored in the completed run.", + }, + { + "product": "Metrics AI Assist", + "plan_line": "Metrics AI assist", + "event_name": METRICS_AI_ASSIST_EVENT, + "meter_name": "Metrics AI Assist", + "aggregation": "COUNT", + "charge": True, + }, + { + "product": "Metric Studio", + "plan_line": "Metric studio items", + "event_name": METRIC_STUDIO_PRIMARY_EVENT, + "meter_name": "Metric Studio", + "aggregation": "SUM quantity", + "charge": True, + "notes": "Quantity = completed items in the run.", + }, + { + "product": "Scenario AI", + "plan_line": "Scenario AI generations", + "event_name": SCENARIO_AI_TEXT_EVENT, + "meter_name": "Scenario AI Text", + "aggregation": "COUNT", + "charge": True, + }, + { + "product": "Prompt Partials", + "plan_line": "Prompt partial AI assist", + "event_name": PROMPT_PARTIAL_AI_ASSISTED_EVENT, + "meter_name": "Prompt Partial AI Assist", + "aggregation": "COUNT", + "charge": True, + "notes": "Generate, improve, flowchart, and flowchart prompt mapping.", + }, ] LICENSE_FEATURES: list[dict[str, Any]] = [ { "name": "Call Imports", "lookup_key": "call_imports", - "description": "CSV/audio call import batches, row processing, and evaluations", - "unit_singular": "batch", - "unit_plural": "batches", - "event_name": "call_import.batch_created", - "aggregation": {"type": "COUNT"}, + "description": "CSV/audio call import batches — row cap and import volume", + "unit_singular": "row", + "unit_plural": "rows", + "event_name": CALL_IMPORT_BATCH_EVENT, + "aggregation": {"type": CALL_IMPORT_BATCH_AGG_TYPE, "field": CALL_IMPORT_BATCH_AGG_FIELD}, + }, + { + "name": "Agent Playground", + "lookup_key": "agent_playground", + "description": "Agent playground calls scored after evaluation completes", + "unit_singular": "minute", + "unit_plural": "minutes", + "event_name": AGENT_PLAYGROUND_PRIMARY_EVENT, + "aggregation": {"type": "SUM", "field": "billable_minutes"}, }, { "name": "Voice Playground", "lookup_key": "voice_playground", - "description": "TTS comparisons, blind tests, and voice quality reports", - "unit_singular": "share", - "unit_plural": "shares", - "event_name": "blind_test.share_created", - "aggregation": {"type": "COUNT"}, + "description": "Blind test responses and voice quality workflows", + "unit_singular": "response", + "unit_plural": "responses", + "event_name": VOICE_PLAYGROUND_PRIMARY_EVENT, + "aggregation": {"type": "SUM", "field": "quantity"}, }, { "name": "GEPA Optimization", "lookup_key": "gepa_optimization", "description": "Prompt optimization (GEPA) runs", + "unit_singular": "candidate", + "unit_plural": "candidates", + "event_name": GEPA_PRIMARY_EVENT, + "aggregation": {"type": "SUM", "field": "quantity"}, + }, + { + "name": "Evaluators", + "lookup_key": "evaluators", + "description": "Evaluator simulation runs that complete scoring", "unit_singular": "run", "unit_plural": "runs", - "event_name": "prompt_optimization.run_started", + "event_name": EVALUATOR_RUN_COMPLETED_EVENT, + "aggregation": {"type": "COUNT"}, + }, + { + "name": "Judge Alignment", + "lookup_key": "judge_alignment", + "description": "Judge calibration runs on labeled datasets", + "unit_singular": "sample", + "unit_plural": "samples", + "event_name": JUDGE_ALIGNMENT_PRIMARY_EVENT, + "aggregation": {"type": "SUM", "field": "quantity"}, + }, + { + "name": "Metrics AI Assist", + "lookup_key": "metrics_ai_assist", + "description": "AI-assisted metric creation in Metrics Management", + "unit_singular": "request", + "unit_plural": "requests", + "event_name": METRICS_AI_ASSIST_EVENT, + "aggregation": {"type": "COUNT"}, + }, + { + "name": "Metric Studio", + "lookup_key": "metric_studio", + "description": "Batch metric scoring runs in Metrics Studio", + "unit_singular": "item", + "unit_plural": "items", + "event_name": METRIC_STUDIO_PRIMARY_EVENT, + "aggregation": {"type": "SUM", "field": "quantity"}, + }, + { + "name": "Scenario AI Text", + "lookup_key": "scenario_ai", + "description": "AI-generated text for scenarios and assistant flows", + "unit_singular": "generation", + "unit_plural": "generations", + "event_name": SCENARIO_AI_TEXT_EVENT, + "aggregation": {"type": "COUNT"}, + }, + { + "name": "Prompt Partials", + "lookup_key": "prompt_partials", + "description": "AI-assisted prompt partial generate, improve, and flowchart workflows", + "unit_singular": "request", + "unit_plural": "requests", + "event_name": PROMPT_PARTIAL_AI_ASSISTED_EVENT, "aggregation": {"type": "COUNT"}, }, ] +FEATURE_OWNED_EVENT_NAMES = frozenset(spec["event_name"] for spec in LICENSE_FEATURES) + +# Backward-compatible alias for tests importing the old name. +EVALUATOR_RUN_REQUESTED_EVENT = EVALUATOR_RUN_COMPLETED_EVENT + def _headers() -> dict[str, str]: return {"x-api-key": settings.FLEXPRICE_API_KEY or "", "Content-Type": "application/json"} @@ -106,8 +396,26 @@ def _meter_payload(name: str, event_name: str, agg_type: str, field: str | None) } -def _list_meters(client: httpx.Client) -> dict[str, dict]: - existing: dict[str, dict] = {} +def meter_aggregation_matches( + meter: dict[str, Any], + *, + agg_type: str, + agg_field: str | None = None, +) -> bool: + aggregation = meter.get("aggregation") or {} + if (aggregation.get("type") or "").upper() != agg_type.upper(): + return False + if agg_field is None: + return not aggregation.get("field") + return (aggregation.get("field") or "") == agg_field + + +def _is_active_meter(meter: dict[str, Any]) -> bool: + return (meter.get("status") or "published").lower() == "published" + + +def _list_all_meters(client: httpx.Client, *, active_only: bool = False) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] offset = 0 while True: resp = client.get( @@ -117,17 +425,26 @@ def _list_meters(client: httpx.Client) -> dict[str, dict]: ) resp.raise_for_status() data = resp.json() - items = data.get("items") or [] - for item in items: - event_name = item.get("event_name") - if event_name: - existing[event_name] = item + batch = data.get("items") or [] + if active_only: + batch = [meter for meter in batch if _is_active_meter(meter)] + items.extend(batch) pagination = data.get("pagination") or {} total = pagination.get("total") - offset += len(items) - if not items or (total is not None and offset >= total): + offset += len(data.get("items") or []) + if not data.get("items") or (total is not None and offset >= total): break - return existing + return items + + +def _list_meters_by_event(client: httpx.Client, *, active_only: bool = False) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for meter in _list_all_meters(client, active_only=active_only): + event_name = meter.get("event_name") + if not event_name: + continue + grouped.setdefault(event_name, []).append(meter) + return grouped def _create_meter(client: httpx.Client, event_name: str, name: str, agg_type: str, field: str | None) -> dict: @@ -141,6 +458,623 @@ def _create_meter(client: httpx.Client, event_name: str, name: str, agg_type: st return resp.json() +def _delete_meter(client: httpx.Client, meter_id: str) -> None: + resp = client.delete(f"{_base_url()}/meters/{meter_id}", headers=_headers()) + resp.raise_for_status() + + +def _archive_meter(client: httpx.Client, meter_id: str) -> dict[str, Any]: + """Remove or archive a meter; archive is the fallback when delete is blocked.""" + for method, suffix in (("DELETE", ""), ("POST", "/archive")): + url = f"{_base_url()}/meters/{meter_id}{suffix}" + if method == "DELETE": + resp = client.delete(url, headers=_headers()) + else: + resp = client.post(url, headers=_headers()) + if resp.status_code in (200, 204, 404): + return {"meter_id": meter_id, "method": method, "status": resp.status_code} + resp.raise_for_status() + return {"meter_id": meter_id, "status": resp.status_code} + + +PLAYGROUND_DEPRECATED_EVENTS = frozenset( + { + "playground.call_evaluated", + "playground.web_call_started", + "playground.websocket_session_started", + } +) + + +def _list_prices(client: httpx.Client) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + while True: + resp = client.get( + f"{_base_url()}/prices", + headers=_headers(), + params={"limit": 200, "offset": offset}, + ) + resp.raise_for_status() + data = resp.json() + batch = data.get("items") or [] + items.extend(batch) + pagination = data.get("pagination") or {} + total = pagination.get("total") + offset += len(batch) + if not batch or (total is not None and offset >= total): + break + return items + + +def _active_prices_for_meter(prices: list[dict[str, Any]], meter_id: str) -> list[dict[str, Any]]: + active: list[dict[str, Any]] = [] + for price in prices: + if price.get("meter_id") != meter_id: + continue + if price.get("end_date"): + continue + if (price.get("status") or "").lower() in {"deleted", "archived", "disabled"}: + continue + active.append(price) + return active + + +def _create_usage_price_from_template( + client: httpx.Client, + *, + template: dict[str, Any], + meter_id: str, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "entity_type": template.get("entity_type"), + "entity_id": template.get("entity_id"), + "type": template.get("type", "USAGE"), + "billing_model": template.get("billing_model", "FLAT_FEE"), + "billing_period": template.get("billing_period", "MONTHLY"), + "billing_period_count": template.get("billing_period_count", 1), + "billing_cadence": template.get("billing_cadence", "RECURRING"), + "currency": template.get("currency", "usd"), + "invoice_cadence": template.get("invoice_cadence", "ARREAR"), + "price_unit_type": template.get("price_unit_type", "FIAT"), + "meter_id": meter_id, + "amount": template.get("amount", "1"), + "display_name": template.get("display_name") or CALL_IMPORT_BATCH_METER_NAME, + } + if template.get("description"): + payload["description"] = template["description"] + if template.get("lookup_key"): + payload["lookup_key"] = template["lookup_key"] + if template.get("transform_quantity"): + payload["transform_quantity"] = template["transform_quantity"] + resp = client.post(f"{_base_url()}/prices", headers=_headers(), json=payload) + resp.raise_for_status() + return resp.json() + + +def _terminate_price(client: httpx.Client, price_id: str) -> None: + resp = client.request("DELETE", f"{_base_url()}/prices/{price_id}", headers=_headers(), json={}) + if resp.status_code == 404: + return + resp.raise_for_status() + + +def _dedupe_canonical_prices( + client: httpx.Client, + *, + canonical_meter_id: str, + prices: list[dict[str, Any]], + result: dict[str, Any], +) -> None: + active = _active_prices_for_meter(prices, canonical_meter_id) + if len(active) <= 1: + return + active.sort(key=lambda price: price.get("created_at") or "", reverse=True) + for duplicate in active[1:]: + _terminate_price(client, duplicate["id"]) + result["terminated_prices"].append(duplicate["id"]) + result["subscription_resync_required"] = True + + +def _terminate_orphaned_batch_prices( + client: httpx.Client, + *, + prices: list[dict[str, Any]], + active_batch_meter_ids: set[str], + result: dict[str, Any], +) -> None: + batch_meter_ids = { + meter["id"] + for meter in _list_all_meters(client) + if meter.get("event_name") == CALL_IMPORT_BATCH_EVENT + } + for price in prices: + meter_id = price.get("meter_id") + if not meter_id or meter_id not in batch_meter_ids: + continue + if meter_id in active_batch_meter_ids: + continue + if price.get("end_date"): + continue + if (price.get("status") or "").lower() in {"deleted", "archived", "disabled"}: + continue + _terminate_price(client, price["id"]) + result["terminated_prices"].append(price["id"]) + result["subscription_resync_required"] = True + + +PLAN_USAGE_RESTORE_NAMES = ("Voice Playground", "Blind Test") +PLAN_USAGE_ARCHIVE_TEMPLATE_IDS: dict[str, str] = { + "Voice Playground": "price_01KVT8P9T8E52KHB1CA6HCVPV0", + "Blind Test": "price_01KW9E4W02NND35R6W87Y7863K", +} + + +def _fetch_price(client: httpx.Client, price_id: str) -> dict[str, Any] | None: + resp = client.get(f"{_base_url()}/prices/{price_id}", headers=_headers()) + if resp.status_code == 404: + return None + resp.raise_for_status() + return resp.json() + + +def restore_missing_plan_usage_prices(client: httpx.Client) -> dict[str, Any]: + """Recreate plan usage prices that were terminated but still have archived templates.""" + result: dict[str, Any] = {"restored": [], "skipped": []} + all_prices = _list_prices(client) + active_names = { + price.get("display_name") + for price in all_prices + if price.get("entity_type") == "PLAN" + and price.get("type") == "USAGE" + and not price.get("end_date") + and (price.get("status") or "published").lower() == "published" + } + + for display_name in PLAN_USAGE_RESTORE_NAMES: + if display_name in active_names: + result["skipped"].append(display_name) + continue + templates = [ + price + for price in all_prices + if price.get("display_name") == display_name + and price.get("entity_type") == "PLAN" + and price.get("type") == "USAGE" + and price.get("end_date") + ] + template: dict[str, Any] | None = None + if templates: + templates.sort(key=lambda price: price.get("end_date") or "", reverse=True) + template = templates[0] + else: + archive_id = PLAN_USAGE_ARCHIVE_TEMPLATE_IDS.get(display_name) + if archive_id: + template = _fetch_price(client, archive_id) + if not template: + result["skipped"].append(display_name) + continue + restored = _create_usage_price_from_template( + client, + template=template, + meter_id=template["meter_id"], + ) + result["restored"].append({"display_name": display_name, "price_id": restored.get("id")}) + return result + + +def repair_playground_meters( + client: httpx.Client, + sdk: Flexprice, + *, + dry_run: bool = False, +) -> dict[str, Any]: + """Archive legacy playground meters; keep SUM(billable_minutes) evaluation_completed only.""" + result: dict[str, Any] = { + "kept_meter_id": None, + "archived_meters": [], + "removed_features": [], + "skipped": [], + "errors": [], + "dry_run": dry_run, + } + + meters_by_event = _list_meters_by_event(client, active_only=True) + all_prices = _list_prices(client) + canonical = _pick_canonical_meter( + meters_by_event.get(AGENT_PLAYGROUND_PRIMARY_EVENT, []), + all_prices, + agg_type="SUM", + agg_field="billable_minutes", + ) + if canonical is None: + result["errors"].append( + "No published playground.evaluation_completed meter with SUM(billable_minutes)" + ) + return result + + result["kept_meter_id"] = canonical["id"] + to_archive: list[dict[str, Any]] = [] + for event_name in PLAYGROUND_DEPRECATED_EVENTS: + to_archive.extend(meters_by_event.get(event_name, [])) + for meter in meters_by_event.get(AGENT_PLAYGROUND_PRIMARY_EVENT, []): + if meter["id"] != canonical["id"]: + to_archive.append(meter) + + seen_meter_ids: set[str] = set() + for meter in to_archive: + meter_id = meter["id"] + if meter_id in seen_meter_ids: + continue + seen_meter_ids.add(meter_id) + entry = { + "id": meter_id, + "event_name": meter.get("event_name"), + "name": meter.get("name"), + } + if dry_run: + result["archived_meters"].append({**entry, "dry_run": True}) + continue + try: + archived = _archive_meter(client, meter_id) + result["archived_meters"].append({**entry, **archived}) + except Exception as exc: + result["errors"].append(f"meter {meter_id}: {exc}") + + agent_playground_features = [ + feature + for feature in _list_all_features(client) + if feature.get("lookup_key") == "agent_playground" + ] + for feature in agent_playground_features: + if feature.get("meter_id") == canonical["id"]: + result["skipped"].append(feature["id"]) + continue + entry = { + "id": feature["id"], + "meter_id": feature.get("meter_id"), + "name": feature.get("name"), + } + if dry_run: + result["removed_features"].append({**entry, "dry_run": True}) + continue + try: + sdk.features.delete_feature(id=feature["id"]) + result["removed_features"].append(entry) + except Exception as exc: + result["errors"].append(f"feature {feature['id']}: {exc}") + + if not dry_run and len(agent_playground_features) == len(result["removed_features"]): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "agent_playground") + try: + created = sdk.features.create_feature( + name=spec["name"], + type_="metered", + lookup_key=spec["lookup_key"], + description=spec["description"], + unit_singular=spec.get("unit_singular"), + unit_plural=spec.get("unit_plural"), + meter_id=canonical["id"], + ) + result["recreated_feature_id"] = created.id + except Exception as exc: + message = str(exc).lower() + if "already exist" not in message and "duplicate" not in message: + result["errors"].append(f"recreate agent_playground feature: {exc}") + + return result + + +def repair_call_import_batch_meter(client: httpx.Client) -> dict[str, Any]: + """Ensure one SUM(quantity) meter for call_import.batch_created and remove duplicates.""" + result: dict[str, Any] = { + "deleted_duplicate_meters": [], + "deleted_legacy_meters": [], + "terminated_prices": [], + "created_meter_id": None, + "created_price_ids": [], + "kept_meter_id": None, + "subscription_resync_required": False, + } + + all_prices = _list_prices(client) + batch_meters = _list_meters_by_event(client, active_only=True).get(CALL_IMPORT_BATCH_EVENT, []) + + correct_meters = [ + meter + for meter in batch_meters + if meter_aggregation_matches( + meter, + agg_type=CALL_IMPORT_BATCH_AGG_TYPE, + agg_field=CALL_IMPORT_BATCH_AGG_FIELD, + ) + ] + incorrect_meters = [meter for meter in batch_meters if meter not in correct_meters] + + canonical_meter: dict[str, Any] | None = None + if correct_meters: + priced_correct = [ + meter + for meter in correct_meters + if _active_prices_for_meter(all_prices, meter["id"]) + ] + canonical_meter = priced_correct[0] if priced_correct else correct_meters[0] + result["kept_meter_id"] = canonical_meter["id"] + + prices_to_recreate: list[dict[str, Any]] = [] + for meter in incorrect_meters: + meter_id = meter["id"] + attached_prices = _active_prices_for_meter(all_prices, meter_id) + if attached_prices: + prices_to_recreate.extend(attached_prices) + else: + _delete_meter(client, meter_id) + result["deleted_duplicate_meters"].append( + {"id": meter_id, "name": meter.get("name")} + ) + + if canonical_meter is None or prices_to_recreate: + if canonical_meter is None: + canonical_meter = _create_meter( + client, + CALL_IMPORT_BATCH_EVENT, + CALL_IMPORT_BATCH_METER_NAME, + CALL_IMPORT_BATCH_AGG_TYPE, + CALL_IMPORT_BATCH_AGG_FIELD, + ) + result["created_meter_id"] = canonical_meter.get("id") + result["kept_meter_id"] = canonical_meter["id"] + + existing_canonical_prices = _active_prices_for_meter(all_prices, canonical_meter["id"]) + for old_price in prices_to_recreate: + if existing_canonical_prices: + _terminate_price(client, old_price["id"]) + result["terminated_prices"].append(old_price["id"]) + result["subscription_resync_required"] = True + continue + new_price = _create_usage_price_from_template( + client, + template=old_price, + meter_id=canonical_meter["id"], + ) + result["created_price_ids"].append(new_price.get("id")) + existing_canonical_prices = [new_price] + _terminate_price(client, old_price["id"]) + result["terminated_prices"].append(old_price["id"]) + result["subscription_resync_required"] = True + + for meter in incorrect_meters: + meter_id = meter["id"] + if meter_id == canonical_meter["id"]: + continue + _delete_meter(client, meter_id) + result["deleted_legacy_meters"].append( + {"id": meter_id, "name": meter.get("name")} + ) + + extra_correct = [ + meter + for meter in correct_meters + if canonical_meter and meter["id"] != canonical_meter["id"] + ] + for meter in extra_correct: + meter_id = meter["id"] + attached_prices = _active_prices_for_meter(all_prices, meter_id) + if attached_prices: + continue + _delete_meter(client, meter_id) + result["deleted_duplicate_meters"].append( + {"id": meter_id, "name": meter.get("name")} + ) + + if canonical_meter: + refreshed_prices = _list_prices(client) + active_batch_meter_ids = { + meter["id"] + for meter in _list_meters_by_event(client, active_only=True).get(CALL_IMPORT_BATCH_EVENT, []) + if meter_aggregation_matches( + meter, + agg_type=CALL_IMPORT_BATCH_AGG_TYPE, + agg_field=CALL_IMPORT_BATCH_AGG_FIELD, + ) + } + _terminate_orphaned_batch_prices( + client, + prices=refreshed_prices, + active_batch_meter_ids=active_batch_meter_ids, + result=result, + ) + refreshed_prices = _list_prices(client) + _dedupe_canonical_prices( + client, + canonical_meter_id=canonical_meter["id"], + prices=refreshed_prices, + result=result, + ) + + return result + + +def _aggregation_from_spec(spec: dict[str, Any]) -> tuple[str, str | None]: + aggregation = spec["aggregation"] + return aggregation["type"], aggregation.get("field") + + +def _pick_canonical_meter( + meters: list[dict[str, Any]], + prices: list[dict[str, Any]], + *, + agg_type: str, + agg_field: str | None, +) -> dict[str, Any] | None: + """Prefer published meters with correct aggregation; favor meters that already have plan prices.""" + matching = [ + meter + for meter in meters + if meter_aggregation_matches(meter, agg_type=agg_type, agg_field=agg_field) + ] + if not matching: + return None + priced = [meter for meter in matching if _active_prices_for_meter(prices, meter["id"])] + if priced: + return priced[0] + active = [meter for meter in matching if _is_active_meter(meter)] + pool = active or matching + return sorted(pool, key=lambda meter: meter.get("updated_at") or "", reverse=True)[0] + + +def _list_all_features(client: httpx.Client) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + while True: + resp = client.get( + f"{_base_url()}/features", + headers=_headers(), + params={"limit": 200, "offset": offset}, + ) + resp.raise_for_status() + data = resp.json() + batch = data.get("items") or [] + items.extend(batch) + pagination = data.get("pagination") or {} + total = pagination.get("total") + offset += len(batch) + if not batch or (total is not None and offset >= total): + break + return items + + +def _feature_meter_is_canonical( + feature: dict[str, Any], + meter: dict[str, Any] | None, + *, + canonical_meter_id: str, + agg_type: str, + agg_field: str | None, +) -> bool: + if feature.get("meter_id") != canonical_meter_id: + return False + if meter is None: + return False + if not _is_active_meter(meter): + return False + return meter_aggregation_matches(meter, agg_type=agg_type, agg_field=agg_field) + + +def repair_license_features( + client: httpx.Client, + sdk: Flexprice, + *, + dry_run: bool = False, +) -> dict[str, Any]: + """Repoint LICENSE_FEATURES to canonical published meters (delete + recreate via API). + + Flexprice feature PUT does not change meter_id; recreate is required. + Plan usage charges stay on meters and are unaffected. + """ + result: dict[str, Any] = { + "repaired": [], + "skipped": [], + "failed": [], + "missing_feature": [], + "missing_meter": [], + "dry_run": dry_run, + } + + meters_by_event = _list_meters_by_event(client, active_only=False) + all_prices = _list_prices(client) + features_by_key = { + feature["lookup_key"]: feature + for feature in _list_all_features(client) + if feature.get("lookup_key") + } + + for spec in LICENSE_FEATURES: + lookup_key = spec["lookup_key"] + feature = features_by_key.get(lookup_key) + if feature is None: + result["missing_feature"].append(lookup_key) + continue + + agg_type, agg_field = _aggregation_from_spec(spec) + event_name = spec["event_name"] + event_meters = meters_by_event.get(event_name, []) + active_event_meters = [meter for meter in event_meters if _is_active_meter(meter)] + canonical = _pick_canonical_meter( + active_event_meters or event_meters, + all_prices, + agg_type=agg_type, + agg_field=agg_field, + ) + + if canonical is None: + if dry_run: + result["missing_meter"].append( + {"lookup_key": lookup_key, "event_name": event_name, "would_create": True} + ) + continue + try: + canonical = _create_meter( + client, + event_name, + spec["name"], + agg_type, + agg_field, + ) + except Exception as exc: + result["failed"].append(f"{lookup_key}: create meter: {exc}") + continue + if canonical.get("skipped"): + result["failed"].append(f"{lookup_key}: could not create meter for {event_name}") + continue + + canonical_id = canonical["id"] + current_meter_id = feature.get("meter_id") + current_meter = next( + (meter for meter in event_meters if meter.get("id") == current_meter_id), + None, + ) + if _feature_meter_is_canonical( + feature, + current_meter, + canonical_meter_id=canonical_id, + agg_type=agg_type, + agg_field=agg_field, + ): + result["skipped"].append(lookup_key) + continue + + entry: dict[str, Any] = { + "lookup_key": lookup_key, + "feature_id": feature["id"], + "from_meter_id": current_meter_id, + "to_meter_id": canonical_id, + "event_name": event_name, + } + if dry_run: + result["repaired"].append({**entry, "dry_run": True}) + continue + + try: + sdk.features.delete_feature(id=feature["id"]) + recreated = sdk.features.create_feature( + name=spec["name"], + type_="metered", + lookup_key=lookup_key, + description=spec["description"], + unit_singular=spec.get("unit_singular"), + unit_plural=spec.get("unit_plural"), + meter_id=canonical_id, + ) + result["repaired"].append( + {**entry, "new_feature_id": recreated.id, "dry_run": False} + ) + except Exception as exc: + result["failed"].append(f"{lookup_key}: {exc}") + + return result + + def _list_feature_lookup_keys(sdk: Flexprice) -> set[str]: keys: set[str] = set() offset = 0 @@ -181,14 +1115,7 @@ def _create_license_feature(sdk: Flexprice, spec: dict[str, Any]) -> dict: raise -def main() -> int: - if CONFIG_PATH.exists(): - load_config_from_file(str(CONFIG_PATH)) - - if not settings.FLEXPRICE_ENABLED or not settings.FLEXPRICE_API_KEY: - print("Flexprice is not enabled or FLEXPRICE_API_KEY is missing.", file=sys.stderr) - return 1 - +def bootstrap_catalog() -> dict[str, Any]: created_meters: list[str] = [] skipped_meters: list[str] = [] failed_meters: list[str] = [] @@ -197,9 +1124,12 @@ def main() -> int: failed_features: list[str] = [] with httpx.Client(timeout=60.0) as http_client: - existing_meters = _list_meters(http_client) + existing_by_event = _list_meters_by_event(http_client) for event_name, name, agg_type, field in METERS: - if event_name in existing_meters: + if event_name in FEATURE_OWNED_EVENT_NAMES: + skipped_meters.append(event_name) + continue + if event_name in existing_by_event: skipped_meters.append(event_name) continue try: @@ -230,12 +1160,130 @@ def main() -> int: except Exception as exc: failed_features.append(f"{key}: {exc}") - summary = { + return { "meters": {"created": created_meters, "skipped": skipped_meters, "failed": failed_meters}, "features": {"created": created_features, "skipped": skipped_features, "failed": failed_features}, } + + +def print_plan_guide() -> None: + """Print SaaS plan usage charge checklist (stdout JSON).""" + paid = [row for row in PLAN_BILLABLE_METERS if row.get("charge")] + entitlements = [row for row in PLAN_BILLABLE_METERS if not row.get("charge")] + print( + json.dumps( + { + "paid_usage_charges": paid, + "entitlement_only": entitlements, + "docs": "docs/billing/flexprice-saas-setup.md", + }, + indent=2, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Bootstrap or repair Flexprice metering catalog.") + parser.add_argument( + "--repair-call-imports", + action="store_true", + help="Remove duplicate batch_created meters and migrate pricing to SUM(quantity).", + ) + parser.add_argument( + "--repair-playground", + action="store_true", + help="Archive legacy playground meters (call_evaluated, duplicates) and fix agent_playground feature.", + ) + parser.add_argument( + "--restore-plan-usage-prices", + action="store_true", + help="Recreate missing Voice Playground and Blind Test plan usage prices from archived templates.", + ) + parser.add_argument( + "--plan-guide", + action="store_true", + help="Print JSON checklist of plan usage charges (no API calls).", + ) + parser.add_argument( + "--repair-features", + action="store_true", + help="Repoint license features to canonical completion-only meters (delete + recreate).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="With --repair-features, report changes without calling Flexprice write APIs.", + ) + args = parser.parse_args() + + if args.plan_guide: + print_plan_guide() + return 0 + + if CONFIG_PATH.exists(): + load_config_from_file(str(CONFIG_PATH)) + + if not settings.FLEXPRICE_ENABLED or not settings.FLEXPRICE_API_KEY: + print("Flexprice is not enabled or FLEXPRICE_API_KEY is missing.", file=sys.stderr) + return 1 + + summary: dict[str, Any] = {} + with httpx.Client(timeout=60.0) as http_client: + if args.repair_playground: + with Flexprice( + server_url=settings.FLEXPRICE_API_HOST, + api_key_auth=settings.FLEXPRICE_API_KEY, + ) as sdk: + summary["playground_repair"] = repair_playground_meters( + http_client, + sdk, + dry_run=args.dry_run, + ) + if args.dry_run: + print(json.dumps(summary, indent=2)) + failed = summary.get("playground_repair", {}).get("errors") or [] + return 1 if failed else 0 + if args.repair_call_imports: + summary["call_import_batch_repair"] = repair_call_import_batch_meter(http_client) + if args.restore_plan_usage_prices: + summary["plan_usage_restore"] = restore_missing_plan_usage_prices(http_client) + if args.repair_features: + with Flexprice( + server_url=settings.FLEXPRICE_API_HOST, + api_key_auth=settings.FLEXPRICE_API_KEY, + ) as sdk: + summary["feature_repair"] = repair_license_features( + http_client, + sdk, + dry_run=args.dry_run, + ) + + if args.repair_features and args.dry_run: + print(json.dumps(summary, indent=2)) + failed = summary.get("feature_repair", {}).get("failed") or [] + return 1 if failed else 0 + + bootstrap_summary = bootstrap_catalog() + summary.update(bootstrap_summary) + print(json.dumps(summary, indent=2)) - return 1 if (failed_meters or failed_features) else 0 + + repair = summary.get("call_import_batch_repair") or {} + if repair.get("subscription_resync_required"): + print( + "\nNOTE: Plan prices were recreated. In Flexprice, open the plan and click " + "'Sync Usage Charges', then refresh the customer subscription so line items " + "reference the new price IDs.", + file=sys.stderr, + ) + + failed_meters = bootstrap_summary["meters"]["failed"] + failed_features = bootstrap_summary["features"]["failed"] + repair_failed = (summary.get("feature_repair") or {}).get("failed") or [] + playground_failed = (summary.get("playground_repair") or {}).get("errors") or [] + if args.repair_playground and not args.repair_features and not args.repair_call_imports: + return 1 if playground_failed else 0 + return 1 if (failed_meters or failed_features or repair_failed or playground_failed) else 0 if __name__ == "__main__": diff --git a/src/efficientai/services/sarvam/stt.py b/src/efficientai/services/sarvam/stt.py index 2b775812..eedb493c 100644 --- a/src/efficientai/services/sarvam/stt.py +++ b/src/efficientai/services/sarvam/stt.py @@ -145,6 +145,7 @@ def __init__( self._websocket_context = None self._socket_client = None self._receive_task = None + self._disconnected_logged = False def language_to_service_language(self, language: Language) -> str: """Convert efficientai Language enum to Sarvam's language code. @@ -244,7 +245,17 @@ async def run_stt(self, audio: bytes): Frame: None (transcription results come via WebSocket callbacks). """ if not self._socket_client: - logger.warning("WebSocket not connected, cannot process audio") + if not self._disconnected_logged: + self._disconnected_logged = True + logger.error( + "Sarvam STT WebSocket not connected; dropping audio until reconnect" + ) + await self.push_error( + ErrorFrame( + "Sarvam speech-to-text is unavailable. Check your Sarvam API key " + "and network connection, then try the call again." + ) + ) yield None return @@ -329,15 +340,18 @@ def _message_handler(message): # Start receive task using EfficientAI's task management self._receive_task = self.create_task(self._receive_task_handler()) + self._disconnected_logged = False logger.info("Connected to Sarvam successfully") except ApiError as e: logger.error(f"Sarvam API error: {e}") + self._disconnected_logged = True await self.push_error(ErrorFrame(f"Sarvam API error: {e}")) except Exception as e: logger.error(f"Failed to connect to Sarvam: {e}") self._socket_client = None self._websocket_context = None + self._disconnected_logged = True await self.push_error(ErrorFrame(f"Failed to connect to Sarvam: {e}")) async def _disconnect(self): @@ -356,6 +370,7 @@ async def _disconnect(self): logger.debug("Disconnected from Sarvam WebSocket") self._socket_client = None self._websocket_context = None + self._disconnected_logged = False async def _receive_task_handler(self): """Handle incoming messages from Sarvam WebSocket. diff --git a/tests/conftest.py b/tests/conftest.py index ed46d3bf..fb680ccb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,10 +11,18 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy import create_engine, event, text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.engine import make_url +from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool + +@compiles(JSONB, "sqlite") +def _compile_jsonb_for_sqlite(_element, _compiler, **_kw): + """Allow in-memory SQLite tests to create tables with JSONB columns.""" + return "JSON" + # Some local environments provide ALLOWED_AUDIO_FORMATS as a non-JSON string, # which breaks pydantic-settings parsing during module import in tests. os.environ["ALLOWED_AUDIO_FORMATS"] = '["wav","mp3","flac","m4a"]' @@ -22,6 +30,10 @@ os.environ["UPLOAD_DIR"] = "/tmp/efficientai-test-uploads" # Local dev often sets SERVICE_MODE=media and config.yml media URLs; keep API tests on full app mode. os.environ["SERVICE_MODE"] = "api" +# Pytest never uses real Flexprice billing (see tests/helpers/flexprice_stubs.py). +os.environ["EFFICIENTAI_PYTEST"] = "1" +os.environ["FLEXPRICE_ENABLED"] = "false" +os.environ.pop("FLEXPRICE_API_KEY", None) _REPO_ROOT = Path(__file__).resolve().parents[1] _SRC_ROOT = _REPO_ROOT / "src" @@ -34,6 +46,27 @@ ) +@pytest.fixture(autouse=True) +def mock_flexprice_locally(monkeypatch, request): + """Stub all Flexprice billing I/O; only test_flexprice_service.py uses a mocked SDK.""" + from tests.helpers.flexprice_stubs import ( + install_flexprice_test_isolation, + is_flexprice_unit_test_path, + ) + + fspath = getattr(request.node, "fspath", None) + if is_flexprice_unit_test_path(fspath): + yield + return + + install_flexprice_test_isolation(monkeypatch) + from app.config import settings + + monkeypatch.setattr(settings, "FLEXPRICE_ENABLED", False, raising=False) + monkeypatch.setattr(settings, "FLEXPRICE_API_KEY", None, raising=False) + yield + + @pytest.fixture(autouse=True) def isolate_service_mode_for_app_factory(monkeypatch): """Prevent local SERVICE_MODE / media URL config from breaking create_app tests.""" diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/flexprice_stubs.py b/tests/helpers/flexprice_stubs.py new file mode 100644 index 00000000..4e754bbd --- /dev/null +++ b/tests/helpers/flexprice_stubs.py @@ -0,0 +1,73 @@ +"""Local Flexprice stubs for pytest — no real API or dashboard I/O.""" + +from __future__ import annotations + +_FLEXPRICE_FORBIDDEN = "Real Flexprice SDK invoked during pytest — use a mock" + + +def _noop_record_event(*_args, **_kwargs) -> bool: + return False + + +def _noop_ensure_customer(*_args, **_kwargs) -> None: + return None + + +def _noop_ensure_subscription(*_args, **_kwargs) -> None: + return None + + +def _noop_provision_billing_customer(*_args, **_kwargs) -> None: + return None + + +class _ForbiddenFlexprice: + def __init__(self, *_args, **_kwargs): + raise RuntimeError(_FLEXPRICE_FORBIDDEN) + + def __enter__(self): + raise RuntimeError(_FLEXPRICE_FORBIDDEN) + + def __exit__(self, *_args): + return False + + +def _is_flexprice_url(url) -> bool: + return "flexprice.io" in str(url).lower() + + +def install_flexprice_test_isolation(monkeypatch) -> None: + """Replace all Flexprice external I/O with local no-ops (suite-wide default).""" + from app.services.billing import flexprice_service as fp + from app.services import organization_provisioning as org_prov + + monkeypatch.setattr(fp, "record_event", _noop_record_event) + monkeypatch.setattr(fp, "ensure_customer", _noop_ensure_customer) + monkeypatch.setattr(fp, "ensure_subscription", _noop_ensure_subscription) + monkeypatch.setattr(org_prov, "provision_billing_customer", _noop_provision_billing_customer) + monkeypatch.setattr("flexprice.Flexprice", _ForbiddenFlexprice) + + try: + import httpx + except ImportError: + return + + original_get = httpx.get + original_request = httpx.request + + def guarded_get(url, *args, **kwargs): + if _is_flexprice_url(url): + raise RuntimeError(f"Blocked Flexprice HTTP during pytest: {url}") + return original_get(url, *args, **kwargs) + + def guarded_request(method, url, *args, **kwargs): + if _is_flexprice_url(url): + raise RuntimeError(f"Blocked Flexprice HTTP during pytest: {url}") + return original_request(method, url, *args, **kwargs) + + monkeypatch.setattr(httpx, "get", guarded_get) + monkeypatch.setattr(httpx, "request", guarded_request) + + +def is_flexprice_unit_test_path(path) -> bool: + return path is not None and "test_flexprice_service.py" in str(path) diff --git a/tests/test_api/test_evaluator_results_routes.py b/tests/test_api/test_evaluator_results_routes.py index 9f84a04d..2e8ebf41 100644 --- a/tests/test_api/test_evaluator_results_routes.py +++ b/tests/test_api/test_evaluator_results_routes.py @@ -1,14 +1,6 @@ """API tests for evaluator results routes.""" -def _blob_storage_service(): - """Resolve the blob storage singleton exposed as s3_service.""" - import importlib - - s3_module = importlib.import_module("app.services.storage.s3_service") - return s3_module.s3_service - - def test_derive_speaker_segments_supports_smallest_payload(): from app.api.v1.routes.evaluator_results import _derive_speaker_segments_from_call_data @@ -244,11 +236,27 @@ def test_evaluator_results_overview_and_aggregate( assert agg["completed_rows"] == 1 +def _patch_blob_storage_download(monkeypatch, *, audio_bytes: bytes = b"fake-audio-bytes"): + """Patch both the blob singleton and the lazy s3_service alias.""" + import importlib + from types import SimpleNamespace + + fake = SimpleNamespace( + is_enabled=lambda: True, + download_file_by_key=lambda _key: audio_bytes, + upload_file_by_key=lambda *_args, **_kwargs: None, + ) + blob_module = importlib.import_module("app.services.storage.blob_storage_service") + s3_module = importlib.import_module("app.services.storage.s3_service") + # Patch the lazy s3_service alias first so undo restores the real singleton. + monkeypatch.setattr(s3_module, "s3_service", fake, raising=False) + monkeypatch.setattr(blob_module, "blob_storage_service", fake) + return fake + + def test_stream_evaluator_result_audio_from_s3( authenticated_client, make_evaluator_result, monkeypatch ): - storage = _blob_storage_service() - make_evaluator_result( result_id="991122", audio_s3_key="audio/organizations/test/evaluations/call-1/recording.mp3", @@ -258,12 +266,7 @@ def test_stream_evaluator_result_audio_from_s3( }, ) - monkeypatch.setattr(storage, "is_enabled", lambda: True) - monkeypatch.setattr( - storage, - "download_file_by_key", - lambda _key: b"fake-audio-bytes", - ) + _patch_blob_storage_download(monkeypatch) response = authenticated_client.get("/api/v1/evaluator-results/991122/audio") @@ -456,12 +459,7 @@ def fake_get(url, headers=None, timeout=120): monkeypatch.setattr("requests.get", fake_get) monkeypatch.setattr("app.core.encryption.decrypt_api_key", lambda _key: "vapi-secret") - storage = _blob_storage_service() - monkeypatch.setattr( - storage, - "upload_file_by_key", - lambda *_args, **_kwargs: None, - ) + _patch_blob_storage_download(monkeypatch) class FakeTask: id = "task-1" @@ -478,3 +476,35 @@ class FakeTask: assert response.status_code == 200 assert captured["headers"]["Authorization"] == "Bearer vapi-secret" + +def test_re_evaluate_playground_result_without_evaluator_id( + authenticated_client, + make_agent, + make_evaluator_result, + monkeypatch, +): + agent = make_agent() + result = make_evaluator_result( + result_id="776655", + evaluator_id=None, + agent_id=agent.id, + transcription="Speaker 1: hello\nSpeaker 2: hi there", + provider_platform="vapi", + status="completed", + ) + + class FakeTask: + id = "task-playground-re-eval" + + monkeypatch.setattr( + "app.workers.celery_app.process_evaluator_result_task.delay", + lambda *_args, **_kwargs: FakeTask(), + ) + + response = authenticated_client.post( + f"/api/v1/evaluator-results/{result.result_id}/re-evaluate" + ) + + assert response.status_code == 200 + assert response.json()["status"] == "queued" + diff --git a/tests/test_api/test_platform_admin.py b/tests/test_api/test_platform_admin.py index 4c650c0f..206fee50 100644 --- a/tests/test_api/test_platform_admin.py +++ b/tests/test_api/test_platform_admin.py @@ -191,6 +191,43 @@ def test_platform_reset_password(platform_admin_client, client, db_session, enab assert login.status_code == 200 +def test_platform_reset_password_rejects_weak_password( + platform_admin_client, db_session, enable_local_password +): + org = Organization(name="Weak Reset Org") + user = User( + email="weak@reset.org", + password_hash=hash_password(TEST_PASSWORD), + is_active=True, + auth_provider="local", + ) + db_session.add_all([org, user]) + db_session.flush() + db_session.add( + OrganizationMember( + organization_id=org.id, + user_id=user.id, + role=RoleEnum.ADMIN.value, + ) + ) + db_session.commit() + + response = platform_admin_client.post( + f"/api/v1/platform/organizations/{org.id}/users/{user.id}/reset-password", + json={"new_password": "alllowercase"}, + ) + assert response.status_code == 400 + assert "uppercase" in response.json()["detail"].lower() + + +def test_platform_logout_revokes_access_token(platform_admin_client): + logout = platform_admin_client.post("/api/v1/platform/auth/logout") + assert logout.status_code == 200 + + me = platform_admin_client.get("/api/v1/platform/auth/me") + assert me.status_code == 401 + + def test_create_and_use_signup_reference_code( platform_admin_client, client, db_session, enable_local_password, monkeypatch ): diff --git a/tests/test_core/test_flexprice_test_isolation.py b/tests/test_core/test_flexprice_test_isolation.py new file mode 100644 index 00000000..a280c259 --- /dev/null +++ b/tests/test_core/test_flexprice_test_isolation.py @@ -0,0 +1,55 @@ +"""Regression tests: pytest must never reach real Flexprice APIs.""" + +from uuid import uuid4 + +import httpx +import pytest + +from app.config import settings +from app.services.billing import flexprice_service as fp + + +def test_record_event_is_stubbed_even_when_settings_enabled(): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "dummy-key-for-test" + + assert ( + fp.record_event("metrics.ai_assist", uuid4(), uuid4(), properties={"mode": "x"}) + is False + ) + + +def test_ensure_customer_is_stubbed_even_when_settings_enabled(): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "dummy-key-for-test" + + fp.ensure_customer(uuid4(), name="Test Org", email="test@example.com") + + +def test_record_wrappers_do_not_invoke_flexprice_sdk(): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "dummy-key-for-test" + org_id = uuid4() + ws_id = uuid4() + + fp.record_metrics_llm_assist(org_id, uuid4(), workspace_id=ws_id, mode="description") + fp.record_call_import_batch_created( + org_id, + uuid4(), + workspace_id=ws_id, + total_rows=1, + source="csv", + ) + fp.record_test_agent_conversation_ended(org_id, uuid4(), workspace_id=ws_id, duration_seconds=125.0) + + +def test_flexprice_sdk_construction_is_forbidden(): + with pytest.raises(RuntimeError, match="Real Flexprice SDK invoked during pytest"): + from flexprice import Flexprice + + Flexprice(server_url="https://us.api.flexprice.io/v1", api_key_auth="x") + + +def test_httpx_to_flexprice_host_is_blocked(): + with pytest.raises(RuntimeError, match="Blocked Flexprice HTTP during pytest"): + httpx.get("https://us.api.flexprice.io/v1/customers") diff --git a/tests/test_core/test_security_headers_middleware.py b/tests/test_core/test_security_headers_middleware.py index 64ae2e8b..5b7f9af3 100644 --- a/tests/test_core/test_security_headers_middleware.py +++ b/tests/test_core/test_security_headers_middleware.py @@ -77,6 +77,40 @@ def test_csp_enforcing_header_when_report_only_disabled(security_client, monkeyp assert "Content-Security-Policy-Report-Only" not in response.headers +def test_csp_allows_voice_provider_connect_src(security_client, monkeypatch): + monkeypatch.setattr(settings, "CSP_ENABLED", True) + monkeypatch.setattr(settings, "CSP_REPORT_ONLY", False) + + response = security_client.get("/health") + policy = response.headers["Content-Security-Policy"] + + assert "https://api.vapi.ai" in policy + assert "https://*.daily.co" in policy + assert "wss://*.livekit.cloud" in policy + assert "https://api.elevenlabs.io" in policy + assert "https://*.ingest.sentry.io" in policy + assert "'unsafe-eval'" in policy + assert "blob:" in policy + assert "https://c.daily.co" in policy + assert "worker-src 'self' blob:" in policy + + +def test_csp_allows_frame_src_for_pdf_preview_and_voice(security_client, monkeypatch): + monkeypatch.setattr(settings, "CSP_ENABLED", True) + monkeypatch.setattr(settings, "CSP_REPORT_ONLY", False) + + response = security_client.get("/health") + policy = response.headers["Content-Security-Policy"] + + assert "frame-src 'self' blob:" in policy + assert "https://*.daily.co" in policy + assert "https://*.s3.amazonaws.com" in policy + assert "https://*.amazonaws.com" in policy + assert "https://*.cloudfront.net" in policy + assert "https://storage.googleapis.com" in policy + assert "https://*.blob.core.windows.net" in policy + + def test_asset_routes_use_long_cache(security_client): response = security_client.get("/assets/app.js") diff --git a/tests/test_scripts/test_setup_flexprice_meters.py b/tests/test_scripts/test_setup_flexprice_meters.py new file mode 100644 index 00000000..64026479 --- /dev/null +++ b/tests/test_scripts/test_setup_flexprice_meters.py @@ -0,0 +1,190 @@ +from unittest.mock import MagicMock + +from scripts.setup_flexprice_meters import ( + AGENT_PLAYGROUND_PRIMARY_EVENT, + EVALUATOR_RUN_COMPLETED_EVENT, + GEPA_PRIMARY_EVENT, + JUDGE_ALIGNMENT_PRIMARY_EVENT, + LICENSE_FEATURES, + METRICS_AI_ASSIST_EVENT, + METRIC_STUDIO_PRIMARY_EVENT, + PLAN_BILLABLE_METERS, + SCENARIO_AI_TEXT_EVENT, + VOICE_PLAYGROUND_PRIMARY_EVENT, + _feature_meter_is_canonical, + _pick_canonical_meter, + meter_aggregation_matches, + repair_license_features, +) +from app.api.v1.routes.call_import_evaluations import ( + DISCOVERED_METRICS_KEY, + _is_metric_scores_meta_key, +) + + +def test_meter_aggregation_matches_sum_quantity(): + meter = {"aggregation": {"type": "SUM", "field": "quantity"}} + assert meter_aggregation_matches(meter, agg_type="SUM", agg_field="quantity") is True + + +def test_meter_aggregation_matches_rejects_count(): + meter = {"aggregation": {"type": "COUNT"}} + assert meter_aggregation_matches(meter, agg_type="SUM", agg_field="quantity") is False + + +def test_call_imports_plan_has_separate_paid_meters(): + call_import_lines = [row for row in PLAN_BILLABLE_METERS if row["product"] == "Call Imports"] + paid_events = {row["event_name"] for row in call_import_lines if row.get("charge")} + assert paid_events == { + "call_import.evaluation_completed", + "call_import.recording_minutes_billed", + "call_import.pdf_report_generated", + "call_import.user_insights_generated", + "call_import.prompt_improvements_generated", + } + + +def test_agent_playground_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "agent_playground") + assert spec["event_name"] == AGENT_PLAYGROUND_PRIMARY_EVENT + assert spec["event_name"] == "playground.evaluation_completed" + assert spec["aggregation"] == {"type": "SUM", "field": "billable_minutes"} + + +def test_voice_playground_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "voice_playground") + assert spec["event_name"] == VOICE_PLAYGROUND_PRIMARY_EVENT + assert spec["aggregation"] == {"type": "SUM", "field": "quantity"} + + +def test_evaluators_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "evaluators") + assert spec["event_name"] == EVALUATOR_RUN_COMPLETED_EVENT + assert spec["aggregation"] == {"type": "COUNT"} + + +def test_evaluators_plan_includes_audio_minutes_meter(): + evaluator_lines = [row for row in PLAN_BILLABLE_METERS if row["product"] == "Evaluators"] + paid_events = {row["event_name"] for row in evaluator_lines if row.get("charge")} + assert paid_events == { + "evaluator.run_completed", + "evaluator.recording_minutes_billed", + } + + +def test_gepa_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "gepa_optimization") + assert spec["event_name"] == GEPA_PRIMARY_EVENT + assert spec["aggregation"] == {"type": "SUM", "field": "quantity"} + + +def test_judge_alignment_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "judge_alignment") + assert spec["event_name"] == JUDGE_ALIGNMENT_PRIMARY_EVENT + assert spec["aggregation"] == {"type": "SUM", "field": "quantity"} + + +def test_metrics_ai_assist_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "metrics_ai_assist") + assert spec["event_name"] == METRICS_AI_ASSIST_EVENT + assert spec["aggregation"] == {"type": "COUNT"} + + +def test_metric_studio_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "metric_studio") + assert spec["event_name"] == METRIC_STUDIO_PRIMARY_EVENT + assert spec["aggregation"] == {"type": "SUM", "field": "quantity"} + + +def test_scenario_ai_license_feature_spec(): + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "scenario_ai") + assert spec["event_name"] == SCENARIO_AI_TEXT_EVENT + assert spec["aggregation"] == {"type": "COUNT"} + + +def test_prompt_partials_license_feature_spec(): + from scripts.setup_flexprice_meters import PROMPT_PARTIAL_AI_ASSISTED_EVENT + + spec = next(item for item in LICENSE_FEATURES if item["lookup_key"] == "prompt_partials") + assert spec["event_name"] == PROMPT_PARTIAL_AI_ASSISTED_EVENT + assert spec["aggregation"] == {"type": "COUNT"} + + +def test_metric_scores_meta_keys(): + assert _is_metric_scores_meta_key("_billing") is True + assert _is_metric_scores_meta_key(DISCOVERED_METRICS_KEY) is True + assert _is_metric_scores_meta_key("parent-id__discovered") is True + assert _is_metric_scores_meta_key("550e8400-e29b-41d4-a716-446655440000") is False + + +def test_pick_canonical_meter_prefers_priced_meter(): + meters = [ + {"id": "m-old", "aggregation": {"type": "SUM", "field": "quantity"}, "status": "published"}, + {"id": "m-new", "aggregation": {"type": "SUM", "field": "quantity"}, "status": "published"}, + ] + prices = [{"meter_id": "m-new", "amount": "0", "status": "published"}] + picked = _pick_canonical_meter(meters, prices, agg_type="SUM", agg_field="quantity") + assert picked["id"] == "m-new" + + +def test_feature_meter_is_canonical(): + feature = {"meter_id": "m1"} + meter = { + "id": "m1", + "status": "published", + "aggregation": {"type": "SUM", "field": "quantity"}, + } + assert _feature_meter_is_canonical( + feature, + meter, + canonical_meter_id="m1", + agg_type="SUM", + agg_field="quantity", + ) + + +def test_repair_license_features_dry_run_flags_stale_call_imports(): + client = MagicMock() + sdk = MagicMock() + client.get.side_effect = [ + MagicMock( + status_code=200, + json=lambda: { + "items": [ + { + "id": "meter_old", + "event_name": "call_import.batch_created", + "aggregation": {"type": "COUNT"}, + "status": "archived", + }, + { + "id": "meter_new", + "event_name": "call_import.batch_created", + "aggregation": {"type": "SUM", "field": "quantity"}, + "status": "published", + }, + ], + "pagination": {"total": 2}, + }, + ), + MagicMock(status_code=200, json=lambda: {"items": [], "pagination": {"total": 0}}), + MagicMock( + status_code=200, + json=lambda: { + "items": [ + { + "id": "feat_1", + "lookup_key": "call_imports", + "meter_id": "meter_old", + } + ], + "pagination": {"total": 1}, + }, + ), + ] + client.get.return_value.raise_for_status = MagicMock() + + result = repair_license_features(client, sdk, dry_run=True) + repaired_keys = {row["lookup_key"] for row in result["repaired"]} + assert "call_imports" in repaired_keys + sdk.features.delete_feature.assert_not_called() diff --git a/tests/test_services/test_flexprice_service.py b/tests/test_services/test_flexprice_service.py index 714cead5..912d2b61 100644 --- a/tests/test_services/test_flexprice_service.py +++ b/tests/test_services/test_flexprice_service.py @@ -10,11 +10,16 @@ @pytest.fixture(autouse=True) -def reset_flexprice_settings(): +def reset_flexprice_settings(monkeypatch): + monkeypatch.setenv("FLEXPRICE_TEST_ALLOW", "1") previous = ( settings.FLEXPRICE_ENABLED, settings.FLEXPRICE_API_KEY, settings.FLEXPRICE_API_HOST, + settings.FLEXPRICE_AUTO_SUBSCRIBE, + settings.FLEXPRICE_DEFAULT_PLAN_ID, + settings.FLEXPRICE_DEFAULT_CURRENCY, + settings.FLEXPRICE_DEFAULT_BILLING_PERIOD, svc._disabled_skip_logged, ) svc._disabled_skip_logged = False @@ -23,6 +28,10 @@ def reset_flexprice_settings(): settings.FLEXPRICE_ENABLED, settings.FLEXPRICE_API_KEY, settings.FLEXPRICE_API_HOST, + settings.FLEXPRICE_AUTO_SUBSCRIBE, + settings.FLEXPRICE_DEFAULT_PLAN_ID, + settings.FLEXPRICE_DEFAULT_CURRENCY, + settings.FLEXPRICE_DEFAULT_BILLING_PERIOD, svc._disabled_skip_logged, ) = previous @@ -98,47 +107,99 @@ def test_ensure_customer_swallows_already_exists(mock_flexprice): @patch("flexprice.Flexprice") -def test_record_blind_test_share_created_no_op_when_disabled(mock_flexprice): - settings.FLEXPRICE_ENABLED = False +def test_ensure_subscription_no_op_when_auto_subscribe_disabled(mock_flexprice): + settings.FLEXPRICE_ENABLED = True settings.FLEXPRICE_API_KEY = "test-key" + settings.FLEXPRICE_AUTO_SUBSCRIBE = False + settings.FLEXPRICE_DEFAULT_PLAN_ID = "plan_test" - svc.record_blind_test_share_created(uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4()) + svc.ensure_subscription(uuid4()) mock_flexprice.assert_not_called() @patch("flexprice.Flexprice") -def test_record_blind_test_share_created_ingests_event(mock_flexprice): +def test_ensure_subscription_no_op_when_plan_id_missing(mock_flexprice): settings.FLEXPRICE_ENABLED = True settings.FLEXPRICE_API_KEY = "test-key" + settings.FLEXPRICE_AUTO_SUBSCRIBE = True + settings.FLEXPRICE_DEFAULT_PLAN_ID = None - org_id = uuid4() - share_id = uuid4() - workspace_id = uuid4() - comparison_id = uuid4() + svc.ensure_subscription(uuid4()) + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_ensure_subscription_creates_when_none_exists(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + settings.FLEXPRICE_API_HOST = "https://api.cloud.flexprice.io/v1" + settings.FLEXPRICE_AUTO_SUBSCRIBE = True + settings.FLEXPRICE_DEFAULT_PLAN_ID = "plan_01KVT8BTT0HRB419QVCTNHS9RV" + settings.FLEXPRICE_DEFAULT_CURRENCY = "usd" + settings.FLEXPRICE_DEFAULT_BILLING_PERIOD = "MONTHLY" + + org_id = uuid4() mock_client = MagicMock() + mock_client.subscriptions.query_subscription.return_value = MagicMock(items=[]) + mock_client.subscriptions.create_subscription.return_value = MagicMock( + id="sub_test123" + ) mock_flexprice.return_value.__enter__.return_value = mock_client - svc.record_blind_test_share_created( - org_id, - share_id, - workspace_id=workspace_id, - comparison_id=comparison_id, - ) + svc.ensure_subscription(org_id) - mock_client.events.ingest_event.assert_called_once_with( - event_name="blind_test.share_created", + mock_client.subscriptions.query_subscription.assert_called_once_with( external_customer_id=str(org_id), - event_id=str(share_id), - source="efficientai", - properties={ - "share_id": str(share_id), - "workspace_id": str(workspace_id), - "comparison_id": str(comparison_id), - "feature": "voice_playground", - }, + plan_id="plan_01KVT8BTT0HRB419QVCTNHS9RV", + limit=1, + ) + mock_client.subscriptions.create_subscription.assert_called_once_with( + billing_period="MONTHLY", + currency="usd", + plan_id="plan_01KVT8BTT0HRB419QVCTNHS9RV", + external_customer_id=str(org_id), + subscription_status="active", + ) + + +@patch("flexprice.Flexprice") +def test_ensure_subscription_skips_when_subscription_exists(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + settings.FLEXPRICE_AUTO_SUBSCRIBE = True + settings.FLEXPRICE_DEFAULT_PLAN_ID = "plan_test" + + mock_client = MagicMock() + mock_client.subscriptions.query_subscription.return_value = MagicMock( + items=[MagicMock(id="sub_existing")] ) + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.ensure_subscription(uuid4()) + + mock_client.subscriptions.create_subscription.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_blind_test_share_created_no_op_when_disabled(mock_flexprice): + settings.FLEXPRICE_ENABLED = False + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_blind_test_share_created(uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4()) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_blind_test_share_created_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_blind_test_share_created(uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4()) + + mock_flexprice.assert_not_called() @patch("flexprice.Flexprice") @@ -150,7 +211,9 @@ def test_record_blind_test_share_created_logs_and_swallows_errors(mock_flexprice mock_client.events.ingest_event.side_effect = RuntimeError("network down") mock_flexprice.return_value.__enter__.return_value = mock_client - svc.record_blind_test_share_created(uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4()) + svc.record_blind_test_response_submitted( + uuid4(), uuid4(), share_id=uuid4(), workspace_id=uuid4(), response_count=1 + ) @patch("flexprice.Flexprice") @@ -165,7 +228,9 @@ def test_ingest_usage_event_falls_back_to_request_dict(mock_flexprice): ] mock_flexprice.return_value.__enter__.return_value = mock_client - svc.record_blind_test_share_created(uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4()) + svc.record_blind_test_response_submitted( + uuid4(), uuid4(), share_id=uuid4(), workspace_id=uuid4(), response_count=1 + ) assert mock_client.events.ingest_event.call_count == 2 assert "request" in mock_client.events.ingest_event.call_args_list[1].kwargs @@ -195,9 +260,10 @@ def test_record_call_import_batch_created_includes_volume_properties(mock_flexpr mock_client.events.ingest_event.assert_called_once() payload = mock_client.events.ingest_event.call_args.kwargs assert payload["event_name"] == "call_import.batch_created" - assert payload["properties"]["total_rows"] == "42" assert payload["properties"]["quantity"] == "42" assert payload["properties"]["feature"] == "call_imports" + assert payload["properties"]["source"] == "csv" + assert payload["properties"]["provider"] == "exotel" @patch("flexprice.Flexprice") @@ -220,6 +286,7 @@ def test_record_call_import_evaluation_completed_meters_pass_delta(mock_flexpric call_import_id=call_import_id, rows_billed=50, completed_total=1950, + total_rows=2000, metric_count=5, ) @@ -231,17 +298,118 @@ def test_record_call_import_evaluation_completed_meters_pass_delta(mock_flexpric properties={ "workspace_id": str(workspace_id), "feature": "call_imports", - "call_import_id": str(call_import_id), + "quantity": "50", "evaluation_id": str(evaluation_id), - "rows_billed": "50", + "call_import_id": str(call_import_id), "completed_total": "1950", + "total_rows": "2000", "metric_count": "5", - "quantity": "50", + "rows_billed": "50", }, ) assert accepted is True +@patch("flexprice.Flexprice") +def test_record_call_import_evaluation_started(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_call_import_evaluation_started( + uuid4(), + uuid4(), + workspace_id=uuid4(), + call_import_id=uuid4(), + total_rows=100, + metric_count=3, + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_call_import_recording_minutes_billed(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + eval_row_id = uuid4() + evaluation_id = uuid4() + workspace_id = uuid4() + call_import_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + accepted = svc.record_call_import_recording_minutes_billed( + org_id, + eval_row_id, + workspace_id=workspace_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=90, + billable_minutes=2, + ) + + mock_client.events.ingest_event.assert_called_once_with( + event_name="call_import.recording_minutes_billed", + external_customer_id=str(org_id), + event_id=str(eval_row_id), + source="efficientai", + properties={ + "workspace_id": str(workspace_id), + "feature": "call_imports", + "billable_minutes": "2", + "quantity": "2", + "evaluation_row_id": str(eval_row_id), + "evaluation_id": str(evaluation_id), + "call_import_id": str(call_import_id), + "audio_seconds": "90", + }, + ) + assert accepted is True + + +@patch("flexprice.Flexprice") +def test_record_call_import_pdf_report_generated(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + pdf_report_id = uuid4() + evaluation_id = uuid4() + workspace_id = uuid4() + call_import_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_call_import_pdf_report_generated( + org_id, + pdf_report_id, + workspace_id=workspace_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + report_type="external", + ) + + mock_client.events.ingest_event.assert_called_once_with( + event_name="call_import.pdf_report_generated", + external_customer_id=str(org_id), + event_id=str(pdf_report_id), + source="efficientai", + properties={ + "workspace_id": str(workspace_id), + "feature": "call_imports", + "quantity": "1", + "pdf_report_id": str(pdf_report_id), + "evaluation_id": str(evaluation_id), + "call_import_id": str(call_import_id), + "report_type": "external", + }, + ) + + @patch("flexprice.Flexprice") def test_record_event_returns_false_when_disabled(mock_flexprice): settings.FLEXPRICE_ENABLED = False @@ -260,37 +428,598 @@ def test_record_event_returns_false_when_disabled(mock_flexprice): @patch("flexprice.Flexprice") -def test_record_playground_call_evaluated_uses_per_attempt_event_id(mock_flexprice): +def test_record_playground_call_evaluated_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_playground_call_evaluated( + uuid4(), + "attempt-1", + evaluator_result_id=uuid4(), + workspace_id=uuid4(), + call_short_id="123456", + metric_count=4, + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_event_skips_deprecated_playground_call_evaluated(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + assert ( + svc.record_event( + svc.PLAYGROUND_CALL_EVALUATED, + uuid4(), + "attempt-1", + properties={"workspace_id": uuid4(), "call_short_id": "123456"}, + ) + is False + ) + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_playground_web_call_started_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_playground_web_call_started( + uuid4(), "abc123", workspace_id=uuid4(), agent_id=uuid4() + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_playground_websocket_session_started_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_playground_websocket_session_started(uuid4(), "ws456", workspace_id=uuid4()) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_playground_evaluation_completed_includes_feature(mock_flexprice): settings.FLEXPRICE_ENABLED = True settings.FLEXPRICE_API_KEY = "test-key" org_id = uuid4() evaluator_result_id = uuid4() - evaluation_attempt_id = f"{evaluator_result_id}:celery-task-abc" + evaluation_attempt_id = f"{evaluator_result_id}:task-1" workspace_id = uuid4() mock_client = MagicMock() mock_flexprice.return_value.__enter__.return_value = mock_client - svc.record_playground_call_evaluated( + svc.record_playground_evaluation_completed( org_id, evaluation_attempt_id, evaluator_result_id=evaluator_result_id, workspace_id=workspace_id, - call_short_id="123456", - metric_count=4, + call_short_id="999", + duration_seconds=12.5, + metric_count=2, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "playground.evaluation_completed" + assert payload["properties"]["feature"] == "agent_playground" + assert payload["properties"]["quantity"] == "1" + assert payload["properties"]["billable_minutes"] == "1" + assert payload["properties"]["duration_seconds"] == "12.5" + assert payload["properties"]["metric_count"] == "2" + assert payload["properties"]["billable_minutes"] == "1" + + +def test_event_properties_ui_surface_is_audit_only(): + props = svc._event_properties( + uuid4(), + svc.FEATURE_AGENT_PLAYGROUND, + quantity=1, + ui_surface=svc.UI_SURFACE_AGENTS_TALK, + ) + assert props["feature"] == "agent_playground" + assert props["ui_surface"] == "agents_talk" + assert "ui_surface" not in svc._billing_properties(uuid4(), svc.FEATURE_AGENT_PLAYGROUND, quantity=1) + + +def test_event_properties_omits_blank_ui_surface(): + props = svc._event_properties(uuid4(), svc.FEATURE_EVALUATORS, quantity=1, ui_surface=" ") + assert "ui_surface" not in props + + +@patch("flexprice.Flexprice") +def test_record_playground_evaluation_completed_includes_ui_surface(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_playground_evaluation_completed( + uuid4(), + "attempt-1", + evaluator_result_id=uuid4(), + workspace_id=uuid4(), + call_short_id="999", + duration_seconds=30, + ui_surface=svc.UI_SURFACE_AGENT_PLAYGROUND, + ) + + props = mock_client.events.ingest_event.call_args.kwargs["properties"] + assert props["ui_surface"] == "agent_playground" + + +@patch("flexprice.Flexprice") +def test_record_test_agent_conversation_started_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_test_agent_conversation_started(uuid4(), uuid4(), workspace_id=uuid4()) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_test_agent_conversation_ended_bills_one_conversation(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + conversation_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_test_agent_conversation_ended( + org_id, + conversation_id, + workspace_id=workspace_id, + duration_seconds=120.0, + turn_count=8, + ) + + mock_client.events.ingest_event.assert_called_once_with( + event_name="test_agent.conversation_ended", + external_customer_id=str(org_id), + event_id=str(conversation_id), + source="efficientai", + properties={ + "workspace_id": str(workspace_id), + "feature": "agent_playground", + "quantity": "2", + "billable_minutes": "2", + "conversation_id": str(conversation_id), + "duration_seconds": "120.0", + "turn_count": "8", + }, + ) + + +@patch("flexprice.Flexprice") +def test_record_test_agent_conversation_started_with_metadata_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_test_agent_conversation_started( + uuid4(), + uuid4(), + workspace_id=uuid4(), + result_id="123456", + agent_id=uuid4(), + call_short_id="654321", + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_evaluator_run_requested_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_evaluator_run_requested( + uuid4(), uuid4(), workspace_id=uuid4(), quantity=3 + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_evaluator_run_completed_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + evaluator_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_evaluator_run_completed( + org_id, + "res-001", + workspace_id=workspace_id, + evaluator_id=evaluator_id, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "evaluator.run_completed" + assert payload["properties"]["feature"] == "evaluators" + assert payload["properties"]["quantity"] == "1" + + +@patch("flexprice.Flexprice") +def test_record_judge_alignment_run_completed_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + run_id = uuid4() + dataset_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_judge_alignment_run_completed( + org_id, + run_id, + workspace_id=workspace_id, + dataset_id=dataset_id, + samples_scored=12, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "judge_alignment.run_completed" + assert payload["properties"]["feature"] == "judge_alignment" + assert payload["properties"]["quantity"] == "12" + + +@patch("flexprice.Flexprice") +def test_record_judge_alignment_run_completed_skips_when_no_samples(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_judge_alignment_run_completed( + uuid4(), uuid4(), workspace_id=uuid4(), dataset_id=uuid4(), samples_scored=0 + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_prompt_optimization_run_completed_bills_candidates(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + run_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_prompt_optimization_run_completed( + org_id, + run_id, + workspace_id=workspace_id, + agent_id=uuid4(), + candidates_count=5, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "prompt_optimization.run_completed" + assert payload["properties"]["feature"] == "gepa_optimization" + assert payload["properties"]["quantity"] == "5" + + +@patch("flexprice.Flexprice") +def test_record_evaluator_recording_minutes_billed(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + result_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + accepted = svc.record_evaluator_recording_minutes_billed( + org_id, + result_id, + workspace_id=workspace_id, + duration_seconds=125.0, ) mock_client.events.ingest_event.assert_called_once_with( - event_name="playground.call_evaluated", + event_name="evaluator.recording_minutes_billed", external_customer_id=str(org_id), - event_id=evaluation_attempt_id, + event_id=str(result_id), source="efficientai", properties={ "workspace_id": str(workspace_id), - "call_short_id": "123456", - "evaluator_result_id": str(evaluator_result_id), - "evaluation_attempt_id": evaluation_attempt_id, - "metric_count": "4", + "feature": "evaluators", + "quantity": "3", + "billable_minutes": "3", + "evaluator_result_id": str(result_id), + "duration_seconds": "125.0", + "audio_seconds": "125", }, ) + assert accepted is True + + +@patch("flexprice.Flexprice") +def test_record_observability_call_evaluated_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_observability_call_evaluated(uuid4(), "call-1", workspace_id=uuid4()) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_metrics_ai_assist_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + request_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_metrics_ai_assist( + org_id, + request_id, + workspace_id=workspace_id, + mode="generate", + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "metrics.ai_assist" + assert payload["properties"]["feature"] == "metrics_ai_assist" + assert payload["properties"]["quantity"] == "1" + + +@patch("flexprice.Flexprice") +def test_record_metric_studio_run_completed_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + run_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_metric_studio_run_completed( + org_id, + run_id, + workspace_id=workspace_id, + run_status="completed", + total_items=3, + completed_items=3, + failed_items=0, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "metric_studio.run_completed" + assert payload["properties"]["feature"] == "metric_studio" + assert payload["properties"]["quantity"] == "3" + + +@patch("flexprice.Flexprice") +def test_record_scenario_ai_text_generated_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + request_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_scenario_ai_text_generated( + org_id, + request_id, + workspace_id=workspace_id, + model="gpt-4o", + purpose="scenario_description", + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "scenario.ai_text_generated" + assert payload["properties"]["feature"] == "scenario_ai" + assert payload["properties"]["quantity"] == "1" + + +@patch("flexprice.Flexprice") +def test_record_prompt_partial_ai_assisted_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + request_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_prompt_partial_ai_assisted( + org_id, + request_id, + workspace_id=workspace_id, + mode="generate", + model="gpt-4o", + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "prompt_partial.ai_assisted" + assert payload["properties"]["feature"] == "prompt_partials" + assert payload["properties"]["mode"] == "generate" + + +@patch("flexprice.Flexprice") +def test_record_call_import_user_insights_generated_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + evaluation_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_call_import_user_insights_generated( + org_id, + uuid4(), + workspace_id=workspace_id, + evaluation_id=evaluation_id, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "call_import.user_insights_generated" + assert payload["properties"]["feature"] == "call_imports" + + +@patch("flexprice.Flexprice") +def test_record_agent_test_setup_generated_includes_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + workspace_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_agent_test_setup_generated( + org_id, + uuid4(), + workspace_id=workspace_id, + purpose="full_setup", + scenario_count=3, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "agent.test_setup_generated" + assert payload["properties"]["feature"] == "agent_playground" + assert payload["properties"]["purpose"] == "full_setup" + + +@patch("flexprice.Flexprice") +def test_record_tts_generation_started_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_tts_generation_started( + uuid4(), uuid4(), workspace_id=uuid4(), sample_count=4 + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_tts_sample_synthesized_includes_voice_playground_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + sample_id = uuid4() + comparison_id = uuid4() + workspace_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_tts_sample_synthesized( + org_id, + sample_id, + workspace_id=workspace_id, + comparison_id=comparison_id, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "tts.sample_synthesized" + assert payload["properties"]["feature"] == "voice_playground" + assert payload["properties"]["quantity"] == "1" + + +@patch("flexprice.Flexprice") +def test_record_tts_report_requested_does_not_ingest(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + svc.record_tts_report_requested( + uuid4(), uuid4(), workspace_id=uuid4(), comparison_id=uuid4() + ) + + mock_flexprice.assert_not_called() + + +@patch("flexprice.Flexprice") +def test_record_tts_report_completed_includes_voice_playground_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + report_job_id = uuid4() + comparison_id = uuid4() + workspace_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_tts_report_completed( + org_id, + report_job_id, + workspace_id=workspace_id, + comparison_id=comparison_id, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "tts.report_completed" + assert payload["properties"]["feature"] == "voice_playground" + assert payload["properties"]["quantity"] == "1" + + +@patch("flexprice.Flexprice") +def test_record_blind_test_response_submitted_includes_voice_playground_feature(mock_flexprice): + settings.FLEXPRICE_ENABLED = True + settings.FLEXPRICE_API_KEY = "test-key" + + org_id = uuid4() + response_id = uuid4() + share_id = uuid4() + workspace_id = uuid4() + + mock_client = MagicMock() + mock_flexprice.return_value.__enter__.return_value = mock_client + + svc.record_blind_test_response_submitted( + org_id, + response_id, + share_id=share_id, + workspace_id=workspace_id, + response_count=3, + ) + + payload = mock_client.events.ingest_event.call_args.kwargs + assert payload["event_name"] == "blind_test.response_submitted" + assert payload["properties"]["feature"] == "voice_playground" + assert payload["properties"]["quantity"] == "3" diff --git a/tests/test_services/test_flexprice_tracking_wiring.py b/tests/test_services/test_flexprice_tracking_wiring.py new file mode 100644 index 00000000..44ae6865 --- /dev/null +++ b/tests/test_services/test_flexprice_tracking_wiring.py @@ -0,0 +1,174 @@ +"""Catalog tests: Flexprice event wiring expectations by product area.""" + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +VOICE_PLAYGROUND_WIRED = { + "blind_test.share_created": "app/api/v1/routes/voice_playground.py", + "blind_test.response_submitted": "app/api/v1/routes/public_blind_test.py", + "tts.generation_started": "app/api/v1/routes/voice_playground.py", + "tts.sample_synthesized": "app/workers/tasks/tts_comparison.py", + "tts.report_requested": "app/api/v1/routes/voice_playground.py", + "tts.report_completed": "app/workers/tasks/tts_report.py", +} + +AGENT_PLAYGROUND_WIRED = { + "playground.web_call_started": "app/api/v1/routes/playground.py", + "playground.websocket_session_started": "app/api/v1/routes/playground.py", + "playground.evaluation_completed": "app/workers/tasks/process_evaluator_result.py", + "test_agent.conversation_ended": "app/api/v1/routes/test_agents.py", +} + +EVALUATORS_WIRED = { + "evaluator.run_requested": "app/api/v1/routes/evaluators.py", + "evaluator.run_completed": "app/workers/tasks/process_evaluator_result.py", + "evaluator.recording_minutes_billed": "app/workers/tasks/process_evaluator_result.py", +} + +JUDGE_ALIGNMENT_WIRED = { + "judge_alignment.run_started": "app/api/v1/routes/judge_alignment.py", + "judge_alignment.run_completed": "app/workers/tasks/run_judge_alignment.py", +} + +METRICS_AI_ASSIST_WIRED = { + "metrics.ai_assist": "app/api/v1/routes/metrics.py", +} + +METRIC_STUDIO_WIRED = { + "metric_studio.item_evaluated": "app/workers/tasks/evaluate_studio_run_item.py", + "metric_studio.run_completed": "app/services/metric_studio/run_rollup.py", +} + +SCENARIO_AI_WIRED = { + "scenario.ai_text_generated": [ + "app/api/v1/routes/chat.py", + ], +} + +PROMPT_PARTIALS_WIRED = { + "prompt_partial.ai_assisted": [ + "app/api/v1/routes/prompt_partials.py", + "app/workers/tasks/agent_flowchart_jobs.py", + ], +} + +CALL_IMPORT_AI_ADDONS_WIRED = { + "call_import.user_insights_generated": [ + "app/workers/tasks/generate_evaluation_user_insights.py", + ], + "call_import.prompt_improvements_generated": [ + "app/workers/tasks/generate_evaluation_prompt_improvements.py", + ], +} + +AGENT_AI_HELPERS_WIRED = { + "persona.prompt_generated": ["app/api/v1/routes/personas.py"], + "agent.test_setup_generated": ["app/api/v1/routes/agents.py"], +} + + +def _file_contains(path: str, needle: str) -> bool: + text = (REPO_ROOT / path).read_text(encoding="utf-8") + return needle in text + + +def test_voice_playground_events_are_wired(): + for event_name, path in VOICE_PLAYGROUND_WIRED.items(): + record_fn = { + "blind_test.share_created": "record_blind_test_share_created", + "blind_test.response_submitted": "record_blind_test_response_submitted", + "tts.generation_started": "record_tts_generation_started", + "tts.sample_synthesized": "record_tts_sample_synthesized", + "tts.report_requested": "record_tts_report_requested", + "tts.report_completed": "record_tts_report_completed", + }[event_name] + assert _file_contains(path, record_fn), f"{event_name} missing {record_fn} in {path}" + + +def test_agent_playground_events_are_wired(): + for event_name, path in AGENT_PLAYGROUND_WIRED.items(): + record_fn = { + "playground.web_call_started": "record_playground_web_call_started", + "playground.websocket_session_started": "record_playground_websocket_session_started", + "playground.evaluation_completed": "record_playground_evaluation_completed", + "test_agent.conversation_ended": "record_test_agent_conversation_ended", + }[event_name] + assert _file_contains(path, record_fn), f"{event_name} missing {record_fn} in {path}" + + +def test_evaluator_events_are_wired(): + for event_name, path in EVALUATORS_WIRED.items(): + record_fn = { + "evaluator.run_requested": "record_evaluator_run_requested", + "evaluator.run_completed": "record_evaluator_run_completed", + "evaluator.recording_minutes_billed": "record_evaluator_recording_minutes_billed", + }[event_name] + assert _file_contains(path, record_fn), f"{event_name} missing {record_fn} in {path}" + + +def test_judge_alignment_events_are_wired(): + for event_name, path in JUDGE_ALIGNMENT_WIRED.items(): + record_fn = { + "judge_alignment.run_started": "record_judge_alignment_run_started", + "judge_alignment.run_completed": "record_judge_alignment_run_completed", + }[event_name] + assert _file_contains(path, record_fn), f"{event_name} missing {record_fn} in {path}" + + +def test_metrics_ai_assist_events_are_wired(): + for event_name, path in METRICS_AI_ASSIST_WIRED.items(): + assert _file_contains(path, "record_metrics_llm_assist"), ( + f"{event_name} missing record_metrics_llm_assist in {path}" + ) + + +def test_metric_studio_events_are_wired(): + for event_name, path in METRIC_STUDIO_WIRED.items(): + record_fn = { + "metric_studio.item_evaluated": "record_metric_studio_item_evaluated", + "metric_studio.run_completed": "record_metric_studio_run_completed", + }[event_name] + assert _file_contains(path, record_fn), f"{event_name} missing {record_fn} in {path}" + + +def test_scenario_ai_text_events_are_wired(): + for event_name, paths in SCENARIO_AI_WIRED.items(): + for path in paths: + assert _file_contains(path, "record_scenario_ai_text_generated") or _file_contains( + path, "record_chat_completion" + ), f"{event_name} missing scenario flexprice hook in {path}" + + +def test_prompt_partial_ai_events_are_wired(): + for event_name, paths in PROMPT_PARTIALS_WIRED.items(): + for path in paths: + assert _file_contains(path, "record_prompt_partial_ai_assisted"), ( + f"{event_name} missing record_prompt_partial_ai_assisted in {path}" + ) + + +def test_call_import_ai_addon_events_are_wired(): + record_fns = { + "call_import.user_insights_generated": "record_call_import_user_insights_generated", + "call_import.prompt_improvements_generated": ( + "record_call_import_prompt_improvements_generated" + ), + } + for event_name, paths in CALL_IMPORT_AI_ADDONS_WIRED.items(): + for path in paths: + assert _file_contains(path, record_fns[event_name]), ( + f"{event_name} missing {record_fns[event_name]} in {path}" + ) + + +def test_agent_ai_helper_events_are_wired(): + record_fns = { + "persona.prompt_generated": "record_persona_prompt_generated", + "agent.test_setup_generated": "record_agent_test_setup_generated", + } + for event_name, paths in AGENT_AI_HELPERS_WIRED.items(): + for path in paths: + assert _file_contains(path, record_fns[event_name]), ( + f"{event_name} missing {record_fns[event_name]} in {path}" + ) diff --git a/tests/test_services/test_playground_post_call_processing.py b/tests/test_services/test_playground_post_call_processing.py new file mode 100644 index 00000000..1a26ed43 --- /dev/null +++ b/tests/test_services/test_playground_post_call_processing.py @@ -0,0 +1,316 @@ +"""Tests for atomic playground post-call processing.""" + +from uuid import uuid4 + +import pytest + +from app.models.database import ( + Agent, + CallRecording, + CallRecordingSource, + CallRecordingStatus, + EvaluatorResult, + EvaluatorResultStatus, +) +from app.services.playground.post_call_processing import ( + claim_playground_evaluator_result_slot, + merge_playground_call_data, + record_playground_post_call_usage_once, +) + + +def test_merge_playground_call_data_preserves_ui_surface(): + merged = merge_playground_call_data( + {"ui_surface": "agents_talk", "external_usage_recorded": True}, + {"call_status": "ended", "duration_seconds": 30}, + ) + assert merged["ui_surface"] == "agents_talk" + assert merged["external_usage_recorded"] is True + assert merged["call_status"] == "ended" + + +def test_record_playground_post_call_usage_once_preserves_ui_surface( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + call_data={"ui_surface": "agent_playground", "call_status": "ended"}, + ) + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + lambda **_kwargs: None, + ) + + proceed, updated = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics={"call_status": "ended", "duration_seconds": 42}, + ) + assert proceed is True + assert updated["ui_surface"] == "agent_playground" + assert updated["external_usage_recorded"] is True + + +@pytest.fixture +def playground_agent(db_session, org_id, default_workspace): + agent = Agent( + id=uuid4(), + organization_id=org_id, + workspace_id=default_workspace.id, + name="Test Agent", + ) + db_session.add(agent) + db_session.commit() + return agent + + +def _make_playground_recording(db_session, *, org_id, workspace_id, agent_id, **overrides): + recording = CallRecording( + id=overrides.get("id", uuid4()), + organization_id=org_id, + workspace_id=workspace_id, + call_short_id=overrides.get("call_short_id", "654321"), + status=CallRecordingStatus.UPDATED, + source=CallRecordingSource.PLAYGROUND, + call_data=overrides.get("call_data", {}), + provider_call_id=overrides.get("provider_call_id", "provider-call-1"), + provider_platform=overrides.get("provider_platform", "retell"), + agent_id=agent_id, + evaluator_result_id=overrides.get("evaluator_result_id"), + ) + db_session.add(recording) + db_session.commit() + db_session.refresh(recording) + return recording + + +def test_record_playground_post_call_usage_once_is_idempotent( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + call_data={"call_status": "ended"}, + ) + call_metrics = {"call_status": "ended", "duration_seconds": 42} + calls = [] + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + lambda **kwargs: calls.append(kwargs), + ) + + proceed, updated = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics=call_metrics, + ) + assert proceed is True + assert updated["external_usage_recorded"] is True + assert len(calls) == 1 + + proceed_again, updated_again = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics=call_metrics, + ) + assert proceed_again is True + assert updated_again["external_usage_recorded"] is True + assert len(calls) == 1 + + +def test_record_playground_post_call_usage_once_skips_when_evaluator_linked( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + existing = EvaluatorResult( + id=uuid4(), + result_id="111111", + organization_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + name="Existing", + status=EvaluatorResultStatus.COMPLETED.value, + ) + db_session.add(existing) + db_session.flush() + + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + evaluator_result_id=existing.id, + ) + + def _fail_apply(**_kwargs): + raise AssertionError("usage should not be recorded when evaluator already exists") + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + _fail_apply, + ) + + proceed, _ = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics={"call_status": "ended"}, + ) + assert proceed is False + + +def test_claim_playground_evaluator_result_slot_skips_after_evaluator_linked( + db_session, org_id, default_workspace, playground_agent +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + ) + + locked = claim_playground_evaluator_result_slot(db_session, recording.id) + assert locked is not None + + evaluator_result = EvaluatorResult( + result_id="222222", + organization_id=locked.organization_id, + workspace_id=locked.workspace_id, + agent_id=locked.agent_id, + name="Voice AI Call", + status=EvaluatorResultStatus.QUEUED.value, + ) + db_session.add(evaluator_result) + db_session.flush() + locked.evaluator_result_id = evaluator_result.id + db_session.commit() + + locked_again = claim_playground_evaluator_result_slot(db_session, recording.id) + assert locked_again is None + + assert ( + db_session.query(EvaluatorResult) + .filter(EvaluatorResult.id == evaluator_result.id) + .count() + == 1 + ) + + +def test_record_playground_post_call_usage_once_retries_when_storage_fails( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + call_data={"call_status": "ended"}, + ) + + def _fail_storage(**_kwargs): + from app.services.usage.external_agent_usage import ExternalUsageRecordingError + + raise ExternalUsageRecordingError("simulated storage failure") + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + _fail_storage, + ) + + proceed, _ = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="vapi", + call_metrics={ + "costBreakdown": { + "llmPromptTokens": 40, + "llmCompletionTokens": 12, + }, + "durationSeconds": 20, + }, + ) + assert proceed is False + + db_session.refresh(recording) + assert not (recording.call_data or {}).get("external_usage_recorded") + + +def test_record_playground_post_call_usage_once_marks_recorded_without_billable_usage( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + call_data={"call_status": "ended"}, + ) + apply_calls = [] + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + lambda **kwargs: apply_calls.append(kwargs), + ) + + proceed, updated = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics={"call_status": "ended"}, + ) + assert proceed is True + assert updated["external_usage_recorded"] is True + assert len(apply_calls) == 1 + + +def test_record_playground_post_call_usage_once_retries_when_apply_fails( + db_session, org_id, default_workspace, playground_agent, monkeypatch +): + recording = _make_playground_recording( + db_session, + org_id=org_id, + workspace_id=default_workspace.id, + agent_id=playground_agent.id, + call_data={"call_status": "ended"}, + ) + apply_attempts = {"count": 0} + + def _apply_fail_once(**kwargs): + apply_attempts["count"] += 1 + if apply_attempts["count"] == 1: + raise RuntimeError("simulated apply failure") + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.apply_playground_provider_usage_from_call_data", + _apply_fail_once, + ) + + proceed, _ = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics={"call_status": "ended", "duration_seconds": 42}, + ) + assert proceed is False + + db_session.refresh(recording) + assert not (recording.call_data or {}).get("external_usage_recorded") + + proceed, updated = record_playground_post_call_usage_once( + db_session, + recording.id, + provider_platform="retell", + call_metrics={"call_status": "ended", "duration_seconds": 42}, + ) + assert proceed is True + assert updated["external_usage_recorded"] is True + assert apply_attempts["count"] == 2 diff --git a/tests/test_services/test_usage/test_llm_to_llm_evaluator_simulation.py b/tests/test_services/test_usage/test_llm_to_llm_evaluator_simulation.py new file mode 100644 index 00000000..9b26367c --- /dev/null +++ b/tests/test_services/test_usage/test_llm_to_llm_evaluator_simulation.py @@ -0,0 +1,111 @@ +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from app.models.enums import ModelProvider +from app.services.testing.llm_to_llm_evaluator_simulation import ( + run_llm_to_llm_evaluator_simulation, +) + + +def test_run_llm_to_llm_evaluator_simulation_builds_transcript(monkeypatch): + org_id = uuid4() + agent_id = uuid4() + evaluator_id = uuid4() + persona_id = uuid4() + scenario_id = uuid4() + result_id = uuid4() + bundle_id = uuid4() + + responses = iter( + [ + {"text": "Hi, how can I help you today?"}, + {"text": "I need help with my order."}, + {"text": "Sure, what is your order number?"}, + {"text": "Thanks, goodbye."}, + ] + ) + + def _fake_generate(**_kwargs): + return next(responses) + + import app.services.testing.llm_to_llm_evaluator_simulation as sim_mod + + monkeypatch.setattr(sim_mod.llm_service, "generate_response", _fake_generate) + + evaluator = SimpleNamespace( + id=evaluator_id, + evaluator_id="ev-1", + workspace_id=uuid4(), + ) + agent = SimpleNamespace( + id=agent_id, + name="Support Bot", + voice_bundle_id=bundle_id, + description="You help customers with orders.", + provider_prompt=None, + ) + persona = SimpleNamespace( + id=persona_id, + name="Alex", + description="Impatient customer", + gender=None, + max_turns=4, + tts_provider=None, + tts_voice_name=None, + tts_voice_id=None, + ) + scenario = SimpleNamespace( + id=scenario_id, + name="Order status", + description="Check an order", + required_info={"goal": "Get order status"}, + ) + result = SimpleNamespace( + id=result_id, + result_id="res-1", + transcription=None, + speaker_segments=None, + provider_platform=None, + call_data=None, + duration_seconds=None, + ) + voice_bundle = SimpleNamespace( + id=bundle_id, + llm_provider=ModelProvider.OPENAI, + llm_model="gpt-4o-mini", + llm_config=None, + llm_credential_id=None, + ) + db = SimpleNamespace() + + def _query(model): + class _Q: + def filter(self, *_args, **_kwargs): + return self + + def first(self): + if model.__name__ == "VoiceBundle": + return voice_bundle + return None + + return _Q() + + db.query = _query + + output = run_llm_to_llm_evaluator_simulation( + evaluator=evaluator, + result=result, + agent=agent, + persona=persona, + scenario=scenario, + organization_id=org_id, + db=db, + ) + + assert output["provider_platform"] == "internal" + assert result.provider_platform == "internal" + assert "Speaker 1:" in result.transcription + assert "Speaker 2:" in result.transcription + assert result.call_data["source"] == "llm_to_llm_simulation" diff --git a/tests/test_services/test_usage/test_llm_usage.py b/tests/test_services/test_usage/test_llm_usage.py index d08887bd..011cc3c1 100644 --- a/tests/test_services/test_usage/test_llm_usage.py +++ b/tests/test_services/test_usage/test_llm_usage.py @@ -294,12 +294,13 @@ def test_record_call_usage(fake_redis, org_ctx): def test_agent_usage_context_reuses_single_bucket(fake_redis): - """Stable agent context avoids per-call Redis/DB bucket explosion.""" + """Stable evaluator context avoids per-call Redis/DB bucket explosion.""" from app.services.usage.context import usage_context_for_evaluator_result org_id = uuid4() workspace_id = uuid4() agent_id = uuid4() + evaluator_id = uuid4() prefixes = set() for idx in range(3): result = SimpleNamespace( @@ -307,7 +308,7 @@ def test_agent_usage_context_reuses_single_bucket(fake_redis): result_id=f"res-{idx}", organization_id=org_id, workspace_id=workspace_id, - evaluator_id=uuid4(), + evaluator_id=evaluator_id, agent_id=agent_id, ) ctx = usage_context_for_evaluator_result(result) @@ -375,6 +376,7 @@ def _fake_buffer(organization_id, bucket, deltas): captured["organization_id"] = organization_id captured["bucket"] = bucket captured["deltas"] = deltas + return True monkeypatch.setattr(usage_mod, "_buffer_to_postgres", _fake_buffer) diff --git a/tests/test_services/test_usage/test_pre_prod_usage.py b/tests/test_services/test_usage/test_pre_prod_usage.py new file mode 100644 index 00000000..8e6d124a --- /dev/null +++ b/tests/test_services/test_usage/test_pre_prod_usage.py @@ -0,0 +1,304 @@ +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from app.services.usage.context import ( + LLMUsageProductSection, + usage_context_for_evaluator_result, + usage_context_for_metric_studio_run, + usage_context_for_persona_generation, +) +from app.services.usage.external_agent_usage import ( + extract_external_agent_usage, + record_external_agent_usage, +) + + +@pytest.fixture +def fake_redis(monkeypatch): + class _FakeRedis: + def __init__(self): + self.hashes = {} + + def hincrby(self, key, field, amount): + self.hashes.setdefault(key, {}) + self.hashes[key][field] = int(self.hashes[key].get(field, 0)) + int(amount) + + def hgetall(self, key): + return dict(self.hashes.get(key, {})) + + def sadd(self, key, member): + return True + + def smembers(self, key): + return set() + + def expire(self, key, ttl): + return True + + def pipeline(self): + client = self + + class _Pipe: + def hincrby(self, key, field, amount): + client.hincrby(key, field, amount) + return self + + def sadd(self, key, member): + return self + + def expire(self, key, ttl): + return self + + def execute(self): + return [] + + return _Pipe() + + client = _FakeRedis() + from app.services.usage import llm_usage as usage_mod + import app.services.usage.read_cache as read_cache_mod + + usage_mod._redis = client + read_cache_mod._redis = client + + def _forbid_real_redis(*_args, **_kwargs): + raise AssertionError("usage tests must not open real Redis") + + monkeypatch.setattr(usage_mod.redis, "from_url", _forbid_real_redis) + monkeypatch.setattr(read_cache_mod.redis, "from_url", _forbid_real_redis) + yield client + usage_mod._redis = None + read_cache_mod._redis = None + + +def test_usage_context_for_evaluator_result_uses_evaluators_section(): + org_id = uuid4() + workspace_id = uuid4() + agent_id = uuid4() + evaluator_id = uuid4() + persona_id = uuid4() + scenario_id = uuid4() + result_id = uuid4() + + ctx = usage_context_for_evaluator_result( + SimpleNamespace( + id=result_id, + result_id="res-42", + organization_id=org_id, + workspace_id=workspace_id, + agent_id=agent_id, + evaluator_id=evaluator_id, + persona_id=persona_id, + scenario_id=scenario_id, + provider_platform="vapi", + ) + ) + + assert ctx.product_section == LLMUsageProductSection.EVALUATORS + assert ctx.resource_id == evaluator_id + assert ctx.resource_type == "evaluator" + assert ctx.extra["agent_id"] == str(agent_id) + assert ctx.extra["evaluator_id"] == str(evaluator_id) + assert ctx.extra["persona_id"] == str(persona_id) + assert ctx.extra["scenario_id"] == str(scenario_id) + assert ctx.extra["synthetic_testing"] == "pre_prod" + assert ctx.extra["provider_platform"] == "vapi" + + +def test_usage_context_for_metric_studio_run(): + run_id = uuid4() + org_id = uuid4() + workspace_id = uuid4() + result_row_id = uuid4() + + run = SimpleNamespace( + id=run_id, + organization_id=org_id, + workspace_id=workspace_id, + ) + + ctx = usage_context_for_metric_studio_run( + run, + source_kind="evaluator_result", + source_ref=str(uuid4()), + result_row_id=result_row_id, + ) + + assert ctx.product_section == LLMUsageProductSection.METRICS + assert ctx.resource_id == run_id + assert ctx.resource_type == "metric_studio_run" + assert ctx.extra["metric_studio_run_id"] == str(run_id) + assert ctx.extra["metric_studio_result_id"] == str(result_row_id) + assert ctx.extra["source_kind"] == "evaluator_result" + assert ctx.extra["synthetic_testing"] == "pre_prod" + + +def test_usage_context_for_persona_generation(): + org_id = uuid4() + workspace_id = uuid4() + agent_id = uuid4() + agent = SimpleNamespace( + id=agent_id, + organization_id=org_id, + workspace_id=workspace_id, + ) + + ctx = usage_context_for_persona_generation(agent, workspace_id=workspace_id) + + assert ctx.product_section == LLMUsageProductSection.PERSONAS + assert ctx.resource_id == agent_id + assert ctx.extra["agent_id"] == str(agent_id) + assert ctx.extra["synthetic_testing"] == "pre_prod" + + +def test_extract_external_agent_usage_retell(): + call_data = { + "llm_token_usage": {"values": [1200, 800, 500], "average": 833, "num_requests": 3}, + "call_cost": {"total_duration_seconds": 95}, + "model": "gpt-4.1", + } + extracted = extract_external_agent_usage(call_data, platform="retell") + assert extracted is not None + assert extracted.model == "gpt-4.1" + assert extracted.llm.prompt_tokens == 1750 + assert extracted.llm.completion_tokens == 750 + assert extracted.stt_audio_seconds == 95 + + +def test_extract_external_agent_usage_retell_explicit_split(): + call_data = { + "llm_token_usage": { + "values": [900], + "prompt_tokens": 600, + "completion_tokens": 300, + }, + "call_cost": {"total_duration_seconds": 30}, + } + extracted = extract_external_agent_usage(call_data, platform="retell") + assert extracted is not None + assert extracted.llm.prompt_tokens == 600 + assert extracted.llm.completion_tokens == 300 + + +def test_usage_context_for_evaluator_result_playground_section(): + ctx = usage_context_for_evaluator_result( + SimpleNamespace( + id=uuid4(), + result_id="res-playground", + organization_id=uuid4(), + workspace_id=uuid4(), + agent_id=uuid4(), + evaluator_id=None, + persona_id=None, + scenario_id=None, + provider_platform="vapi", + ) + ) + assert ctx.product_section == LLMUsageProductSection.PLAYGROUND + + +def test_record_external_agent_usage_returns_false_when_storage_fails(fake_redis, monkeypatch): + from app.services.usage import llm_usage as usage_mod + from app.services.usage.context import usage_context_for_playground_voice_call + + org_id = uuid4() + agent_id = uuid4() + usage_ctx = usage_context_for_playground_voice_call( + organization_id=org_id, + workspace_id=uuid4(), + agent_id=agent_id, + provider_platform="vapi", + call_short_id="abc123", + ) + result = SimpleNamespace( + organization_id=org_id, + provider_platform="vapi", + result_id="abc123", + call_data={ + "costBreakdown": { + "llmPromptTokens": 40, + "llmCompletionTokens": 12, + }, + "durationSeconds": 20, + }, + ) + + monkeypatch.setattr(usage_mod, "_incr_pending", lambda *_args, **_kwargs: False) + + assert record_external_agent_usage(result, usage_ctx=usage_ctx) is False + + +def test_record_playground_provider_usage_from_call_data_raises_when_storage_fails( + fake_redis, monkeypatch +): + from app.services.usage import llm_usage as usage_mod + from app.services.usage.external_agent_usage import ( + ExternalUsageRecordingError, + record_playground_provider_usage_from_call_data, + ) + + monkeypatch.setattr(usage_mod, "_incr_pending", lambda *_args, **_kwargs: False) + + with pytest.raises(ExternalUsageRecordingError): + record_playground_provider_usage_from_call_data( + organization_id=uuid4(), + workspace_id=uuid4(), + agent_id=uuid4(), + provider_platform="vapi", + call_short_id="abc123", + call_data={ + "costBreakdown": { + "llmPromptTokens": 40, + "llmCompletionTokens": 12, + }, + "durationSeconds": 20, + }, + ) + + +def test_record_playground_provider_usage_from_call_data(fake_redis): + from app.services.usage import llm_usage as usage_mod + from app.services.usage.external_agent_usage import ( + record_playground_provider_usage_from_call_data, + ) + + org_id = uuid4() + agent_id = uuid4() + updated = record_playground_provider_usage_from_call_data( + organization_id=org_id, + workspace_id=uuid4(), + agent_id=agent_id, + provider_platform="vapi", + call_short_id="abc123", + call_data={ + "costBreakdown": { + "llmPromptTokens": 40, + "llmCompletionTokens": 12, + }, + "durationSeconds": 20, + }, + ) + assert updated["external_usage_recorded"] is True + fields = fake_redis.hgetall(usage_mod._pending_hash_key(org_id)) + assert any("playground" in k for k in fields) + + +def test_extract_external_agent_usage_smallest_duration_only(): + extracted = extract_external_agent_usage( + {"duration_seconds": 42}, + platform="smallest", + ) + assert extracted is not None + assert extracted.llm is None + assert extracted.stt_audio_seconds == 42 + + +def test_extract_external_agent_usage_elevenlabs_duration_only(): + extracted = extract_external_agent_usage( + {"metadata": {"call_duration_secs": 33}}, + platform="elevenlabs", + ) + assert extracted is not None + assert extracted.stt_audio_seconds == 33 diff --git a/tests/test_services/test_usage/test_test_agent_simulation_usage.py b/tests/test_services/test_usage/test_test_agent_simulation_usage.py new file mode 100644 index 00000000..e2fa94ab --- /dev/null +++ b/tests/test_services/test_usage/test_test_agent_simulation_usage.py @@ -0,0 +1,318 @@ +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from app.services.usage import context as usage_context_mod +from app.services.usage import llm_usage as usage_mod +from app.services.usage.context import ( + LLMUsageProductSection, + usage_context_for_test_agent_simulation, +) +from app.services.usage.external_agent_usage import ( + extract_external_agent_usage, + record_external_agent_usage, +) +from app.services.webrtc_bridge.test_agent_processor import TestAgentConfig, TestAgentProcessor +from app.workers.tasks import process_evaluator_result as per_mod + + +@pytest.fixture +def fake_redis(monkeypatch): + class _FakeRedis: + def __init__(self): + self.hashes = {} + self.sets = {} + self.kv = {} + + def hincrby(self, key, field, amount): + self.hashes.setdefault(key, {}) + self.hashes[key][field] = int(self.hashes[key].get(field, 0)) + int(amount) + + def hgetall(self, key): + return dict(self.hashes.get(key, {})) + + def sadd(self, key, member): + self.sets.setdefault(key, set()).add(member) + + def smembers(self, key): + return set(self.sets.get(key, set())) + + def expire(self, key, ttl): + return True + + def pipeline(self): + client = self + + class _Pipe: + def hincrby(self, key, field, amount): + client.hincrby(key, field, amount) + return self + + def sadd(self, key, member): + client.sadd(key, member) + return self + + def expire(self, key, ttl): + return self + + def execute(self): + return [] + + return _Pipe() + + client = _FakeRedis() + usage_mod._redis = client + import app.services.usage.read_cache as read_cache_mod + + read_cache_mod._redis = client + + def _forbid_real_redis(*_args, **_kwargs): + raise AssertionError( + "usage tests must not open real Redis; fake_redis fixture failed to isolate" + ) + + monkeypatch.setattr(usage_mod.redis, "from_url", _forbid_real_redis) + monkeypatch.setattr(read_cache_mod.redis, "from_url", _forbid_real_redis) + yield client + usage_mod._redis = None + read_cache_mod._redis = None + + +def test_usage_context_for_test_agent_simulation(): + org_id = uuid4() + agent_id = uuid4() + evaluator_id = uuid4() + persona_id = uuid4() + scenario_id = uuid4() + result_id = uuid4() + conversation_id = uuid4() + workspace_id = uuid4() + + ctx = usage_context_for_test_agent_simulation( + organization_id=org_id, + workspace_id=workspace_id, + agent_id=agent_id, + evaluator_id=evaluator_id, + persona_id=persona_id, + scenario_id=scenario_id, + evaluator_result_id=result_id, + conversation_id=conversation_id, + provider_platform="vapi", + ) + + assert ctx.product_section == LLMUsageProductSection.TEST_AGENT + assert ctx.resource_id == agent_id + assert ctx.extra["agent_id"] == str(agent_id) + assert ctx.extra["evaluator_id"] == str(evaluator_id) + assert ctx.extra["persona_id"] == str(persona_id) + assert ctx.extra["scenario_id"] == str(scenario_id) + assert ctx.extra["evaluator_result_id"] == str(result_id) + assert ctx.extra["conversation_id"] == str(conversation_id) + assert ctx.extra["simulation"] == "llm_to_llm" + + +def test_extract_external_agent_usage_vapi(): + call_data = { + "costBreakdown": { + "llmPromptTokens": 120, + "llmCompletionTokens": 45, + "llmCachedPromptTokens": 10, + "ttsCharacters": 300, + }, + "durationSeconds": 88, + "model": "gpt-4o", + } + extracted = extract_external_agent_usage(call_data, platform="vapi") + assert extracted is not None + assert extracted.model == "gpt-4o" + assert extracted.llm.prompt_tokens == 120 + assert extracted.llm.completion_tokens == 45 + assert extracted.llm.cache_read_tokens == 10 + assert extracted.tts_characters == 300 + assert extracted.stt_audio_seconds == 88 + + +def test_record_external_agent_usage_vapi(fake_redis): + org_id = uuid4() + agent_id = uuid4() + usage_ctx = usage_context_mod.usage_context_for_evaluator_result( + SimpleNamespace( + organization_id=org_id, + workspace_id=uuid4(), + agent_id=agent_id, + evaluator_id=uuid4(), + id=uuid4(), + result_id="res-1", + ) + ) + result = SimpleNamespace( + organization_id=org_id, + provider_platform="vapi", + result_id="res-1", + call_data={ + "costBreakdown": { + "llmPromptTokens": 50, + "llmCompletionTokens": 20, + } + }, + ) + + record_external_agent_usage(result, usage_ctx=usage_ctx) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + assert any("evaluators" in k for k in fields) + prompt = sum(int(v) for k, v in fields.items() if k.endswith("|prompt_tokens")) + completion = sum(int(v) for k, v in fields.items() if k.endswith("|completion_tokens")) + assert prompt == 50 + assert completion == 20 + + +def test_test_agent_processor_records_with_simulation_context(fake_redis): + org_id = uuid4() + agent_id = uuid4() + config = TestAgentConfig( + organization_id=org_id, + workspace_id=uuid4(), + agent_id=agent_id, + evaluator_id=uuid4(), + persona_id=uuid4(), + scenario_id=uuid4(), + llm_model="gpt-4o-mini", + ) + processor = TestAgentProcessor(config) + + class _Usage: + def __init__(self): + self.prompt_tokens = 11 + self.completion_tokens = 4 + + class _Response: + usage = _Usage() + + processor._record_llm_usage(response=_Response()) + + pending_key = usage_mod._pending_hash_key(org_id) + fields = fake_redis.hgetall(pending_key) + assert any("test_agent" in k for k in fields) + prompt = sum(int(v) for k, v in fields.items() if k.endswith("|prompt_tokens")) + assert prompt == 11 + + +def test_record_external_agent_llm_usage_skips_internal_platform(monkeypatch): + called = {"value": False} + + def _fake_record(*_args, **_kwargs): + called["value"] = True + + monkeypatch.setattr( + "app.services.usage.external_agent_usage.record_external_agent_usage", + _fake_record, + ) + + result = SimpleNamespace(agent_id=uuid4(), provider_platform="internal") + usage_ctx = SimpleNamespace() + per_mod._record_external_agent_llm_usage(result, usage_ctx=usage_ctx) + assert called["value"] is False + + +def test_process_audio_chunk_with_context_uses_test_agent_section(monkeypatch): + org_id = uuid4() + agent_id = uuid4() + captured_sections = [] + + def _fake_generate(**_kwargs): + ctx = usage_context_mod.get_usage_context() + if ctx: + captured_sections.append(ctx.product_section) + return {"text": "hi there", "processing_time": 0.2} + + from app.models.database import ( + TestAgentConversation, + TestAgentConversationStatus, + ModelProvider, + ) + from app.services.testing.test_agent_service import TestAgentService + from app.services.usage.context import llm_usage_context + + conversation = TestAgentConversation( + id=uuid4(), + organization_id=org_id, + workspace_id=uuid4(), + agent_id=agent_id, + persona_id=uuid4(), + scenario_id=uuid4(), + voice_bundle_id=uuid4(), + status=TestAgentConversationStatus.ACTIVE, + live_transcription=[], + ) + voice_bundle = SimpleNamespace( + stt_provider=ModelProvider.OPENAI, + stt_model="whisper-1", + llm_provider=ModelProvider.OPENAI, + llm_model="gpt-4o-mini", + llm_config=None, + llm_temperature=0.7, + llm_max_tokens=100, + tts_provider=ModelProvider.OPENAI, + tts_model="gpt-4o-mini-tts", + tts_config=None, + ) + + import importlib + + tas_module = importlib.import_module("app.services.testing.test_agent_service") + service = TestAgentService() + monkeypatch.setattr( + tas_module, + "s3_service", + SimpleNamespace(upload_file=lambda **_kwargs: "audio/key.wav"), + ) + monkeypatch.setattr( + tas_module.transcription_service, + "transcribe", + lambda **_kwargs: {"transcript": "hello", "processing_time": 0.1}, + ) + monkeypatch.setattr( + tas_module.llm_service, + "generate_response", + _fake_generate, + ) + monkeypatch.setattr( + tas_module.tts_service, + "synthesize", + lambda **_kwargs: b"mp3", + ) + monkeypatch.setattr( + service, + "_build_system_prompt", + lambda *_args, **_kwargs: "system", + ) + monkeypatch.setattr( + "app.services.voice_agent.resolve_tts_voice.resolve_effective_tts_voice_id", + lambda **_kwargs: "voice", + ) + + usage_ctx = usage_context_for_test_agent_simulation( + organization_id=org_id, + workspace_id=conversation.workspace_id, + agent_id=agent_id, + conversation_id=conversation.id, + ) + db = SimpleNamespace(commit=lambda: None) + + with llm_usage_context(usage_ctx): + service._process_audio_chunk_with_context( + conversation=conversation, + wav_audio_bytes=b"wav", + voice_bundle=voice_bundle, + agent=SimpleNamespace(id=agent_id, language=SimpleNamespace(value="en")), + persona=SimpleNamespace(id=conversation.persona_id, tts_voice_id=None), + scenario=SimpleNamespace(id=conversation.scenario_id), + organization_id=org_id, + db=db, + chunk_timestamp=0.0, + ) + + assert captured_sections == [LLMUsageProductSection.TEST_AGENT] diff --git a/tests/test_workers/test_celery_task_workflows.py b/tests/test_workers/test_celery_task_workflows.py index a74aca2c..5e9aecd0 100644 --- a/tests/test_workers/test_celery_task_workflows.py +++ b/tests/test_workers/test_celery_task_workflows.py @@ -11,12 +11,15 @@ from app.models.database import ( Agent, + Evaluator, EvaluatorResult, + Integration, Metric, Organization, PromptOptimizationRun, TTSComparison, TTSSample, + VoiceBundle, Workspace, ) @@ -48,6 +51,25 @@ class RetryCalled(Exception): """Raised by task.retry in tests to assert retry paths.""" +def _load_run_evaluator_module(): + """Load the real task module even when conftest/API tests stub workers.tasks.""" + module_name = "app.workers.tasks.run_evaluator" + module_path = ( + Path(__file__).resolve().parents[2] + / "app" + / "workers" + / "tasks" + / "run_evaluator.py" + ) + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load task module from {module_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + def _load_run_prompt_optimization_module(): """Load the real task module even when conftest/API tests stub workers.tasks.""" module_name = "app.workers.tasks.run_prompt_optimization" @@ -74,15 +96,26 @@ def _load_run_prompt_optimization_module(): return module +class _FakeCeleryTaskSelf: + request = types.SimpleNamespace(id="test-celery-task-id") + + def retry(self, **kwargs): + raise kwargs.get("exc") + + def _invoke_bound_task(task, *args): """Call a bind=True Celery task under real or conftest-fake decorators.""" run = getattr(task, "run", task) - try: - return run(*args) - except TypeError as exc: - if "missing 1 required positional argument" in str(exc): - return run(None, *args) - raise + last_exc = None + for call_args in (args, (_FakeCeleryTaskSelf(), *args)): + try: + return run(*call_args) + except TypeError as exc: + last_exc = exc + continue + if last_exc is not None: + raise last_exc + raise RuntimeError("bind=True task invocation failed") def _seed_org(db_session): @@ -451,22 +484,13 @@ def test_process_evaluator_result_emits_playground_billing_with_metric_count( ) db_session.commit() - billing = {"evaluated": [], "completed": []} - - def _capture_evaluated(_org_id, evaluation_attempt_id, **kw): - billing["evaluated"].append( - {"evaluation_attempt_id": evaluation_attempt_id, **kw} - ) + billing = {"completed": []} def _capture_completed(_org_id, evaluation_attempt_id, **kw): billing["completed"].append( {"evaluation_attempt_id": evaluation_attempt_id, **kw} ) - monkeypatch.setattr( - "app.services.billing.flexprice_service.record_playground_call_evaluated", - _capture_evaluated, - ) monkeypatch.setattr( "app.services.billing.flexprice_service.record_playground_evaluation_completed", _capture_completed, @@ -491,23 +515,12 @@ def _capture_completed(_org_id, evaluation_attempt_id, **kw): result = task_module.process_evaluator_result_task.run(str(eval_result.id)) assert result["status"] == "completed" - assert len(billing["evaluated"]) == 1 - assert billing["evaluated"][0]["metric_count"] == 3 - assert billing["evaluated"][0]["evaluator_result_id"] == eval_result.id - assert billing["evaluated"][0]["call_short_id"] == "123456" - assert str(billing["evaluated"][0]["evaluation_attempt_id"]).startswith( - f"{eval_result.id}:" - ) assert len(billing["completed"]) == 1 assert billing["completed"][0]["metric_count"] == 3 assert billing["completed"][0]["evaluator_result_id"] == eval_result.id assert str(billing["completed"][0]["evaluation_attempt_id"]).startswith( f"{eval_result.id}:" ) - assert ( - billing["evaluated"][0]["evaluation_attempt_id"] - == billing["completed"][0]["evaluation_attempt_id"] - ) def test_process_evaluator_result_categorizes_audio_metrics_as_skipped_without_audio(db_session): @@ -525,8 +538,104 @@ def test_process_evaluator_result_categorizes_audio_metrics_as_skipped_without_a assert skipped_scores[str(audio_metric.id)]["skipped"] == "audio_required" +def test_run_evaluator_bridge_runs_without_existing_event_loop(db_session, monkeypatch): + task_module = _load_run_evaluator_module() + + org = _seed_org(db_session) + workspace_id = _default_workspace_id(db_session, org.id) + voice_bundle = VoiceBundle( + id=uuid4(), + organization_id=org.id, + name="Bridge Bundle", + bundle_type="stt_llm_tts", + stt_provider="openai", + stt_model="whisper-1", + llm_provider="openai", + llm_model="gpt-4o-mini", + tts_provider="openai", + tts_model="tts-1", + tts_voice="alloy", + ) + db_session.add(voice_bundle) + db_session.flush() + + integration = Integration( + id=uuid4(), + organization_id=org.id, + platform="retell", + name="Bridge Integration", + api_key="encrypted-test-key", + is_active=True, + is_default=True, + ) + db_session.add(integration) + db_session.flush() + + agent = Agent( + id=uuid4(), + organization_id=org.id, + workspace_id=workspace_id, + name="Bridge Agent", + language="en", + description="Bridge test agent", + call_type="outbound", + call_medium="phone_call", + voice_bundle_id=voice_bundle.id, + voice_ai_integration_id=integration.id, + voice_ai_agent_id="provider-agent-1", + ) + evaluator = Evaluator( + id=uuid4(), + evaluator_id="611111", + organization_id=org.id, + workspace_id=workspace_id, + name="Bridge Evaluator", + agent_id=agent.id, + persona_id=None, + scenario_id=None, + ) + eval_result = EvaluatorResult( + id=uuid4(), + result_id="611112", + organization_id=org.id, + workspace_id=workspace_id, + evaluator_id=evaluator.id, + agent_id=agent.id, + status="queued", + ) + db_session.add_all([integration, agent, evaluator, eval_result]) + db_session.commit() + + async def fake_bridge(**_kwargs): + return {"status": "bridged"} + + fake_bridge_module = types.ModuleType("app.services.testing.test_agent_bridge_service") + fake_bridge_module.test_agent_bridge_service = types.SimpleNamespace( + bridge_test_agent_to_voice_agent=fake_bridge + ) + monkeypatch.setitem( + sys.modules, + "app.services.testing.test_agent_bridge_service", + fake_bridge_module, + ) + monkeypatch.setattr(task_module, "SessionLocal", lambda: _worker_db(db_session)) + monkeypatch.setattr( + "app.services.billing.flexprice_service.record_evaluator_run_completed", + lambda *_args, **_kwargs: None, + ) + + result = _invoke_bound_task( + task_module.run_evaluator_task, + str(evaluator.id), + str(eval_result.id), + ) + + assert result["status"] == "initiated" + assert result["bridge_result"] == {"status": "bridged"} + + def test_run_evaluator_returns_error_when_evaluator_missing(db_session, monkeypatch): - from app.workers.tasks import run_evaluator as task_module + task_module = _load_run_evaluator_module() org = _seed_org(db_session) eval_result = EvaluatorResult( @@ -544,7 +653,11 @@ def test_run_evaluator_returns_error_when_evaluator_missing(db_session, monkeypa monkeypatch.setitem(sys.modules, "app.services.testing.test_agent_bridge_service", fake_bridge_module) monkeypatch.setattr(task_module, "SessionLocal", lambda: _worker_db(db_session)) - result = task_module.run_evaluator_task.run(str(uuid4()), str(eval_result.id)) + result = _invoke_bound_task( + task_module.run_evaluator_task, + str(uuid4()), + str(eval_result.id), + ) assert result == {"error": "Evaluator not found"} diff --git a/tests/test_workers/test_evaluate_call_import_row.py b/tests/test_workers/test_evaluate_call_import_row.py index e79c9369..1d3a6a6c 100644 --- a/tests/test_workers/test_evaluate_call_import_row.py +++ b/tests/test_workers/test_evaluate_call_import_row.py @@ -155,6 +155,29 @@ def _run(*args, **kwargs): return task +class _EvalRowFakeStorage: + """Minimal blob storage stub for eval-row worker tests.""" + + prefix = "test-prefix/" + + def is_enabled(self): + return True + + def download_file_by_key(self, _key): + return b"fake-audio-bytes" + + +def _patch_eval_row_storage(monkeypatch): + import importlib + + fake = _EvalRowFakeStorage() + blob_module = importlib.import_module("app.services.storage.blob_storage_service") + s3_module = importlib.import_module("app.services.storage.s3_service") + monkeypatch.setattr(blob_module, "blob_storage_service", fake) + monkeypatch.setattr(s3_module, "s3_service", fake, raising=False) + return fake + + def _patch_row_location(monkeypatch, db_session): """Route shard-aware row lookup to the pytest session.""" @@ -199,6 +222,7 @@ def _patch_dependencies(monkeypatch, db_session, *, evaluate_with_llm=None): """Stub SessionLocal and the LLM helper inside the eval task module.""" import importlib + _patch_eval_row_storage(monkeypatch) session_factory = lambda: _NonClosingSession(db_session) monkeypatch.setattr("app.database.SessionLocal", session_factory) _patch_row_location(monkeypatch, db_session) @@ -239,6 +263,7 @@ def _default_eval(*_args, **_kwargs): def _patch_audio_task(monkeypatch, db_session): import importlib + _patch_eval_row_storage(monkeypatch) _patch_row_location(monkeypatch, db_session) audio_module = _load_real_task_module( "app.workers.tasks.evaluate_call_import_row_audio", diff --git a/tests/test_workers/test_process_call_import_row.py b/tests/test_workers/test_process_call_import_row.py index 9c598414..c13c5dec 100644 --- a/tests/test_workers/test_process_call_import_row.py +++ b/tests/test_workers/test_process_call_import_row.py @@ -174,6 +174,9 @@ def upload_file_by_key(self, file_content, key, content_type="audio/mpeg"): self.uploads.append({"key": key, "size": len(file_content), "content_type": content_type}) return key + def download_file_by_key(self, key): + return b"fake-audio-bytes" + def get_organization_root_prefix(self, organization_id: str) -> str: return f"{self.prefix}organizations/{organization_id}/"