feat: Simulation upgrades

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,169 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "efficientAI"
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                bash
+#   bsl                    clojure                cpp                    cpp_ccls               crystal
+#   csharp                 csharp_omnisharp       cue                    dart                   deno
+#   elixir                 elm                    erlang                 fortran                fsharp
+#   gdscript               gleam                  go                     groovy                 haskell
+#   haxe                   hlsl                   html                   java                   json
+#   julia                  kotlin                 latex                  lean4                  lua
+#   luau                   markdown               matlab                 msl                    nextflow
+#   nix                    ocaml                  pascal                 perl                   php
+#   php_phpactor           php_phpantom           powershell             python                 python_basedpyright
+#   python_jedi            python_pyrefly         python_ty              qml                    r
+#   rego                   ruby                   ruby_solargraph        rust                   scala
+#   scss                   solidity               svelte                 swift                  systemverilog
+#   terraform              toml                   typescript             typescript_vts         vue
+#   wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- python
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- "."
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

README.md

@@ -63,12 +63,12 @@ There are two ways to run the application:
    | `redis` | Redis (Celery broker + usage counters) |
    | `api` | HTTP API + frontend |
    | `media` | Live voice WebSocket media server |
-   | `worker` | Celery: `celery` (evaluator cron dispatch), `audio-metrics` queues |
+   | `worker` | Celery: `celery` (evaluator cron runs), `audio-metrics` queues |
    | `beat` | Celery Beat scheduler + `platform` queue worker (alerts, FX, OSS prune) — **single replica** |
    | `worker-imports` | Celery: `imports`, `diarization`, `eval-control`, `evaluations` |
-   | `worker-usage` | Celery: `usage` queue (flush Redis counters + cost recompute) |
+   | `worker-usage` | Celery: `usage` queue (flush Redis counters, cost recompute, evaluator cron dispatch) |
 
-   **Usage costs:** token/cost rollups stay stale without `beat`, `worker-usage`, and default `worker` (evaluator crons; or `eai start-all`).
+   **Usage costs:** token/cost rollups stay stale without `beat`, `worker-usage`, and default `worker` (evaluator cron runs; or `eai start-all`).
 
    **Using a specific version:**
    ```bash
@@ -416,7 +416,7 @@ eai usage recompute --config config.yml --sync
 | `USAGE_FLUSH_BEAT_SECONDS` | `120` | Celery Beat flush interval (~2 min lag vs Redis) |
 | `USAGE_FLUSH_LOCK_TTL_SECONDS` | `300` | Per-org flush lock TTL |
 | `USAGE_READ_CACHE_TTL_SECONDS` | `90` | Redis cache TTL for usage summary/breakdown/filters |
-| `CRON_DISPATCH_INTERVAL_SECONDS` | `30` | Evaluator cron dispatcher tick (default worker) |
+| `CRON_DISPATCH_INTERVAL_SECONDS` | `30` | Evaluator cron dispatch interval (Beat → worker-usage) |
 
 Usage UI reads Postgres only (summary/breakdown/filters); Redis counters flush on the Celery Beat schedule (~2 min eventual consistency). If Redis backlog grows, lower `USAGE_FLUSH_BEAT_SECONDS` or raise `USAGE_FLUSH_MAX_BATCHES_PER_RUN`.
 

app/api/v1/api.py

@@ -24,6 +24,7 @@
     evaluator_suites,
     metrics,
     evaluator_results,
+    evaluator_result_metric_clusters,
     chat,
     playground,
     settings,
@@ -76,6 +77,7 @@
 api_router.include_router(evaluator_suites.router)
 api_router.include_router(metrics.router)
 api_router.include_router(evaluator_results.router)
+api_router.include_router(evaluator_result_metric_clusters.router)
 api_router.include_router(chat.router)
 api_router.include_router(playground.router)
 api_router.include_router(settings.router)

app/api/v1/routes/agents.py

@@ -33,11 +33,57 @@
     GenerateTestSetupResponse,
     GeneratedScenarioDraftResponse,
     TestPromptSectionResponse,
+    TestAgentFirstMessageResponse,
+    TestAgentTemplateResponse,
+    TestAgentTemplateInput,
+)
+from app.services.testing.test_agent_template import (
+    TestAgentFirstMessage,
+    TestAgentTemplate,
+    assemble_test_agent_prompt,
+    normalize_first_message,
+    normalize_sections,
+    template_from_generation,
 )
 
 router = APIRouter(prefix="/agents", tags=["agents"])
 
 
+def _first_message_response(first_message: TestAgentFirstMessage) -> TestAgentFirstMessageResponse:
+    return TestAgentFirstMessageResponse(
+        production_mode=first_message.production_mode,
+        production_message=first_message.production_message,
+        caller_mode=first_message.caller_mode,
+        caller_message=first_message.caller_message,
+    )
+
+
+def _template_response(template: TestAgentTemplate) -> TestAgentTemplateResponse:
+    return TestAgentTemplateResponse(
+        sections=_test_prompt_section_responses(template.sections),
+        first_message=_first_message_response(template.first_message),
+    )
+
+
+def _template_input_to_storage(template_input: TestAgentTemplateInput) -> dict:
+    sections = normalize_sections([s.model_dump() for s in template_input.sections])
+    first_message = normalize_first_message(template_input.first_message.model_dump())
+    return template_from_generation(sections, first_message).to_dict()
+
+
+def _apply_test_agent_template_fields(
+    *,
+    description: Optional[str],
+    template_input: Optional[TestAgentTemplateInput],
+) -> tuple[Optional[str], Optional[dict]]:
+    """Return (description, test_agent_template_json) for persistence."""
+    if template_input is None:
+        return description, None
+    template_dict = _template_input_to_storage(template_input)
+    assembled = assemble_test_agent_prompt(normalize_sections(template_dict.get("sections")))
+    return assembled or description, template_dict
+
+
 def _validate_agent_phone_assignment(
     db: Session,
     *,
@@ -130,7 +176,7 @@ class GenerateAgentDescriptionRequest(BaseModel):
     "well-formatted agent description in markdown.\n\n"
     "Guidelines:\n"
     "- Use clear markdown structure: headings, bullet points, numbered lists\n"
-    "- Include sections for: Purpose, Behavior, Expected Interactions, Personality Traits, and Constraints\n"
+    "- Include sections for: Role and Goal, Talking Style, Questions to Ask, Information to Relay, and Constraints\n"
     "- Be specific about the agent's role, tone of voice, and how it should handle conversations\n"
     "- Include example scenarios or edge cases where helpful\n"
     "- Return ONLY the description in markdown, no preamble or explanation about what you did"
@@ -300,6 +346,8 @@ async def generate_test_prompt(
         return GenerateTestPromptResponse(
             sections=_test_prompt_section_responses(result.sections),
             test_agent_prompt=result.test_agent_prompt,
+            first_message=_first_message_response(result.first_message),
+            test_agent_template=_template_response(result.test_agent_template),
             provider=result.provider,
             model=result.model,
         )
@@ -433,6 +481,8 @@ async def generate_test_setup(
         return GenerateTestSetupResponse(
             sections=_test_prompt_section_responses(prompt_result.sections),
             test_agent_prompt=prompt_result.test_agent_prompt,
+            first_message=_first_message_response(prompt_result.first_message),
+            test_agent_template=_template_response(prompt_result.test_agent_template),
             scenarios=_scenario_draft_responses(scenario_result.scenarios),
             provider=prompt_result.provider,
             model=prompt_result.model,
@@ -575,6 +625,11 @@ async def create_agent(
 
     # Generate unique 6-digit agent_id
     agent_id = generate_unique_agent_id(db)
+
+    description, template_dict = _apply_test_agent_template_fields(
+        description=agent.description,
+        template_input=agent.test_agent_template,
+    )
 
     db_agent = Agent(
         agent_id=agent_id,
@@ -583,7 +638,8 @@ async def create_agent(
         name=agent.name,
         phone_number=agent.phone_number,
         language=agent.language,
-        description=agent.description,
+        description=description,
+        test_agent_template=template_dict,
         call_type=agent.call_type,
         call_medium=agent.call_medium,
         telephony_phone_number_id=agent.telephony_phone_number_id,
@@ -813,6 +869,15 @@ async def update_agent(
 
     update_data = agent_update.model_dump(exclude_unset=True, exclude_none=False)
 
+    if "test_agent_template" in update_data:
+        template_input = agent_update.test_agent_template
+        assembled_description, template_dict = _apply_test_agent_template_fields(
+            description=update_data.get("description", db_agent.description),
+            template_input=template_input,
+        )
+        update_data["description"] = assembled_description
+        update_data["test_agent_template"] = template_dict
+
     effective_call_medium = (
         agent_update.call_medium if agent_update.call_medium is not None else db_agent.call_medium
     )