feat(evaluation): preserve criteria after agent failures

docs/REPORT_SCHEMA.md

@@ -28,7 +28,8 @@ read). Times are ISO-8601.
 | `<variant>/variant.json` / `.md` | `VariantAggregate` | Per variant |
 
 `<NN>` is the zero-padded replicate index. Judge transcripts spill to sibling files
-(e.g. `judge-0.yaml`) referenced by a `transcript_path`.
+(`judge-N.yaml`, or `post-failure-judge-N.yaml` for diagnostic records) referenced
+by a `transcript_path`.
 
 ---
 
@@ -126,6 +127,7 @@ The authoritative per-replicate record.
 | `max_turns_exhausted` | `bool` | Ran out of turns. |
 | `iteration_count` | `int` | Number of turns. |
 | `success_criteria_results` | `list[CriterionResult]` | Per-criterion results — see [below](#criterionresult). |
+| `post_failure_criteria_results` | `list[CriterionResult]` | Diagnostic artifact evidence collected after a terminal agent failure. It does not affect `final_status`, `weighted_score`, gating, or suite aggregation. |
 
 **Transcript:** `iterations: list[TurnRecord]` (accepts legacy alias `turns`) — see
 [TurnRecord](#turnrecord).
@@ -166,17 +168,31 @@ files without `result_kind` are inferred from `criterion_type`. Base fields
 (`result_kind="basic"`):
 
 `criterion_type`, `description`, `score` (0.0–1.0), `details`, `error`,
+`evaluation_status` (`evaluated` by default; `not_evaluated` means the check did
+not run and is distinct from an evaluated 0.0),
 `pass_threshold` (default 0.9), `gating` (default `true`; `false` = informational /
 weight-0, excluded from the score and the pass/fail gate). The base allows extra
 fields so subclass keys round-trip.
 
 - **`classification`** adds `observed_label`, `expected_label` (sentinels like
   `(none)` / `(other)` allowed). Emitted by `classification_match`, `skill_triggered`.
 - **`judge`** adds `findings`, `token_usage` (kept distinct from the agent total), and
-  `transcript_path` (a sibling `judge-N.yaml`). The full `transcript` is **stripped
+  `transcript_path` (a sibling `judge-N.yaml`, or `post-failure-judge-N.yaml` for
+  diagnostic records). The full `transcript` is **stripped
   from `task.json`** — read it from the referenced file. Emitted by `llm_judge`,
   `agent_judge`.
 
+### Post-failure criterion evidence
+
+When an agent crashes or its turn times out, coder-eval runs only deterministic,
+read-only artifact criteria while the sandbox is still live: `file_exists`,
+`file_contains`, `file_matches_regex`, `file_check`, `json_check`,
+`reference_comparison`, and `classification_match`. Judges, trajectory checks,
+`run_command`, and `uipath_eval` are recorded with
+`evaluation_status="not_evaluated"`; they are not invoked on this recovery path.
+The diagnostic list is additive evidence. An `ERROR` run remains `ERROR`, and its
+canonical score remains 0.0.
+
 ### TurnRecord
 
 `iteration`, `user_input`, `agent_output`, `commands` (`list[CommandTelemetry]`),

docs/TASK_DEFINITION_GUIDE.md

@@ -239,7 +239,7 @@ run_limits:
   # Structural caps
   max_turns: 20                       # hard cap on agent inner-loop turns per iteration
   expected_turns: 8                   # SOFT efficiency budget (visible turns) — never aborts
-  task_timeout: 600                   # wall-clock cap across all iterations, seconds
+  task_timeout: 300                   # wall-clock cap for the full run envelope, seconds
   turn_timeout: 300                   # per-communicate() timeout, seconds
 
   # Budget caps
@@ -254,8 +254,8 @@ run_limits:
 |-------|---------|------------|-------------|
 | `max_turns` | *unset* | `> 0` | Hard cap on agent inner-loop turns per iteration. Unset uses the SDK default. |
 | `expected_turns` | *unset* | `>= 1` | **Soft** target for cumulative visible turns. Exceeding it warns and badges the report; it never aborts. See [`expected_turns`](#expected_turns-soft-efficiency-budget). |
-| `task_timeout` | *unset* | `>= 30` | Max seconds for the whole evaluation loop (all iterations). |
-| `turn_timeout` | *unset* | `>= 10` | Max seconds for a single agent `communicate()` call. |
+| `task_timeout` | *unset* | `>= 30` | Max seconds for the full run envelope, including agent work, grading, and post-run work. |
+| `turn_timeout` | *unset* | `>= 10` | Max seconds for the agent's single `communicate()` iteration. |
 | `max_input_tokens` | *unset* | `>= 1` | Max cumulative input (prompt) tokens. |
 | `max_output_tokens` | *unset* | `>= 1` | Max cumulative output (completion) tokens. |
 | `max_total_tokens` | *unset* | `>= 1` | Max cumulative input + output tokens. Distinct from [`simulation.max_total_tokens`](#simulation-multi-turn-user-dialog) — see the note below. |
@@ -265,6 +265,11 @@ run_limits:
 | `stop_early` | *unset* | `false` or unset | Run-level early-stop **kill switch** — there is no master arm. Unset: the criteria's own `stop_early:` blocks decide. `false`: force-disarm every block for this run. `true` (the removed master arm) is rejected at resolution. See [`stop_early`](#stop_early-opt-in-early-stop). |
 | `stop_early_gate_threshold` | `1.0` | `[0.0, 1.0]` (but `> 0.0` is enforced at resolution on an armed task) | Minimum weighted score over the armed subset required for an **early-stopped** run to gate as a pass. See [`stop_early`](#stop_early-opt-in-early-stop). |
 
+If resolved `task_timeout` is larger than `turn_timeout`, `plan` and runtime emit
+a non-blocking warning: a larger `task_timeout` cannot extend the agent's single
+iteration; the agent budget is `turn_timeout`. The values are not rejected or
+changed because `task_timeout` still governs grading and other run-envelope work.
+
 The authoritative source is `src/coder_eval/models/limits.py`. A lint rule (CE030) fails the build if
 a field defined there goes undocumented in this guide, so the table can't quietly fall behind the
 model.

evalboard/app/runs/[id]/[...task]/__tests__/criteria.test.tsx

@@ -0,0 +1,49 @@
+import { render, screen } from "@testing-library/react";
+import { describe, expect, test } from "vitest";
+import type { CriterionResult } from "@/lib/runs";
+import { CriteriaSection } from "../_sections";
+
+function criterion(
+    overrides: Partial<CriterionResult> = {},
+): CriterionResult {
+    return {
+        criterionType: "file_exists",
+        description: "artifact exists",
+        score: 1,
+        details: null,
+        error: null,
+        evaluationStatus: "evaluated",
+        passThreshold: 0.9,
+        gating: true,
+        ...overrides,
+    };
+}
+
+describe("CriteriaSection", () => {
+    test("renders unavailable post-failure checks without calling them failures", () => {
+        render(
+            <CriteriaSection
+                title="Post-failure artifact evidence"
+                diagnostic
+                criteria={[
+                    criterion(),
+                    criterion({
+                        criterionType: "run_command",
+                        description: "validator runs",
+                        score: 0,
+                        evaluationStatus: "not_evaluated",
+                    }),
+                ]}
+            />,
+        );
+
+        expect(
+            screen.getByText("Post-failure artifact evidence (2)"),
+        ).toBeInTheDocument();
+        expect(screen.getByText("NOT EVALUATED")).toBeInTheDocument();
+        expect(screen.getByText("no score")).toBeInTheDocument();
+        expect(
+            screen.getByText(/does not affect status, score, or pass\/fail gating/),
+        ).toBeInTheDocument();
+    });
+});

evalboard/app/runs/[id]/[...task]/_sections.tsx

@@ -105,39 +105,62 @@ export function FlowDebugSection({ flowDebug }: { flowDebug: FlowDebugResult })
     );
 }
 
-export function CriteriaSection({ criteria }: { criteria: CriterionResult[] }) {
+export function CriteriaSection({
+    criteria,
+    title = "Success criteria",
+    diagnostic = false,
+}: {
+    criteria: CriterionResult[];
+    title?: string;
+    diagnostic?: boolean;
+}) {
     return (
         <section className="space-y-2">
             <h2 className="text-sm font-semibold text-gray-900">
-                Success criteria ({criteria.length})
-                {criteria.some((c) => !c.gating) && (
+                {title} ({criteria.length})
+                {!diagnostic && criteria.some((c) => !c.gating) && (
                     <span className="ml-2 font-normal text-gray-500">
                         {criteria.filter((c) => !c.gating).length}{" "}
                         informational
                     </span>
                 )}
             </h2>
+            {diagnostic && (
+                <p className="text-xs text-gray-500">
+                    Diagnostic evidence collected after the agent failed. It
+                    does not affect status, score, or pass/fail gating.
+                </p>
+            )}
             <div className="space-y-2">
                 {criteria.map((c, i) => {
                     // Compare against the criterion's own threshold, not === 1:
                     // fractional criteria pass below 1.0 (default threshold 0.9).
                     const passed = (c.score ?? 0) >= c.passThreshold;
+                    const evaluated = c.evaluationStatus === "evaluated";
                     return (
                         <Expandable
                             key={i}
                             header={
                                 <div className="flex items-center gap-3">
-                                    <ResultPill
-                                        passed={passed}
-                                        gating={c.gating}
-                                    />
+                                    {evaluated ? (
+                                        <ResultPill
+                                            passed={passed}
+                                            gating={c.gating}
+                                        />
+                                    ) : (
+                                        <span className="inline-flex rounded-full bg-gray-100 px-2 py-0.5 text-xs font-medium text-gray-600">
+                                            NOT EVALUATED
+                                        </span>
+                                    )}
                                     <span className="text-sm text-gray-900">
                                         {c.description ??
                                             c.criterionType ??
                                             `criterion ${i + 1}`}
                                     </span>
                                     <span className="ml-auto text-xs text-gray-500 tabular-nums">
-                                        score {c.score ?? "—"}
+                                        {evaluated
+                                            ? `score ${c.score ?? "—"}`
+                                            : "no score"}
                                     </span>
                                 </div>
                             }

evalboard/app/runs/[id]/[...task]/page.tsx

@@ -306,6 +306,13 @@ export default async function TaskPage({
 
             {flowDebug && <FlowDebugSection flowDebug={flowDebug} />}
             <CriteriaSection criteria={task.criteria} />
+            {task.postFailureCriteria.length > 0 && (
+                <CriteriaSection
+                    criteria={task.postFailureCriteria}
+                    title="Post-failure artifact evidence"
+                    diagnostic
+                />
+            )}
             {conversation.length > 0 && (
                 <ConversationSection turns={conversation} />
             )}

evalboard/lib/__tests__/providerCalls.test.ts

@@ -88,6 +88,14 @@ beforeEach(async () => {
         `${RUN}/default/${TASK}/00/task.json`,
         JSON.stringify({
             final_status: "success",
+            post_failure_criteria_results: [
+                {
+                    criterion_type: "file_exists",
+                    description: "artifact exists",
+                    score: 1,
+                    evaluation_status: "evaluated",
+                },
+            ],
             iterations: [
                 {
                     model_used: "deepseek/deepseek-v4-pro",
@@ -139,5 +147,9 @@ describe("readTaskDetail: providerCalls", () => {
         });
         expect(turn.calls[1].callId).toBe("call-2");
         expect(turn.calls[1].costUsd).toBe(0.0034);
+        expect(detail!.postFailureCriteria).toHaveLength(1);
+        expect(detail!.postFailureCriteria[0].evaluationStatus).toBe(
+            "evaluated",
+        );
     });
 });

evalboard/lib/__tests__/runs.test.ts

@@ -20,12 +20,46 @@ import {
     findMatureSourceRuns,
     isExcludedArtifact,
     type MessageEvent,
+    parseCriterionResults,
     sortArtifacts,
     toTaskRow,
     visibleTurnsFromRaw,
     walkArtifacts,
 } from "../runs";
 
+describe("parseCriterionResults", () => {
+    test("preserves evaluated versus not-evaluated evidence", () => {
+        const results = parseCriterionResults([
+            {
+                criterion_type: "file_exists",
+                description: "artifact exists",
+                score: 1,
+                evaluation_status: "evaluated",
+            },
+            {
+                criterion_type: "run_command",
+                description: "validator runs",
+                score: 0,
+                evaluation_status: "not_evaluated",
+            },
+        ]);
+
+        expect(results.map((result) => result.evaluationStatus)).toEqual([
+            "evaluated",
+            "not_evaluated",
+        ]);
+    });
+
+    test("legacy results default to evaluated", () => {
+        const [result] = parseCriterionResults([
+            { criterion_type: "file_exists", score: 0 },
+        ]);
+        expect(result.evaluationStatus).toBe("evaluated");
+        expect(result.passThreshold).toBe(0.9);
+        expect(result.gating).toBe(true);
+    });
+});
+
 describe("toTaskRow", () => {
     test("propagates total_turns and expected_turns", () => {
         const row = toTaskRow({

evalboard/lib/runs.ts

@@ -118,6 +118,7 @@ export interface CriterionResult {
     score: number | null;
     details: string | null;
     error: string | null;
+    evaluationStatus: "evaluated" | "not_evaluated";
     // Mirrors the Python CriterionResult fields. `gating: false` (weight: 0) means
     // the criterion is informational — measured, but excluded from the score and
     // the pass/fail gate, so it must not render as PASS/FAIL. Both default the way
@@ -126,6 +127,32 @@ export interface CriterionResult {
     gating: boolean;
 }
 
+interface RawCriterionResult {
+    criterion_type?: string;
+    description?: string;
+    score?: number;
+    details?: string;
+    error?: string | null;
+    evaluation_status?: "evaluated" | "not_evaluated";
+    pass_threshold?: number;
+    gating?: boolean;
+}
+
+export function parseCriterionResults(
+    criteria: RawCriterionResult[] | undefined,
+): CriterionResult[] {
+    return (criteria ?? []).map((criterion) => ({
+        criterionType: criterion.criterion_type ?? null,
+        description: criterion.description ?? null,
+        score: criterion.score ?? null,
+        details: criterion.details ?? null,
+        error: criterion.error ?? null,
+        evaluationStatus: criterion.evaluation_status ?? "evaluated",
+        passThreshold: criterion.pass_threshold ?? 0.9,
+        gating: criterion.gating ?? true,
+    }));
+}
+
 export interface ElementExecution {
     elementId: string;
     elementType: string | null;
@@ -267,6 +294,7 @@ export interface TaskDetail extends TaskResultSummary {
     errorMessage: string | null;
     taskDescription: string | null;
     criteria: CriterionResult[];
+    postFailureCriteria: CriterionResult[];
     artifacts: ArtifactRef[];
     flowDebug: FlowDebugResult | null;
     toolCalls: ToolCall[];
@@ -2003,30 +2031,16 @@ export async function readTaskDetail(
                 initial_prompt?: string;
             };
         };
-        success_criteria_results?: Array<{
-            criterion_type?: string;
-            description?: string;
-            score?: number;
-            details?: string;
-            error?: string | null;
-            pass_threshold?: number;
-            gating?: boolean;
-        }>;
+        success_criteria_results?: RawCriterionResult[];
+        post_failure_criteria_results?: RawCriterionResult[];
         iterations?: TurnEntry[];
         environment_info?: RawRunJson["environment_info"];
     }>(path.join(contentDir, "task.json"));
 
-    const criteria: CriterionResult[] = (
-        task?.success_criteria_results ?? []
-    ).map((c) => ({
-        criterionType: c.criterion_type ?? null,
-        description: c.description ?? null,
-        score: c.score ?? null,
-        details: c.details ?? null,
-        error: c.error ?? null,
-        passThreshold: c.pass_threshold ?? 0.9,
-        gating: c.gating ?? true,
-    }));
+    const criteria = parseCriterionResults(task?.success_criteria_results);
+    const postFailureCriteria = parseCriterionResults(
+        task?.post_failure_criteria_results,
+    );
 
     const artifactRoot = path.join(contentDir, "artifacts");
     // relPath is stored relative to the run root so the /api/file route can
@@ -2093,6 +2107,7 @@ export async function readTaskDetail(
         errorMessage: task?.error_message ?? null,
         taskDescription,
         criteria,
+        postFailureCriteria,
         artifacts,
         flowDebug,
         toolCalls,