Add authorized retained checkpoint analysis to Runs

.github/workflows/tests.yml

@@ -307,6 +307,20 @@ jobs:
         env:
           PYTHON: python
           BROWSER: /usr/bin/google-chrome
+      - name: Qualify retained query and authenticated Runs over one actual checkpoint fixture
+        run: |
+          mkdir -p build/qa/retained-analysis
+          python tests/retained_analysis_actual_fixture.py --directory build/retained-analysis-fixture > build/qa/retained-analysis/fixture.json
+          python -m unittest discover -s tests -p 'test_retained_analysis*.py' -v > build/qa/retained-analysis/query-http-tests.log 2>&1
+          node tests/test_workspace_model_controller.cjs
+          node tests/browser_retained_analysis.cjs > build/qa/retained-analysis/renderer.json
+          node tests/browser_retained_analysis_actual.cjs > build/qa/retained-analysis/runs-browser.json
+        env:
+          PYTHON: python
+          BROWSER: /usr/bin/google-chrome
+          TULDOK_ANALYSIS_ACTUAL: '1'
+          TULDOK_ANALYSIS_HTTP: '1'
+          TULDOK_ANALYSIS_FIXTURE: ${{ github.workspace }}/build/retained-analysis-fixture
       - name: Verify exact retained scientific comparison in the existing workspace
         run: node tests/browser_scientific_comparison.cjs
         env:

agent_api.py

@@ -477,6 +477,12 @@ def model_operation(name,method,path,properties=None,required=None,*,output=None
          'predictions':{'oneOf':[{'type':'null'},_EVIDENCE_DOCUMENT]}})]}
 model_operation('runs.model_evidence','POST','/api/workspace/runs/{run_id}/model-evidence',
                 {'run_id':ID,'checkpoint_sha256':SHA},['run_id'],output=_EVIDENCE_RESULT)
+from retained_analysis_schema import schemas as _retained_analysis_schemas
+_retained_request, _retained_result = _retained_analysis_schemas(obj, text, integer, array, const, ID, SHA)
+for _name, _suffix in (('runs.retained_analysis', ''), ('runs.export_retained_analysis', '/export')):
+    model_operation(_name, 'POST', '/api/workspace/runs/{run_id}/retained-analysis' + _suffix,
+                    _retained_request['properties'], _retained_request['required'], output=_retained_result,
+                    description='Read one exact retained classifier curve and bounded example page; current inspect access required.')
 model_operation('runs.compare_model_evidence','POST','/api/workspace/runs/{run_id}/compare-model-evidence',
                 {'run_id':ID,'other_run_id':ID},output=obj({
                     'state':{'enum':['compatible','incompatible']},'run_ids':array(ID,2,2),

app.py

@@ -958,6 +958,12 @@ def _get(self):
                     '/workspace-external.js': ('workspace-external.js', 'text/javascript'),
                     '/workspace-phase-timings.js': ('workspace-phase-timings.js', 'text/javascript'),
                     '/workspace-service.js': ('workspace-service.js', 'text/javascript'),
+                    '/static/retained-analysis.js': ('retained-analysis.js', 'text/javascript'),
+                    '/static/retained-checkpoint-curve.vl.json': ('retained-checkpoint-curve.vl.json', 'application/json'),
+                    '/static/vendor/retained-analysis/vega.min.js': ('vendor/retained-analysis/vega.min.js', 'text/javascript'),
+                    '/static/vendor/retained-analysis/vega-lite.min.js': ('vendor/retained-analysis/vega-lite.min.js', 'text/javascript'),
+                    '/static/vendor/retained-analysis/vega-embed.min.js': ('vendor/retained-analysis/vega-embed.min.js', 'text/javascript'),
+                    '/static/vendor/retained-analysis/vega-interpreter.min.js': ('vendor/retained-analysis/vega-interpreter.min.js', 'text/javascript'),
                     '/workspace-scientific-comparison.js': ('workspace-scientific-comparison.js', 'text/javascript'),
                     '/workspace-fixtures.json': ('workspace-fixtures.json', 'application/json'),
                     '/pipeline-editor.js': ('pipeline-editor.js', 'text/javascript'),
@@ -1173,6 +1179,14 @@ def graph_guard(graph):
                             self._graph_snapshot_contract_access(body['graph_id'],body['graph_revision'],
                                 'runs.submit_external',2,graph=graph)
                         return self.reply(dataset.runs.submit_external(body,_graph_guard=graph_guard), 201)
+                    if (len(parts) == 3 or len(parts) == 4 and parts[3] == 'export') and parts[0] == 'runs' and parts[2] == 'retained-analysis':
+                        import retained_analysis
+                        operation = 'runs.export_retained_analysis' if len(parts) == 4 else 'runs.retained_analysis'
+                        agent_api.require_selection(self._contract_selection, operation)
+                        # The selected-contract decoder rejects all duplicate
+                        # body Run IDs before this path-owned identity is added.
+                        return self.reply(retained_analysis.query(dataset.runs, dict(body, run_id=parts[1]),
+                                                                  export=len(parts) == 4))
                     if len(parts) == 3 and parts[0] == 'runs':
                         if parts[2] == 'model-evidence':
                             agent_api.require_selection(self._contract_selection,'runs.model_evidence')

docs/contracts/retained-analysis-query.md

@@ -0,0 +1,95 @@
+# Retained checkpoint examples query v1
+
+Status: implemented and qualified in the verified native classifier Runs view.
+Python entry point: `retained_analysis.query(runs, body)`. HTTP consumers select
+`runs.retained_analysis/1` or `runs.export_retained_analysis/1` explicitly with
+its published capability hash. POST routes are
+`/api/workspace/runs/<run_id>/retained-analysis` and its `/export` sibling.
+HTTP bodies omit run_id: the selected decoder rejects duplicate body IDs and
+the handler injects the validated URL ID. The Python example below includes it
+because the direct query entry point receives the full exact request.
+
+```json
+{
+  "query_id": "retained_checkpoint_examples",
+  "query_version": 1,
+  "run_id": "<exact authorized native Run ID>",
+  "evaluation_sha256": "<model-evidence.json exact artifact hash>",
+  "checkpoint_sha256": "<exact retained checkpoint descriptor hash>",
+  "metric_id": "accuracy",
+  "metric_definition_sha256": "<exact metric definition hash>",
+  "fields": ["example_id", "truth_index", "predicted_index", "confidence", "error"],
+  "errors_only": false,
+  "limit": 50,
+  "offset": 0
+}
+```
+
+Closed object; last four fields optional with shown defaults. Metrics allow
+accuracy/loss/macro_f1 only. Fields must be unique and include example_id.
+Integer booleans are rejected; limit 1–100, offset 0–10000. One exact completed
+native `model_evidence_v1` Run, validation split only. Scientific field and
+one-fit formats require their own later query variants. All hashes lowercase
+64-character SHA256; a valid hash from another result/checkpoint is rejected.
+
+Access: inspect checked before evidence and again after the SQL read snapshot,
+including exports. No cache, training, inference, new analysis, checkpoint
+deserialization or holdout exposure. Existing Runs verification re-tallies
+retained metric evidence; it does not publish a new computed result.
+`export=True` is a host-owned keyword, never caller metadata; it runs the same
+bounded query under `runs.export_retained_analysis`. Each exported page is a
+fresh authorized query; no unbounded export endpoint or persistent URL.
+
+Runs verifies immutable bytes and graph/task compatibility. Query selects the
+exact PostgreSQL workspace_runs row and checks artifact identity. Curve SQL
+reads catalog metric history and joins verified checkpoint epochs. Every value,
+epoch and support must equal retained observations. Bounded verified per-example
+scalars are passed to PostgreSQL as a parameterized disposable recordset for
+error filtering and stable ordinal pagination. This is an initial projection,
+not a new persisted statistics schema. Original text and hash-bound bytes are
+unchanged; JSONB casts never define evidence identity.
+
+Budgets: 128 checkpoints, 10000 fixed-probe examples, 2 MiB scalar projection,
+16 MiB existing Run document, 256 KiB response, 100-row pages; SQL statement and
+lock timeouts 2000ms scoped to the read transaction, two-second ownership wait.
+Existing Runs verification/artifact budgets still apply before SQL; the SQL
+timeout is not a hard wall deadline on artifact verification. These initial
+limits are explicit admission bounds, not measured performance targets. An
+outer transaction is refused; a standalone read owns Repeatable Read and READ
+ONLY. These settings are consistency controls, not an untrusted-SQL sandbox.
+All query structure/relations/functions are server-owned; credentials remain
+server-side. No new roles, privileges, DDL or production migrations.
+
+Response: state verified; query ID/version; frozen mode; provenance including
+catalog/Run/evaluation/checkpoint/metric-definition/population/probe/data hashes
+and retained producer identity. Metric includes its original definition/units
+and unfiltered support. Curve rows carry checkpoint hash, epoch, exact value,
+support. Table columns carry type/units; rows always include stable example_id,
+ordinal and complete frozen_reference. Confidence is the retained predicted
+class probability. Errors mean truth_index != predicted_index. Population
+support, filtered support, page offset/limit/next_offset and visible filters
+remain distinct. Error filtering never recomputes curve statistics or changes
+their population. No display sampling is used: the population is explicitly
+the frozen validation probe, not an implied full dataset. selection_sha256
+identifies canonical query settings and is not a new computed-result identity.
+
+Failures use WorkbenchError: invalid request 400; unsupported query/task 409;
+stale evaluation/metric 409; foreign checkpoint/not-found Run 404; limit 413;
+unavailable standalone/read state 503 where applicable. Existing Access and
+storage errors propagate with their established boundaries. A failed or revoked
+request clears the renderer, cancel obsolete requests, and exclude
+late replies. Already returned bytes cannot be recalled.
+
+The renderer consumes an immutable base spec/result and transient allowlisted
+selections. Persisted revisioned display overrides remain a later milestone.
+Changed analysis creates a new result;
+series/color/range/display filters cannot silently redefine metric meaning.
+No renderer or spec is accepted merely because it is JSON.
+
+
+Qualification: 27 query/contract/actual PostgreSQL/authenticated HTTP tests pass
+without skips; the actual Runs browser uses the normal app, retained native Run,
+real session revocations and a fresh application process. See
+[acceptance evidence](../plans/retained-analysis/reports/m4-integrated/acceptance.md).
+The two-epoch six-image fixture proves custody, transport and numeric behavior;
+it makes no model-quality or production-scale claim.

docs/contracts/retained-analysis-renderer.md

@@ -0,0 +1,132 @@
+# Retained checkpoint renderer adapter
+
+Status: mounted in existing Runs for verified completed native classifier
+results, with lead-approved additive HTTP/contract and lifecycle hooks.
+
+`WorkspaceRetainedAnalysis.mount(root, {load, onSelection})` returns:
+
+- `update(request, authorityKey)`: clear previous private state, abort obsolete
+  work, load one bounded named query page and the pinned local spec, validate
+  identities/counts/types, then render. Host changes authorityKey and calls
+  update/dispose whenever credential, catalog, Run or navigation authority changes.
+- `resize()`: resize the active Vega view; container changes also use an owned
+  ResizeObserver.
+- `selection()`: copy of the current exact query selection, or null after
+  failure/disposal. Declared checkbox and page controls issue fresh queries.
+- `dispose()`: abort pending reads, invalidate late callbacks, finalize Vega,
+  disconnect observer and clear DOM/private state.
+
+`load` is host code accepting `(request, {signal, authorityKey, export})`, returning
+the query response. It must use current authenticated transport and the exact
+registered contract. It must never select credentials from caller metadata.
+The optional host onSelection callback receives a checkpoint hash selected by
+chart or keyboard buttons; arbitrary callbacks cannot come from chart JSON.
+
+Exports reissue the same bounded request with export true, validate the fresh
+response, then download JSON through a short-lived Blob URL that is immediately
+revoked. Failure clears existing private content. Vega export/editor/source
+actions are disabled. No export uses previously displayed bytes as authority.
+
+## Spec and dependency authority
+
+`examples/analysis/retained_checkpoint_curve.py` is a readable Altair 5.5.0 authoring
+script. Run it explicitly with its authoring-only requirements to generate
+`static/retained-checkpoint-curve.vl.json`. View opening never imports Python or
+executes the script. JSON bytes and their reviewed SHA256 are retained; JSONB
+or mutable overrides never replace that byte identity.
+
+This first recipe accepts exactly one local specification, at most 16 KiB, whose
+bytes must match the renderer's pinned digest. The specification contains a
+single named inline curve dataset, two marks and one checkpoint point-selection
+parameter. Arbitrary user specs, URLs, patches, expression functions, embed
+options and HTML/tooltips are not accepted. Changing the script/spec requires a
+new reviewed digest. The host alone supplies validated curve data. The bound
+schema hint is not fetched. A denied Vega loader prevents data/image/link URL
+loading even after compilation; tooltips are disabled and table text uses DOM
+textContent. A JSON format alone is never the security boundary.
+
+Self-hosted unmodified upstream browser distributions:
+
+| Package | Pinned version | License |
+| --- | --- | --- |
+| vega | 5.33.0 | BSD-3-Clause |
+| vega-lite | 5.23.0 | BSD-3-Clause |
+| vega-embed | 6.29.0 | BSD-3-Clause |
+| vega-interpreter | 1.2.0 | BSD-3-Clause |
+
+The v5 family matches the v5 schema emitted by Altair 5.5 and supplies UMD
+distributions without a new application bundler. Load in the table order before
+retained-analysis.js. The interpreter extends window.vega. Embed passes ast true
+and expressionInterpreter. The renderer runs with script-src self, without
+unsafe-eval or inline scripts. Its isolated fixture also uses style-src self.
+The deployed /workspaces policy retains unsafe-inline for existing application
+CSS; this grants no inline script/eval permission. The meta policy precedes all
+assets and restricts data connections to the same origin, objects/base to none,
+images/media to existing local/data/blob needs, and frames/forms to self.
+No pre-existing CSP was relaxed. `static/vendor/retained-analysis/manifest.json` records
+exact asset/tarball hashes, upstream URLs, and retained notices for packages in
+the upstream source maps. Notice-source resolution versions do not claim exact
+embedded transitive versions. BSD, MIT and ISC notices are retained with the
+assets. No license implies dataset/model rights.
+
+## Acceptance boundary
+
+`node tests/browser_retained_analysis.cjs` starts a local fixture server and real
+headless Chromium with a disposable profile. It exercises the actual assets,
+interpreter, strict CSP, exact authored numbers, physical keyboard/focus, SVG
+point selection, pagination/filter, 390px width, stale reply after revoked-mode
+response, decoder/spec tampering limits, fresh export and downloaded JSON,
+cache-bypassed page reload and disposal. No external chart-asset request or CSP
+violation is permitted. Headless/no-sandbox/software rendering is test-only and
+does not establish an application execution sandbox.
+
+The authored renderer test establishes malformed/oversized spec and response
+rejection, precise widget lifecycle and strict-CSP compatibility. It does not
+substitute for custody or access acceptance.
+
+`node tests/browser_retained_analysis_actual.cjs` separately opens the normal
+application with the existing PostgreSQL catalog and a normally trained,
+retained two-epoch native classifier fixture. It checks the actual named query,
+exact retained numbers, keyboard-linked table, 390px layout, actual response
+cancellation, authorized JSON download, real credential revocation, and a fresh
+application-process/cache-bypassed reopen. It also revokes access immediately
+before normal Run detail, prediction and model-evidence requests, proving that
+denials clear previously rendered private content even after identity becomes
+null. Deployed CSP violations are captured; ordinary runtime code generation
+is explicitly checked with the DevTools CSP bypass disabled. No remote chart
+asset is requested. These are bounded workflow checks, not full-application
+accessibility or performance certification.
+
+## Integrated host contract
+
+Two new inspect/read operations, `runs.retained_analysis/1` and
+`runs.export_retained_analysis/1`, have closed typed request/response schemas and
+exact browser pins. Every prior contract/discovery document remains unchanged.
+POST `/api/workspace/runs/<run_id>/retained-analysis` and its `/export` sibling
+require an explicitly selected contract. Bodies omit run_id and reject any
+duplicate; the handler supplies the validated URL ID. The export flag is owned
+by the route.
+
+The existing verified native classifier section constructs requests from its
+exact artifact/metric/checkpoint inventory. The widget owns a separate root and
+uses authenticated bounded transport with timeout/caller-abort wiring. Mount,
+Run replacement, navigation, session change and access failure dispose its
+view. Old responses cannot update a new selection or authority. Failed access
+also clears the containing private Run. The model transport captures Run,
+credential and generation before dispatch and forwards access denials before
+local catches, including old checkpoint/candidate/package/comparison reads.
+Aborted or invalidated-generation replies cannot trigger cleanup. Existing coordinator/phase/publication/
+scientific controllers retain their ownership; no new computation starts on
+view opening. The Dataset selector starts disabled until existing initialization
+attaches its handlers, including during local vendor-asset loading.
+
+The branch composes reliability baseline 83a1e128 and adds CI commands using one
+actual retained fixture outside uploaded QA artifacts; private login material
+stays outside evidence. The repository lead alone merges this dependent PR.
+See [acceptance](../plans/retained-analysis/reports/m4-integrated/acceptance.md).
+
+Official references checked 2026-10-19/20:
+[Altair serialization](https://altair-viz.github.io/user_guide/saving_charts.html),
+[Vega-Lite parameters](https://vega.github.io/vega-lite/docs/parameter.html),
+[vega-embed options](https://vega.github.io/vega-embed/index.html),
+[Vega interpreter](https://vega.github.io/vega/usage/interpreter/).