From 28f61e80db398f2154560a283e3614ee564bc648 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 22:11:16 +0000 Subject: [PATCH 1/7] feat: admit bounded indexed diagnostic evidence --- docs/contracts/indexed-diagnostics.md | 116 +++++++++ docs/contracts/run-preflight-publication.md | 18 ++ .../m1-indexed-diagnostics/pure-tests.log | 4 + .../reports/m1-indexed-diagnostics/report.md | 56 ++++ extension_evidence.py | 17 +- recipe_contract.py | 17 +- tests/test_indexed_diagnostics.py | 223 ++++++++++++++++ tests/test_indexed_diagnostics_publication.py | 217 ++++++++++++++++ tuldok_extension_sdk/diagnostics.py | 240 ++++++++++++++++++ tuldok_extension_sdk/evidence.py | 11 + 10 files changed, 914 insertions(+), 5 deletions(-) create mode 100644 docs/contracts/indexed-diagnostics.md create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md create mode 100644 tests/test_indexed_diagnostics.py create mode 100644 tests/test_indexed_diagnostics_publication.py create mode 100644 tuldok_extension_sdk/diagnostics.py diff --git a/docs/contracts/indexed-diagnostics.md b/docs/contracts/indexed-diagnostics.md new file mode 100644 index 00000000..12e501e1 --- /dev/null +++ b/docs/contracts/indexed-diagnostics.md @@ -0,0 +1,116 @@ +# Indexed diagnostics v1 + +`extension_field_policy_indexed_v1` is an explicit opt-in policy variant for the +existing noncanonical `external_model_evidence_recipe_v1` and bound +`external_model_evidence_recipe_v2` recipe families. It adds one required +`diagnostic_capture` member. Canonical field policies are outside this variant. +Legacy policies remain closed and cannot gain this member. An arbitrary legacy +artifact named `diagnostics.json` or using a diagnostic role gains no index +semantics. + +The existing `model_evidence_v1` envelope, profile capability lists, profile Ref +meaning and public evidence projections are unchanged. The complete admitted +policy and recipe/material/context bind the addon; its `capture_sha256` is the +canonical hash of `diagnostic_capture`. A profile Ref alone does not identify +the addon. Only opted-in execution material adds the shared SDK contract modules +to its recorded host closure. Retained legacy host-name sets remain unchanged. + +## Declaration and admission + +`diagnostic_capture` is closed over these fields: + +| Field | Contract | +| --- | --- | +| `format` | `diagnostic_capture_v1` | +| `payload_schema` | `diagnostic_json_object_v1`: bounded finite JSON object, with no inferred numerical meaning | +| `index` | `{name, max_bytes}` matching a required declared output with role `diagnostic_index_v1` | +| `shards` | Ordered, unique `{name, max_bytes}` entries matching optional declared outputs with role `diagnostic_shard_v1` | +| `phases` | Unique flat names, at most the selected recipe's output count | +| `coverage` | Ordered scopes `{case_id, checkpoint_point, phase, required_count, optional_count}` | +| `records_per_shard` | Positive exact integer; each shard must fit that many maximum-size records plus its JSON envelope | +| `max_record_bytes` | Maximum canonical UTF-8 bytes of one complete record | + +Case IDs must belong to the fixed admitted validation probe. Checkpoint points +must belong to the complete admitted capture cadence. Phases refer to the +explicit declared phase list. Scopes sort by admitted probe order, checkpoint +point, then declared phase order, without repeated scopes. Within each scope, +required records precede optional records. Contiguous global ordinals identify +the complete declared population. + +The population must fit `len(shards) × records_per_shard`. Shards retain the +existing selected per-file ceiling, at most 8 MiB. The recipe's existing output +file-count and aggregate caps remain authoritative. This opt-in variant reserves +the sum of **all declared output maxima** inside the profile aggregate budget; +legacy policies keep their existing aggregate semantics. Admission also reserves +an index large enough for all shard descriptors and uncompressed optional +omission ranges. Impossible record identity, shard, index, population or aggregate +reservations refuse before execution admission and grant consumption. Scientific +array minimum-size checks still apply independently. + +Diagnostic names must be distinct from all common scientific documents and +bound probe/prediction/reference files. They cannot replace scientific evidence. + +## SDK producer + +Use the supported `tuldok_extension_sdk.evidence.Evidence` writer to finish the +complete scientific envelope, then call `writer.diagnostics(records, omitted)`. +The writer checks the full record population, all ranges and serialized byte +bounds before writing diagnostics. It never samples or truncates required data. +`writer.inventory()` includes the exact bytes it wrote; this is construction +metadata, not collection permission. + +Each record is closed over `{ordinal, case_id, checkpoint_point, +checkpoint_sha256, phase, payload}`. Its checkpoint SHA-256 must equal the actual +retained checkpoint descriptor at that admitted point. `payload` is a finite +JSON object under the existing SDK JSON depth/node/string bounds. Its content +does not establish a metric, physical operator or acceptance criterion. + +`omitted` is an explicit sorted nonoverlapping list of `{start, count, reason}` +ranges. Each range stays inside one scope's optional population. Reasons are +`capture_budget`, `not_available` or `not_requested`. Required ordinals cannot be +omitted. Every remaining ordinal must occur exactly once, in order. + +## Retained index and shards + +The index is `application/json`, schema `diagnostic_index_v1`, closed over +`{format, payload_schema, run_id, material_sha256, capture_sha256, shards, +total_shard_bytes, coverage}`. It binds the current Run and exact composition +material. Each ordered shard entry is `{blob, record_count, first_ordinal, +last_ordinal}`. `blob` is `{name, bytes, sha256, media_type}` with media type +`application/json`. Shards are consumed in the declaration's order; all except +the final retained shard contain exactly `records_per_shard` records. Unused +declared optional shard files must be absent. + +Each shard is closed over `{format, run_id, capture_sha256, records}` with format +`diagnostic_shard_v1`. Every file has its exact size/hash, record count and ordinal +range checked. `total_shard_bytes` equals the sum of retained shard sizes. +Coverage is closed over `{total_count, required_count, optional_count, +retained_required_count, retained_optional_count, omitted_count, omitted}`; +counts must agree with the admitted population, retained records and omissions. +Complete mandatory coverage and omitted optional detail remain distinct. + +Runs' existing outer artifact inventory retains its generic +`application/octet-stream` transport media type. The addon specifies JSON's +semantic media type within that unchanged byte transport. + +## Host verification and publication + +The existing evidence owner first verifies the complete scientific envelope and +all current artifact bytes against Runs' retained inventory. Only then does the +shared diagnostic contract verify those already retained bytes. The proof is +private `_diagnostics`; existing query/publication evidence projections strip +private members and retain their closed public grammar. + +Publication continuation, custody, permission, accounting, recovery and result +Refs remain owned by Runs. Consumers inspect or collect the declared index and +its named shards through the existing exact inventory and whole-hash artifact +readers. No arbitrary path, glob, alternate store or new export authority exists. +Recovery reruns the same host verifier. An invalid original scientific output +still fails; an index cannot repair it or confer numerical acceptance. + +Focused executable witnesses are `tests/test_indexed_diagnostics.py` and +`tests/test_indexed_diagnostics_publication.py`. The latter explicitly uses a +tiny `/usr/bin/true` metadata runtime and harness serialization before the +actual no-op child. It qualifies real custody, interrupted-copy recovery, exact +readers and unchanged accounting; it proves no captured Python producer, +training, CUDA, numerical or lab-failure classification result. diff --git a/docs/contracts/run-preflight-publication.md b/docs/contracts/run-preflight-publication.md index 224e9e17..09decd24 100644 --- a/docs/contracts/run-preflight-publication.md +++ b/docs/contracts/run-preflight-publication.md @@ -316,3 +316,21 @@ captures/context have already been written, so the same owner checks its origina space and do not account for future concurrent consumers. The later admission recheck and publication's separate actual-output-bytes plus 512 MiB check remain required. A passing observation cannot promise durable publication or recovery. + +## Opt-in indexed diagnostic capture + +The noncanonical `extension_field_policy_indexed_v1` policy adds a hash-bound +`diagnostic_capture_v1` declaration. Admission reserves its exact bounded index, +ordered shard outputs, record populations and complete mandatory coverage under +the existing output/file/aggregate caps. This opt-in variant reserves all +declared output maxima within the selected profile aggregate budget. Existing +policies and closed evidence/public projection grammars retain their meanings. + +The current evidence owner verifies scientific evidence first and diagnostic +bytes against the same retained artifact inventory. Publication recovery and +exact-hash result readers expose the admitted index/shards under the original +Run and result Ref; no replay or alternate permission path is introduced. See +[Indexed diagnostics v1](indexed-diagnostics.md) for the complete producer/host +contract and the distinction between required coverage and explicit optional +omission ranges. This generic contract does not classify absent original lab +diagnostic bytes or repair an invalid original output. diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests.log new file mode 100644 index 00000000..cf20f2dd --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests.log @@ -0,0 +1,4 @@ +---------------------------------------------------------------------- +Ran 53 tests in 2.342s + +OK (skipped=2) diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md new file mode 100644 index 00000000..62fae0a9 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md @@ -0,0 +1,56 @@ +# M1 bounded indexed diagnostics implementation + +Owner: indexed diagnostics M1 delegated branch +`feature/lab-indexed-diagnostics-20261019`, worktree +`/workspace/Tuldok-lab-indexed-diagnostics-m1`. Implementation base: +`2bf1d4035510003c6bbcc84d94b578a7ee753d69`. Root owns independent review, +publication, integration and terminal worktree disposition. This private branch +is retained for that review; no remote operation or cleanup was performed. + +The source implements the generic C2 addon under the explicit +`extension_field_policy_indexed_v1` policy tag. SDK and host share one bounded +index/record contract. The policy, recipe/material/context, addon digest and +opt-in host module closure bind it. Existing scientific manifests, capabilities, +legacy retained host-name sets and public evidence grammars remain unchanged. +The host proof stays private `_diagnostics`; exact original inventory readers +provide index/shard inspection and collection. + +The new policy reserves all declared output maxima inside the existing profile +aggregate cap. It refuses impossible population, record identity, shard or index +reservations before compute. Complete mandatory records cannot be omitted; +optional omissions carry explicit count/range/reason. Existing scientific +verification runs first and an index cannot repair invalid scientific bytes. + +## Evidence at initial source freeze + +`pure-tests.log`: 53 tests run, 2 expected database-dependent skips, all remaining +tests passed. Command: + +```sh +PYTHONPATH=tests:. /workspace/tuldok-runtime/bin/python -m unittest \ + test_indexed_diagnostics test_extension_field_evidence \ + test_extension_sdk_evidence \ + test_run_preflight_publication.FieldOutputPreflightTests \ + test_run_publication_http.PublicationContractCompatibilityTests -q +``` + +This covers supported SDK serialization, two deterministic shards, cold host +verification, exact inventory precedence over coherent worker forgery, +case/checkpoint/phase identity, byte/count/range/media/coverage failures, +scientific refusal precedence, opted-in admission bounds and unchanged closed +public contract validation. One intermediate test-only failure used an invalid +`selectors` argument for `runs.model_evidence/2`; the test was corrected to the +existing flat request grammar. No production contract was changed to pass it. + +`git diff --check` passed. Actual PostgreSQL/custodian qualification is pending +the coordinated benchmark quiet-window release. Its executable fixture is +`tests/test_indexed_diagnostics_publication.py`: a fresh tiny metadata runtime +(`/usr/bin/true`, dummy 0.0 packages), captured SDK source serialized by the +harness before launch, real no-op child/custodian, injected publication copy +fault, cold recovery, exact inventory reads and unchanged accounting. + +This is generic bounded diagnostic protocol work. Original actual lab diagnostic +failure bytes/path remain unavailable, so PUB-1 classification is blocked. No +captured Python producer execution, training, CUDA, physical operator, +scientific criterion or numerical acceptance is claimed. Full M1/T1/T2 acceptance +remains the root integration owner's decision after review and qualification. diff --git a/extension_evidence.py b/extension_evidence.py index 7c45c23c..92abbbbd 100644 --- a/extension_evidence.py +++ b/extension_evidence.py @@ -133,10 +133,12 @@ def validate_policy(value, definition): import recipe_contract bound=recipe_contract.is_bound_evidence(definition) canonical_mode=recipe_contract.is_canonical_evidence(definition) + indexed=recipe_contract.is_indexed_evidence(definition) closed(value,('format','task','adapter_sha256','profile','probe','fit_members','fields','field_outputs', 'metric_definitions','model_specification','preprocessing_policy','output_policy','probe_inputs', - *(['field_targets'] if bound else []),*(['canonical'] if canonical_mode else [])),'Policy') - require(value['format']==(recipe_contract.CANONICAL_POLICY_FORMAT if canonical_mode else POLICY_FORMAT),'Unsupported policy.','unsupported') + *(['field_targets'] if bound else []),*(['canonical'] if canonical_mode else []), + *(['diagnostic_capture'] if indexed else [])),'Policy') + require(value['format']==(recipe_contract.CANONICAL_POLICY_FORMAT if canonical_mode else recipe_contract.INDEXED_POLICY_FORMAT if indexed else POLICY_FORMAT),'Unsupported policy.','unsupported') require(recipe_contract.is_evidence(definition),'Policy requires the selected model-evidence recipe.','unsupported') require(value['adapter_sha256']==digest(sorted(definition['sources'],key=lambda item:item['path'])),'Adapter source closure differs.') task=value['task'];closed(task,('id','revision','definition'),'Task declaration') @@ -251,6 +253,10 @@ def validate_policy(value, definition): if canonical_mode: import extension_canonical extension_canonical.validate_declaration(value['canonical'],value,definition) + if indexed: + from tuldok_extension_sdk import diagnostics + try:diagnostics.validate_capture(value['diagnostic_capture'],value,definition['outputs']) + except diagnostics.EvidenceError as exc:raise WorkbenchError(str(exc),exc.state,400) from None return value @@ -562,11 +568,16 @@ def blob(item): 'gauge_policy':field['gauge_policy']} field_records.append({'checkpoint_sha256':checkpoint_sha,'member_ordinal':member['ordinal'],'field_id':field['id'],'stage':output['stage'],'shape_zyx':output['shape_zyx'],'units':output['units']}) guard() - return {'state':'verified','validation':'structural_and_identity_closure; numerical_acceptance_is_separate', + result={'state':'verified','validation':'structural_and_identity_closure; numerical_acceptance_is_separate', 'manifest':manifest,'checkpoint_index':index,'probe':static['probe'],'metric_definitions':policy['metric_definitions'], 'observations':observations,'model_definition':policy['model_specification'],'fields':field_records, 'metric_definition_hashes':{item['id']:digest(item) for item in policy['metric_definitions']}, 'checkpoints':[item['descriptor'] for item in checkpoints.values()], '_arrays':arrays} + if recipe_contract.is_indexed_evidence(run['material']['recipe']): + from tuldok_extension_sdk import diagnostics + try:result['_diagnostics']=diagnostics.verify(policy['diagnostic_capture'],policy,payloads,admitted_context,index,guard) + except diagnostics.EvidenceError as exc:raise WorkbenchError(str(exc),exc.state,400) from None + return result @boundary diff --git a/recipe_contract.py b/recipe_contract.py index 68832a10..351d3730 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -34,6 +34,8 @@ CPU_BOOTSTRAP = 'runtime-bootstrap/recipe_process.py' HOST_NAMES = ('recipe_contract.py', 'projects.py', 'pipeline.py', 'recipe_process.py') EVIDENCE_HOST_NAMES = (*HOST_NAMES, 'extension_evidence.py', 'field_evidence.py', 'scientific_fields.py', 'scientific_release.py') +INDEXED_POLICY_FORMAT = 'extension_field_policy_indexed_v1' +DIAGNOSTIC_HOST_NAMES = ('tuldok_extension_sdk/__init__.py', 'tuldok_extension_sdk/evidence.py', 'tuldok_extension_sdk/diagnostics.py') EVIDENCE_FORMAT = 'external_model_evidence_recipe_v1' BOUND_EVIDENCE_FORMAT = 'external_model_evidence_recipe_v2' BOUND_EVIDENCE_HOST_NAMES = (*EVIDENCE_HOST_NAMES, 'extension_bindings.py', 'extension_contract.py', 'agent_api.py', @@ -49,6 +51,7 @@ if all((Path(__file__).resolve().parent/name).is_file() for name in EVIDENCE_HOST_NAMES) else None) IMPORTED_BOUND_EVIDENCE_HOST_FILES = {name: hashlib.sha256((Path(__file__).resolve().parent/name).read_bytes()).hexdigest() for name in BOUND_EVIDENCE_HOST_NAMES} +IMPORTED_DIAGNOSTIC_HOST_FILES = {name: hashlib.sha256((Path(__file__).resolve().parent/name).read_bytes()).hexdigest() for name in DIAGNOSTIC_HOST_NAMES} IMPORTED_CANONICAL_EVIDENCE_HOST_FILES = ({name: hashlib.sha256((Path(__file__).resolve().parent/name).read_bytes()).hexdigest() for name in CANONICAL_EVIDENCE_HOST_NAMES} if all((Path(__file__).resolve().parent/name).is_file() for name in CANONICAL_EVIDENCE_HOST_NAMES) else None) @@ -63,6 +66,11 @@ def is_canonical_evidence(value): and value['model_evidence'].get('format') == CANONICAL_POLICY_FORMAT) +def is_indexed_evidence(value): + return (is_evidence(value) and type(value.get('model_evidence')) is dict + and value['model_evidence'].get('format') == INDEXED_POLICY_FORMAT) + + def is_evidence(value): return (is_bound_evidence(value) or (type(value) is dict and type(value.get('schema_version')) is int and value['schema_version'] == 2 @@ -600,13 +608,16 @@ def validate_canonical_captures(material, captured, *, outputs=None): return inputs -def host_identity(*, evidence=False, bound=False, canonical=False, torch=False): +def host_identity(*, evidence=False, bound=False, canonical=False, torch=False, indexed=False): import recipe_process root = Path(__file__).resolve().parent imported = IMPORTED_CANONICAL_EVIDENCE_HOST_FILES if canonical else IMPORTED_BOUND_EVIDENCE_HOST_FILES if bound else IMPORTED_EVIDENCE_HOST_FILES if evidence else IMPORTED_HOST_FILES if imported is None: raise WorkbenchError('Canonical field evidence dependency is unavailable; restart after explicit provisioning.', 'unavailable', 409) names = CANONICAL_EVIDENCE_HOST_NAMES if canonical else BOUND_EVIDENCE_HOST_NAMES if bound else EVIDENCE_HOST_NAMES if evidence else HOST_NAMES + if indexed: + names = (*names, *DIAGNOSTIC_HOST_NAMES) + imported = {**imported, **IMPORTED_DIAGNOSTIC_HOST_FILES} if torch: names = (*names, *TORCH_HOST_NAMES) imported = {**imported, **IMPORTED_TORCH_HOST_FILES} @@ -618,7 +629,8 @@ def host_identity(*, evidence=False, bound=False, canonical=False, torch=False): def recipe_host_identity(definition): - return host_identity(evidence=is_evidence(definition),bound=is_bound_evidence(definition),canonical=is_canonical_evidence(definition),torch=definition['runtime']==TORCH_PROFILE) + return host_identity(evidence=is_evidence(definition),bound=is_bound_evidence(definition),canonical=is_canonical_evidence(definition), + torch=definition['runtime']==TORCH_PROFILE,indexed=is_indexed_evidence(definition)) def revalidate_bound_material(projects, material, captured, profile=None, *, releases=None): @@ -944,6 +956,7 @@ def receipt(value): if total > RUNTIME_LIMIT or runtime['executable'] not in files or files[runtime['executable']]['sha256'] != runtime['executable_sha256'] or digest(files) != runtime['closure_sha256']: raise WorkbenchError('Runtime closure identity is inconsistent.') host_names = CANONICAL_EVIDENCE_HOST_NAMES if is_canonical_evidence(definition) else BOUND_EVIDENCE_HOST_NAMES if bound else EVIDENCE_HOST_NAMES if evidence else HOST_NAMES + if is_indexed_evidence(definition): host_names = (*host_names, *DIAGNOSTIC_HOST_NAMES) if definition['runtime'] == TORCH_PROFILE: host_names = (*host_names, *TORCH_HOST_NAMES) if not isinstance(material['host_files'],dict) or set(material['host_files']) != set(host_names): raise WorkbenchError('Invalid retained host closure.') for value in material['host_files'].values(): sha(value) diff --git a/tests/test_indexed_diagnostics.py b/tests/test_indexed_diagnostics.py new file mode 100644 index 00000000..841241a8 --- /dev/null +++ b/tests/test_indexed_diagnostics.py @@ -0,0 +1,223 @@ +"""Bounded diagnostic protocol evidence; synthetic JSON is never a science claim.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import extension_evidence as host +import recipe_contract +from tuldok_extension_sdk import diagnostics +from tuldok_extension_sdk.evidence import Evidence, EvidenceError +from workbench import WorkbenchError +from test_extension_field_evidence import authored_fixture + + +def opt_in(definition): + """A fresh explicit policy, preserving every old science output/grammar.""" + policy = definition['model_evidence'] + policy['format'] = diagnostics.POLICY_FORMAT + case = policy['probe']['candidates'][0]['frozen_reference']['member_id'] + capture = dict(format=diagnostics.CAPTURE_FORMAT, payload_schema=diagnostics.PAYLOAD_SCHEMA, + index=dict(name='diagnostic-index.json', max_bytes=16384), + shards=[dict(name=f'diagnostic-{ordinal}.json', max_bytes=8192) for ordinal in range(2)], + records_per_shard=3, max_record_bytes=1024, phases=['serialization'], coverage=[ + dict(case_id=case, checkpoint_point=16, phase='serialization', required_count=1, optional_count=2), + dict(case_id=case, checkpoint_point=32, phase='serialization', required_count=1, optional_count=1)]) + policy['diagnostic_capture'] = capture + for artifact, role, required in [(capture['index'], diagnostics.INDEX_ROLE, True), + *((item, diagnostics.SHARD_ROLE, False) for item in capture['shards'])]: + definition['outputs'].append(dict(path=artifact['name'], max_bytes=artifact['max_bytes'], role=role, required=required)) + return capture + + +def indexed_fixture(root): + definition, model, composition, run, _legacy_writer = authored_fixture(root) + capture = opt_in(definition) + recipe_contract.decode_definition(host.canonical(definition)) + execution = run['material'] + static = host.resolve(definition['model_evidence'], model, execution) + run['material_sha256'] = host.digest(execution) + composition['material']['executable_sha256'] = run['material_sha256'] + context = host.context(static, composition['ref'], run['id'], run['request_sha256']) + context_raw = host.canonical(context) + run['extension_context'] = context + run['spec']['model_evidence_context'].update(bytes=len(context_raw), sha256=hashlib.sha256(context_raw).hexdigest()) + for path in root.glob('*.json'): + if path.name != 'diagnostics.json': + path.unlink() + writer = Evidence(root, context_raw) + writer.transform('transform.npy') + for point in (16, 32, 48, 64): + writer.checkpoint(point, dict(optimizer_update=point, epoch=None, fit_pass=None, physical_time=None, wall_elapsed_ms=point), + f'weights-{point}.npy', dict(reference_mse=0)) + writer.finish() + checkpoints = {entry['clocks']['optimizer_update']: entry['checkpoint']['sha256'] for entry in writer.entries} + records = [dict(ordinal=ordinal, case_id=capture['coverage'][0]['case_id'], checkpoint_point=point, + checkpoint_sha256=checkpoints[point], phase='serialization', + payload=dict(scope='synthetic protocol JSON; no training', label=str(ordinal))) + for ordinal, point in ((0, 16), (1, 16), (3, 32), (4, 32))] + omitted = [dict(start=2, count=1, reason='capture_budget')] + index = writer.diagnostics(records, omitted) + refresh_inventory(root, run) + return definition, run, writer, records, omitted, index + + +def refresh_inventory(root, run): + run['artifacts'] = [dict(name=row['path'], role=row['role'], bytes=(root / row['path']).stat().st_size, + sha256=hashlib.sha256((root / row['path']).read_bytes()).hexdigest(), media_type='application/octet-stream') + for row in run['outputs'] if (root / row['path']).is_file()] + + +class DiagnosticAdmissionTests(unittest.TestCase): + def setUp(self): + self.definition = json.loads((Path(__file__).resolve().parents[1] / 'examples/extensions/field-surrogate/v2/recipe.json').read_bytes()) + self.capture = opt_in(self.definition) + + def decode(self): + with patch('subprocess.Popen', side_effect=AssertionError('Diagnostic admission launched compute')): + return recipe_contract.decode_definition(host.canonical(self.definition)) + + def test_explicit_opt_in_keeps_profile_and_closed_science_capabilities(self): + self.assertEqual(self.decode(), self.definition) + self.assertEqual(self.definition['model_evidence']['profile']['required_capabilities'], host.CAPABILITIES) + self.assertEqual(self.definition['model_evidence']['profile']['optional_capabilities'], []) + self.assertIn('tuldok_extension_sdk/diagnostics.py', recipe_contract.recipe_host_identity(self.definition)) + self.assertEqual(diagnostics.POLICY_FORMAT, recipe_contract.INDEXED_POLICY_FORMAT) + legacy = json.loads((Path(__file__).resolve().parents[1] / 'examples/extensions/field-surrogate/v2/recipe.json').read_bytes()) + self.assertEqual(set(recipe_contract.recipe_host_identity(legacy)), set(recipe_contract.EVIDENCE_HOST_NAMES)) + + def test_impossible_population_record_index_and_aggregate_declarations_refuse_inertly(self): + changes = [ + lambda: self.capture['coverage'][0].update(required_count=7), + lambda: self.capture.update(max_record_bytes=8192), + lambda: self.capture.update(max_record_bytes=1), + lambda: self.definition['model_evidence']['profile']['limits'].update(total_output_bytes=920000), + ] + original = copy.deepcopy(self.definition) + for change in changes: + with self.subTest(change=change): + self.definition = copy.deepcopy(original) + self.capture = self.definition['model_evidence']['diagnostic_capture'] + change() + with self.assertRaises(WorkbenchError) as caught: + self.decode() + self.assertEqual(caught.exception.code, 'unsupported') + + def test_addon_requires_explicit_tag_and_exact_output_roles(self): + original = copy.deepcopy(self.definition) + for change in ( + lambda: self.definition['model_evidence'].update(format=host.POLICY_FORMAT), + lambda: self.definition['outputs'][-1].update(role='arbitrary_diagnostic'), + lambda: self.definition['outputs'][-1].update(required=True), + lambda: self.capture['coverage'][0].update(case_id='unadmitted-case'), + lambda: self.capture['coverage'][0].update(checkpoint_point=17), + lambda: self.capture['coverage'][0].update(phase='unadmitted-phase'), + ): + with self.subTest(change=change): + self.definition = copy.deepcopy(original) + self.capture = self.definition['model_evidence']['diagnostic_capture'] + change() + with self.assertRaises(WorkbenchError): self.decode() + + +class DiagnosticRoundTripTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.definition, self.run, self.writer, self.records, self.omitted, self.index = indexed_fixture(self.root) + self.capture = self.definition['model_evidence']['diagnostic_capture'] + + def verify(self): + return host.verify(self.root, self.run, self.run['extension_context']) + + def rewrite(self, filename, value, refresh=True): + (self.root / filename).write_bytes(host.canonical(value) + b'\n') + if refresh: refresh_inventory(self.root, self.run) + + def test_supported_sdk_two_shards_cold_host_and_unchanged_public_projection(self): + with patch('subprocess.Popen', side_effect=AssertionError('Diagnostic verification launched compute')): + verified = self.verify() + self.assertEqual(verified['_diagnostics'], self.index) + cold = host.verify(Path(str(self.root)), copy.deepcopy(self.run), copy.deepcopy(self.run['extension_context'])) + self.assertEqual(cold['_diagnostics'], self.index) + self.assertEqual(len(self.index['shards']), 2) + self.assertEqual(self.index['coverage'], dict(total_count=5, required_count=2, optional_count=3, + retained_required_count=2, retained_optional_count=2, omitted_count=1, omitted=self.omitted)) + self.assertNotIn('_diagnostics', host.query(verified)) + self.assertNotIn('diagnostics', host.query(verified)) + import agent_api + client = agent_api.AgentClient('http://127.0.0.1:12345', 'x' * 32, self.run['dataset_id']) + client._ready = True; client._selected = {'runs.model_evidence': 2} + response = {'status': 200, 'result': host.query(verified)} + with patch.object(client, '_request', return_value=response): + self.assertEqual(client.call('runs.model_evidence', {'run_id': self.run['id']}), response) + self.assertEqual(set(verified['manifest']), {'format', 'schema_version', 'producer', 'task', 'model_sha256', + 'data_sha256', 'profile_sha256', 'probe', 'metric_definitions', 'checkpoints', 'observations', 'field_index', 'coverage', 'inference'}) + + def test_current_host_inventory_hash_still_dominates_forged_coherent_index(self): + shard = json.loads((self.root / 'diagnostic-0.json').read_bytes()) + shard['records'][0]['payload']['label'] = 'mutated' + self.rewrite('diagnostic-0.json', shard, refresh=False) + index = copy.deepcopy(self.index) + index['shards'][0]['blob'] = dict( + name='diagnostic-0.json', bytes=(self.root / 'diagnostic-0.json').stat().st_size, + sha256=hashlib.sha256((self.root / 'diagnostic-0.json').read_bytes()).hexdigest(), media_type='application/json') + self.rewrite('diagnostic-index.json', index, refresh=False) + with self.assertRaisesRegex(WorkbenchError, 'authoritative retained artifact'): + self.verify() + + def test_index_identity_order_count_ranges_media_and_mandatory_coverage_refuse(self): + for change in ( + lambda index: index.update(capture_sha256='f' * 64), + lambda index: index.update(material_sha256='f' * 64), + lambda index: index['shards'].reverse(), + lambda index: index['shards'][0].update(record_count=2), + lambda index: index['shards'][0].update(first_ordinal=1), + lambda index: index['shards'][0]['blob'].update(media_type='text/plain'), + lambda index: index['coverage'].update(omitted_count=2), + lambda index: index['coverage'].update(omitted=[dict(start=0, count=1, reason='capture_budget')]), + lambda index: index['coverage'].update(omitted=[dict(start=2, count=1, reason='unknown')]), + ): + with self.subTest(change=change): + changed = copy.deepcopy(self.index); change(changed) + self.rewrite('diagnostic-index.json', changed) + with self.assertRaises(WorkbenchError): self.verify() + self.rewrite('diagnostic-index.json', self.index) + + def test_forged_shard_case_checkpoint_phase_order_and_payload_refuse(self): + original = (self.root / 'diagnostic-0.json').read_bytes() + for change in ( + lambda row: row.update(case_id='unadmitted'), + lambda row: row.update(checkpoint_sha256='f' * 64), + lambda row: row.update(phase='unadmitted'), + lambda row: row.update(ordinal=1), + lambda row: row.update(payload=[]), + lambda row: row.update(payload=dict(text='a' * 1024)), + ): + with self.subTest(change=change): + shard = json.loads(original); change(shard['records'][0]) + self.rewrite('diagnostic-0.json', shard) + index = copy.deepcopy(self.index); raw = (self.root / 'diagnostic-0.json').read_bytes() + index['shards'][0]['blob'].update(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + self.rewrite('diagnostic-index.json', index) + with self.assertRaises(WorkbenchError): self.verify() + + def test_diagnostic_capture_never_repairs_invalid_scientific_evidence(self): + (self.root / 'reference.npy').write_bytes(b'invalid scientific array') + refresh_inventory(self.root, self.run) + with patch.object(diagnostics, 'verify', side_effect=AssertionError('Invalid science reached diagnostic verifier')): + with self.assertRaises(WorkbenchError): self.verify() + + def test_sdk_refuses_missing_mandatory_or_unexplained_optional_records_before_writing(self): + for records, omitted in ((self.records[1:], self.omitted), (self.records, []), + (self.records, [dict(start=0, count=1, reason='not_available')])): + with self.subTest(records=records, omitted=omitted): + before = self.writer.inventory() + with self.assertRaises(EvidenceError): self.writer.diagnostics(records, omitted) + self.assertEqual(before, self.writer.inventory()) + + +if __name__ == '__main__': unittest.main() diff --git a/tests/test_indexed_diagnostics_publication.py b/tests/test_indexed_diagnostics_publication.py new file mode 100644 index 00000000..f33fb1a5 --- /dev/null +++ b/tests/test_indexed_diagnostics_publication.py @@ -0,0 +1,217 @@ +"""Fresh owner roundtrip with a tiny metadata runtime, never a training witness. + +The harness executes captured synthetic serialization source before launch. +The admitted runtime is /usr/bin/true with dummy 0.0 packages: its actual child +does no Python/model work. Custody, frozen inventory, copy fault, cold recovery, +semantic verification, exact result reads and accounting are production owners. +""" +import copy +from contextlib import redirect_stdout +import hashlib +import io +import json +import os +from pathlib import Path +import runpy +import shutil +import sys +import tempfile +import unittest +from unittest.mock import patch + +from workbench import WorkbenchError +from test_indexed_diagnostics import opt_in + +ROOT = Path(__file__).resolve().parents[1] +APPEND_DIAGNOSTICS = ''' +capture = writer.policy['diagnostic_capture'] +checkpoints = {entry['clocks']['optimizer_update']: entry['checkpoint']['sha256'] for entry in writer.entries} +records = [dict(ordinal=ordinal, case_id=capture['coverage'][0]['case_id'], checkpoint_point=point, + checkpoint_sha256=checkpoints[point], phase='serialization', + payload=dict(scope='harness synthetic serialization; no captured Python or training', label=str(ordinal))) + for ordinal, point in ((0, 16), (1, 16), (3, 32), (4, 32))] +writer.diagnostics(records, [dict(start=2, count=1, reason='capture_budget')]) +''' + + +def project_fixture(root, runtime_sha): + from qualify_publication_synthetic import project_fixture as scientific_fixture + definition, manifest, _ = scientific_fixture(root, runtime_sha) + opt_in(definition) + producer = root / 'synthetic_producer.py' + producer.write_text(producer.read_text().replace('from evidence_writer import Evidence', + 'from tuldok_extension_sdk.evidence import Evidence') + APPEND_DIAGNOSTICS) + sdk_sources = [] + for filename in ('__init__.py', 'evidence.py', 'diagnostics.py'): + relative = 'tuldok_extension_sdk/' + filename + target = root / relative; target.parent.mkdir(exist_ok=True) + target.write_bytes((ROOT / relative).read_bytes()) + sdk_sources.append(relative) + provider = json.loads((root / 'extension.json').read_bytes()) + provider['sources'] = [dict(path=name, sha256=hashlib.sha256((root / name).read_bytes()).hexdigest(), + bytes=(root / name).stat().st_size) for name in ['synthetic_producer.py', *sdk_sources]] + (root / 'extension.json').write_bytes(json.dumps(provider, sort_keys=True, separators=(',', ':')).encode()) + paths = [row['path'] for row in definition['sources']] + sdk_sources + definition['sources'] = [dict(path=name, sha256=hashlib.sha256((root / name).read_bytes()).hexdigest()) for name in sorted(paths)] + import extension_evidence + policy = definition['model_evidence']; policy['adapter_sha256'] = extension_evidence.digest(definition['sources']) + for metric in policy['metric_definitions']: metric['evaluator_sha256'] = policy['adapter_sha256'] + for field in policy['fields']: field['contract']['source_sha256'] = policy['adapter_sha256'] + raw = extension_evidence.canonical(definition); (root / 'recipe.json').write_bytes(raw) + manifest['recipes'][0]['sha256'] = hashlib.sha256(raw).hexdigest() + (root / 'project.json').write_bytes(extension_evidence.canonical(manifest)) + return definition, manifest, hashlib.sha256(raw).hexdigest() + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class IndexedDiagnosticPublicationTests(unittest.TestCase): + def setUp(self): + from app import Dataset + from auth_helpers import ExplicitOwnerFixture + from runtime_snapshots import RuntimeSnapshots + self.temp = tempfile.TemporaryDirectory(prefix='indexed-diagnostics-'); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) + self.schema = self.dataset.db.schema + self.addCleanup(self.cleanup_catalog) + self.owner = ExplicitOwnerFixture.bootstrap(self.dataset) + python = self.root / 'python'; site = self.root / 'packages' + (python / 'bin').mkdir(parents=True); (python / 'lib/python3.12/lib-dynload').mkdir(parents=True) + shutil.copyfile('/usr/bin/true', python / 'bin/python3.12') + (python / 'lib/python3.12/lib-dynload/fixture.txt').write_text('metadata fixture only') + packages = [] + for package in ('torch', 'numpy'): + (site / package).mkdir(parents=True); (site / package / '__init__.py').write_text('fixture = True\n') + info = site / (package + '-0.0.dist-info'); info.mkdir() + (info / 'METADATA').write_text('Name: ' + package + '\nVersion: 0.0\n') + (info / 'LICENSE').write_text('Test fixture') + packages.extend([package, info.name]) + config = dict(schema_version=1, python_root=str(python), python_version='3.12', site_packages=str(site), packages=packages) + with self.owner.bound(self.dataset): + snapshots = RuntimeSnapshots(self.dataset.db, self.dataset.access, self.dataset.path) + identity = snapshots.inspect(config)['inventory_sha256'] + self.runtime = snapshots.create(config, identity, quiesced=True, reserve_bytes=0) + self.definition, self.manifest, self.recipe_sha = project_fixture(self.root / 'project', identity) + self.project = self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) + + def cleanup_catalog(self): + import psycopg + from psycopg import sql + self.dataset.close() + with psycopg.connect(os.environ['TULDOK_DATABASE_URL'], autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(self.schema))) + + def graph_body(self): + return dict(name='Indexed synthetic diagnostics; metadata runtime only', project_id=self.project['id'], + recipe_id='synthetic', expected_sha256=self.recipe_sha, + inputs=dict(fixture=dict(reference_id='fixture', expected_sha256=self.manifest['data'][0]['sha256']))) + + def counts(self): + return {table: self.dataset.db.execute('SELECT count(*) FROM ' + table).fetchone()[0] + for table in ('workspace_graphs', 'workspace_runs', 'access_execution_grants', 'access_execution_consumptions', 'model_attempt_reservations')} + + def admit(self): + import extension_evidence + dataset = self.dataset; policy = self.definition['model_evidence'] + with self.owner.bound(dataset): + graph = dataset.pipelines.create_external(self.graph_body()) + draft = dataset.models.create(dict(name='Synthetic SDK serialization; no training', kind='captured_source', + task=dict(kind='task', catalog_id=dataset.db.dataset_id, id=policy['task']['id'], revision=policy['task']['revision'], + sha256=extension_evidence.digest(policy['task']['definition'])), specification=policy['model_specification'], + preprocessing_policy=policy['preprocessing_policy'], output_policy=policy['output_policy'], + source_selector=dict(project_id=self.project['id'], recipe_id='synthetic', expected_sha256=self.recipe_sha))) + model = dataset.models.publish(draft['id'], dict(revision=draft['revision'])) + selection = dict(format='selection_policy_v1', metric_definition_sha256=extension_evidence.digest(policy['metric_definitions'][0]), + checkpoint_rule='last', replicate_reduce='individual', required_capabilities=policy['profile']['required_capabilities'], + numerical_acceptance='diagnostic', tie_break='trial_id') + study = dataset.studies.create(dict(name='Synthetic indexed custody', policy=selection, + limits=dict(attempt_slots=1, admitted_wall_seconds=600, retained_bytes=64 * 1024**2, peak_staging_bytes=64 * 1024**2))) + trial = dataset.studies.prepare_trial(study['id'], dict(request_key='indexed-prepare', revision=study['revision'], + graph_id=graph['id'], graph_revision=graph['revision'], model=model['ref'], replicate=0)) + current = dataset.studies.get(study['id'])['study'] + dataset.studies.continue_study(study['id'], dict(request_key='indexed-open', revision=current['revision'], admission_epoch=current['admission_epoch'])) + grant = dataset.studies.approve(study['id'], dict(trial_id=trial['id'], expected_material_sha256=trial['material']['sha256'], + consumer_actor_id=self.owner.identity['actor_id'])) + current = dataset.studies.get(study['id'])['study'] + with patch.object(dataset.runs, '_start'): + admitted = dataset.studies.admit(study['id'], dict(request_key='indexed-admit', revision=current['revision'], + admission_epoch=current['admission_epoch'], trial_id=trial['id'], approval_id=grant['id'])) + return dataset.runs._load(admitted['run']['id']), study, trial + + def test_impossible_population_refuses_before_graph_run_grant_or_reservation(self): + import extension_evidence + self.definition['model_evidence']['diagnostic_capture']['coverage'][0]['required_count'] = 7 + raw = extension_evidence.canonical(self.definition); (self.root / 'project/recipe.json').write_bytes(raw) + self.recipe_sha = hashlib.sha256(raw).hexdigest() + self.manifest['recipes'][0]['sha256'] = self.recipe_sha + (self.root / 'project/project.json').write_bytes(extension_evidence.canonical(self.manifest)) + with self.owner.bound(self.dataset): + self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) + before = self.counts() + with patch('subprocess.Popen', side_effect=AssertionError('Invalid declaration launched compute')): + with self.assertRaises(WorkbenchError) as caught: self.dataset.pipelines.create_external(self.graph_body()) + self.assertEqual(caught.exception.code, 'unsupported') + self.assertEqual(before, self.counts()) + + def test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay(self): + import recipe_process + from app import Dataset + run, study, _trial = self.admit() + launch = recipe_process.launch + serialized = [] + def authored_serialization(spec, custody_dir): + # Harness serialization at the launch boundary, explicitly outside + # the captured no-op child. No runtime/training claim follows. + args = [str(Path(spec['cwd']) / 'source/synthetic_producer.py'), + str(Path(spec['cwd']) / 'input/fixture'), spec['model_evidence_context']['path'], spec['output_dir']] + with patch.object(sys, 'argv', args), redirect_stdout(io.StringIO()): + runpy.run_path(args[0], run_name='__main__') + serialized.append(True) + return launch(spec, custody_dir) + with patch('recipe_process.launch', side_effect=authored_serialization), \ + patch.object(self.dataset.runs, '_copy_verified', side_effect=OSError('Authored indexed publication copy interruption')): + self.dataset.runs._run_attempt(run, (self.dataset.runs._capacity.cores[0],), self.dataset.runs._capacity.claim(run)) + saved = self.dataset.runs._load(run['id']) + self.assertEqual(serialized, [True]) + self.assertEqual((saved['state'], saved['custody_phase']), ('interrupted', 'drained'), saved) + self.assertEqual(saved['publication_fault']['code'], 'unavailable') + stage = self.dataset.runs._external_directory(saved) + custody = recipe_process.load_receipt(stage / 'custody/receipt.json', saved['custody_identity']) + self.assertEqual((custody['outcome'], custody['leader_returncode']), ('completed', 0)) + before_counts = self.counts(); events = copy.deepcopy(saved['events']) + with self.owner.bound(self.dataset): before = self.dataset.studies.reconcile(study['id']) + reservation = self.dataset.db.execute('SELECT document FROM model_attempt_reservations WHERE run_id=%s', (run['id'],)).fetchone()[0] + self.dataset.close() + self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) + checked = []; verify = self.dataset.runs._verify_extension + def semantic(directory, value, **kwargs): + result = verify(directory, value, **kwargs) + self.assertEqual(result['_diagnostics']['coverage']['retained_required_count'], 2) + checked.append(str(directory)); return result + with self.owner.bound(self.dataset), patch('recipe_process.launch', side_effect=AssertionError('Recovery replayed compute')), \ + patch.object(self.dataset.runs, '_verify_extension', side_effect=semantic): + body = dict(run_id=run['id'], source_ref=saved['publication_source'], request_key='indexed-recover') + recovered = self.dataset.runs.recover_publication(body) + self.assertEqual(recovered['state'], 'published') + result = self.dataset.runs.publication_result(recovered['result_ref']) + indexed = {row['name']: row for row in result['artifacts']} + index_raw, _ = self.dataset.runs.publication_artifact(recovered['result_ref'], 'diagnostic-index.json', indexed['diagnostic-index.json']['sha256']) + index = json.loads(index_raw) + for row in index['shards']: + blob = row['blob'] + raw, item = self.dataset.runs.publication_artifact(recovered['result_ref'], blob['name'], blob['sha256']) + self.assertEqual((len(raw), hashlib.sha256(raw).hexdigest()), (blob['bytes'], blob['sha256'])) + self.assertEqual(item, indexed[blob['name']]) + public = self.dataset.runs.publication_model_evidence(recovered['result_ref'], {})['evidence'] + self.assertEqual(public['state'], 'verified'); self.assertNotIn('_diagnostics', public) + self.assertNotIn('diagnostics', public) + with self.assertRaises(WorkbenchError): + self.dataset.runs.publication_artifact(recovered['result_ref'], 'diagnostic-0.json', '0' * 64) + self.assertEqual(self.dataset.runs.recover_publication(body), recovered) + after = self.dataset.studies.reconcile(study['id']) + self.assertEqual((after['charged'], after['reservations']), (before['charged'], before['reservations'])) + self.assertEqual(self.dataset.db.execute('SELECT document FROM model_attempt_reservations WHERE run_id=%s', (run['id'],)).fetchone()[0], reservation) + self.assertEqual(self.counts(), before_counts) + self.assertEqual(self.dataset.runs._load(run['id'])['events'], events) + self.assertTrue(any('.external-' in path for path in checked), checked) + self.assertTrue(any('.external-' not in path for path in checked), checked) diff --git a/tuldok_extension_sdk/diagnostics.py b/tuldok_extension_sdk/diagnostics.py new file mode 100644 index 00000000..a98ad288 --- /dev/null +++ b/tuldok_extension_sdk/diagnostics.py @@ -0,0 +1,240 @@ +"""Versioned bounded diagnostic capture; diagnostic JSON conveys no science verdict. + +The host and SDK share this inert contract. The host supplies only bytes already +bound to its retained inventory; the writer has no read/export authority. +""" +import hashlib + +from .evidence import (EvidenceError, boundary, canonical, closed, decode, digest, + name, natural, require, sha) + +POLICY_FORMAT = 'extension_field_policy_indexed_v1' +CAPTURE_FORMAT = 'diagnostic_capture_v1' +INDEX_FORMAT = 'diagnostic_index_v1' +SHARD_FORMAT = 'diagnostic_shard_v1' +PAYLOAD_SCHEMA = 'diagnostic_json_object_v1' +INDEX_ROLE = 'diagnostic_index_v1' +SHARD_ROLE = 'diagnostic_shard_v1' +MEDIA_TYPE = 'application/json' + + +def _scopes(capture, policy): + """Resolve the declared population from existing fixed probe/capture limits.""" + members = [item['frozen_reference']['member_id'] for item in policy['probe']['candidates']] + clock = policy['profile']['capture'] + positions = [clock['interval'] * step for step in range(1, clock['max_snapshots'] + 1)] + order = {(member, point, phase): ordinal for ordinal, (member, point, phase) in enumerate( + (member, point, phase) for member in members for point in positions for phase in capture['phases'])} + previous = -1 + total = 0 + scopes = [] + for row in capture['coverage']: + closed(row, ('case_id', 'checkpoint_point', 'phase', 'required_count', 'optional_count'), 'Diagnostic scope') + natural(row['checkpoint_point'], 'Diagnostic checkpoint point', 1) + key = (row['case_id'], row['checkpoint_point'], row['phase']) + require(key in order and order[key] > previous, 'Diagnostic scope membership/order differs.') + previous = order[key] + required = natural(row['required_count'], 'Required diagnostic count') + optional = natural(row['optional_count'], 'Optional diagnostic count') + require(required + optional > 0, 'Diagnostic scope must declare a population.') + scopes.append((total, total + required, total + required + optional, row)) + total += required + optional + return scopes, total + + +@boundary +def validate_capture(capture, policy, outputs): + """Admission proof, using current output declarations and selected profile caps. + + Opt-in profiles reserve their declared output maxima within the aggregate + budget. This proves capacity before compute; actual JSON/array checks remain + mandatory after capture. + """ + closed(capture, ('format', 'index', 'shards', 'payload_schema', 'phases', 'coverage', + 'records_per_shard', 'max_record_bytes'), 'Diagnostic capture') + require(capture['format'] == CAPTURE_FORMAT and capture['payload_schema'] == PAYLOAD_SCHEMA, + 'Unsupported diagnostic capture/schema.', 'unsupported') + limits = policy['profile']['limits'] + require(type(outputs) is list and outputs, 'Diagnostic outputs are required.') + declarations = {item['path']: item for item in outputs} + require(type(capture['phases']) is list and 1 <= len(capture['phases']) <= len(outputs), 'Diagnostic phases exceed the output population bound.') + require(all(type(phase) is str and name(phase) for phase in capture['phases']) + and len(set(capture['phases'])) == len(capture['phases']), 'Diagnostic phases must be unique flat names.') + require(type(capture['shards']) is list and 1 <= len(capture['shards']) < len(outputs), 'Diagnostic shard count exceeds declared outputs.') + files = [item['name'] for item in [capture['index'], *capture['shards']]] + require(all(type(filename) is str and name(filename) for filename in files) + and len(set(files)) == len(files), 'Diagnostic filenames must be unique.') + for artifact, role, required in [(capture['index'], INDEX_ROLE, True), + *((item, SHARD_ROLE, False) for item in capture['shards'])]: + closed(artifact, ('name', 'max_bytes'), 'Diagnostic artifact declaration') + filename = artifact['name'] + rule = declarations.get(filename) + require(rule is not None and rule['role'] == role and rule['required'] is required, + 'Diagnostic output role/required declaration differs.') + natural(artifact['max_bytes'], 'Diagnostic file bound', 1, limits['shard_bytes']) + require(rule['max_bytes'] == artifact['max_bytes'], 'Diagnostic output byte declaration differs.') + # A dedicated diagnostic file cannot alias an existing required science file. + science = {item[key] for item in policy['field_outputs'] for key in ('prediction', 'reference')} + science.update(item['artifact'] for row in policy['probe_inputs'] for item in row['fields']) + science.update(('execution-context.json', 'model-evidence.json', 'model-definition.json', 'profile.json', + 'probe.json', 'metric-definitions.json', 'checkpoints.json', 'observations.json', + 'field-index.json', 'fitted-transform.json', 'output-policy.json')) + require(not set(files) & science, 'Diagnostic capture cannot replace scientific evidence.') + require(sum(item['max_bytes'] for item in outputs) <= limits['total_output_bytes'], + 'Declared diagnostic/scientific outputs cannot fit the admitted aggregate bound.', 'unsupported') + per_shard = natural(capture['records_per_shard'], 'Diagnostic records per shard', 1) + record_bytes = natural(capture['max_record_bytes'], 'Diagnostic record bytes', 1, limits['shard_bytes']) + # Exact worst-case envelope: run IDs and SHA-256 identities have fixed widths. + envelope = len(canonical(dict(format=SHARD_FORMAT, run_id='0' * 32, + capture_sha256='0' * 64, records=[]))) + 1 + require(all(envelope + per_shard * (record_bytes + 1) <= item['max_bytes'] + for item in capture['shards']), 'Diagnostic record declaration cannot fit each shard.', 'unsupported') + require(type(capture['coverage']) is list and capture['coverage'], 'Diagnostic coverage is required.') + scopes, total = _scopes(capture, policy) + require(total <= len(capture['shards']) * per_shard, 'Diagnostic population cannot fit declared shards.', 'unsupported') + # Refuse records whose mandatory identity envelope alone exceeds its bound. + for _start, _mandatory_end, _end, scope in scopes: + sample = dict(ordinal=total - 1, case_id=scope['case_id'], checkpoint_point=scope['checkpoint_point'], + checkpoint_sha256='0' * 64, phase=scope['phase'], payload={}) + require(len(canonical(sample)) <= record_bytes, 'Diagnostic record identity cannot fit its bound.', 'unsupported') + # Reserve even the uncompressed optional omission list. Compute its size + # arithmetically, so an impossible count never allocates that population. + worst_shards = [dict(blob=dict(name=item['name'], bytes=item['max_bytes'], sha256='0' * 64, media_type=MEDIA_TYPE), + record_count=per_shard, first_ordinal=total - 1, last_ordinal=total - 1) for item in capture['shards']] + coverage = _coverage(scopes, total, []) + optional = coverage['optional_count'] + coverage.update(omitted_count=optional, retained_optional_count=0) + worst_index = _index(capture, '0' * 32, '0' * 64, worst_shards, coverage) + omission_bytes = len(canonical(dict(start=total - 1, count=total, reason='capture_budget'))) + 1 + require(len(canonical(worst_index)) + optional * omission_bytes + 1 <= capture['index']['max_bytes'], + 'Diagnostic index cannot fit its declared bound.', 'unsupported') + return capture + + +def _omissions(scopes, total, omitted): + require(type(omitted) is list, 'Diagnostic omissions must be explicit ranges.') + end = 0 + for row in omitted: + closed(row, ('start', 'count', 'reason'), 'Diagnostic omission') + start = natural(row['start'], 'Omission start', 0, total) + count = natural(row['count'], 'Omission count', 1, total) + require(start >= end and start + count <= total, 'Omission ranges overlap or exceed the population.') + require(type(row['reason']) is str and row['reason'] in ('capture_budget', 'not_available', 'not_requested'), + 'Unsupported optional diagnostic omission reason.') + require(any(required_end <= start and start + count <= scope_end + for _begin, required_end, scope_end, _scope in scopes), + 'Mandatory diagnostic information cannot be omitted.') + end = start + count + return omitted + + +def _coverage(scopes, total, omitted): + required = sum(mandatory_end - start for start, mandatory_end, _end, _scope in scopes) + omitted_count = sum(row['count'] for row in omitted) + return dict(total_count=total, required_count=required, optional_count=total - required, + retained_required_count=required, retained_optional_count=total - required - omitted_count, + omitted_count=omitted_count, omitted=omitted) + + +def _index(capture, run_id, material_sha256, shards, coverage): + return dict(format=INDEX_FORMAT, payload_schema=PAYLOAD_SCHEMA, run_id=run_id, + material_sha256=material_sha256, capture_sha256=digest(capture), + shards=shards, total_shard_bytes=sum(item['blob']['bytes'] for item in shards), coverage=coverage) + + +def _records(capture, policy, records, omitted, checkpoints, checkpoint=lambda: None): + scopes, total = _scopes(capture, policy) + _omissions(scopes, total, omitted) + require(type(records) is list and len(records) == total - sum(row['count'] for row in omitted), + 'Diagnostic retained count differs from complete declared coverage.') + missing = {ordinal for row in omitted for ordinal in range(row['start'], row['start'] + row['count'])} + expected = (ordinal for ordinal in range(total) if ordinal not in missing) + scope_position = 0 + for ordinal, record in zip(expected, records, strict=True): + checkpoint() + closed(record, ('ordinal', 'case_id', 'checkpoint_point', 'checkpoint_sha256', 'phase', 'payload'), 'Diagnostic record') + while ordinal >= scopes[scope_position][2]: + scope_position += 1 + scope = scopes[scope_position][3] + require(type(record['ordinal']) is int and type(record['checkpoint_point']) is int and record['ordinal'] == ordinal and + record['case_id'] == scope['case_id'] and record['checkpoint_point'] == scope['checkpoint_point'] and + record['phase'] == scope['phase'], 'Diagnostic ordinal/case/checkpoint/phase differs.') + sha(record['checkpoint_sha256']) + require(record['checkpoint_sha256'] == checkpoints[scope['checkpoint_point']], 'Diagnostic checkpoint Ref differs.') + require(type(record['payload']) is dict, 'Diagnostic payload must be a finite JSON object.') + require(len(canonical(record)) <= capture['max_record_bytes'], 'Diagnostic record exceeds its admitted byte bound.') + return _coverage(scopes, total, omitted) + + +@boundary +def write(writer, records, omitted): + """Write exact bounded records; never sample/truncate mandatory information.""" + policy = writer.policy + require(policy.get('format') == POLICY_FORMAT, 'Indexed diagnostics require their explicit policy.', 'unsupported') + capture = policy['diagnostic_capture'] + checkpoints = {entry['clocks'][policy['profile']['capture']['clock']]: entry['checkpoint']['sha256'] for entry in writer.entries} + coverage = _records(capture, policy, records, omitted, checkpoints) + shards = []; documents = [] + for position in range(0, len(records), capture['records_per_shard']): + rows = records[position:position + capture['records_per_shard']] + artifact = capture['shards'][len(shards)] + filename = artifact['name'] + document = dict(format=SHARD_FORMAT, run_id=writer.context['run_id'], capture_sha256=digest(capture), records=rows) + require(len(canonical(document)) + 1 <= artifact['max_bytes'], 'Diagnostic shard exceeds its declared output bound.') + raw = canonical(document) + b'\n' + documents.append((filename, raw)) + blob = dict(name=filename, bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest(), media_type=MEDIA_TYPE) + shards.append(dict(blob=blob, record_count=len(rows), + first_ordinal=rows[0]['ordinal'], last_ordinal=rows[-1]['ordinal'])) + value = _index(capture, writer.context['run_id'], writer.context['composition']['sha256'], shards, coverage) + require(len(canonical(value)) + 1 <= capture['index']['max_bytes'], 'Diagnostic index exceeds its declared output bound.') + documents.append((capture['index']['name'], canonical(value) + b'\n')) + require(sum(item['bytes'] for item in writer.inventory()) + sum(len(raw) for _filename, raw in documents) + <= policy['profile']['limits']['total_output_bytes'], 'Diagnostic capture exceeds the admitted aggregate bound.') + for filename, raw in documents: + writer.write_raw(filename, raw) + return value + + +@boundary +def verify(capture, policy, payloads, context, checkpoint_index, checkpoint=lambda: None): + """Validate only host-retained inventory bytes, without filesystem discovery.""" + index = decode(payloads[capture['index']['name']]) + closed(index, ('format', 'payload_schema', 'run_id', 'material_sha256', 'capture_sha256', 'shards', 'total_shard_bytes', 'coverage'), 'Diagnostic index') + require(index['format'] == INDEX_FORMAT and index['payload_schema'] == PAYLOAD_SCHEMA and + index['run_id'] == context['run_id'] and index['material_sha256'] == context['composition']['sha256'] and + index['capture_sha256'] == digest(capture), 'Diagnostic index admitted identity differs.') + require(type(index['shards']) is list and len(index['shards']) <= len(capture['shards']), 'Diagnostic shard count differs.') + records = [] + for position, shard in enumerate(index['shards']): + checkpoint() + closed(shard, ('blob', 'record_count', 'first_ordinal', 'last_ordinal'), 'Diagnostic shard entry') + blob = shard['blob'] + closed(blob, ('name', 'bytes', 'sha256', 'media_type'), 'Diagnostic Blob') + filename = capture['shards'][position]['name'] + raw = payloads[filename] + require(blob == dict(name=filename, bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest(), media_type=MEDIA_TYPE), + 'Diagnostic shard differs from retained artifact identity/media type.') + document = decode(raw) + closed(document, ('format', 'run_id', 'capture_sha256', 'records'), 'Diagnostic shard') + require(document['format'] == SHARD_FORMAT and document['run_id'] == context['run_id'] and + document['capture_sha256'] == digest(capture), 'Diagnostic shard admitted identity differs.') + rows = document['records'] + require(type(rows) is list and 1 <= len(rows) <= capture['records_per_shard'], 'Diagnostic shard record count exceeds its bound.') + require(position == len(index['shards']) - 1 or len(rows) == capture['records_per_shard'], 'Diagnostic shard packing differs.') + require(type(shard['record_count']) is int and shard['record_count'] == len(rows) and + type(shard['first_ordinal']) is int and shard['first_ordinal'] == rows[0]['ordinal'] and + type(shard['last_ordinal']) is int and shard['last_ordinal'] == rows[-1]['ordinal'], 'Diagnostic shard count/range differs.') + records.extend(rows) + require({item['name'] for item in capture['shards']} & set(payloads) == {item['blob']['name'] for item in index['shards']}, + 'Retained diagnostic shards differ from the indexed set.') + natural(index['total_shard_bytes'], 'Diagnostic aggregate shard bytes') + require(index['total_shard_bytes'] == sum(item['blob']['bytes'] for item in index['shards']), + 'Diagnostic aggregate byte count differs.') + checkpoints = {entry['clocks'][policy['profile']['capture']['clock']]: entry['checkpoint']['sha256'] + for entry in checkpoint_index['entries']} + coverage = index['coverage'] + require(type(coverage) is dict and 'omitted' in coverage, 'Diagnostic coverage is required.') + expected = _records(capture, policy, records, coverage['omitted'], checkpoints, checkpoint) + require(canonical(coverage) == canonical(expected), 'Diagnostic aggregate coverage/count differs.') + return index diff --git a/tuldok_extension_sdk/evidence.py b/tuldok_extension_sdk/evidence.py index b6c12bd0..9137af87 100644 --- a/tuldok_extension_sdk/evidence.py +++ b/tuldok_extension_sdk/evidence.py @@ -292,6 +292,17 @@ def inventory(self): """Blobs actually consumed/written by this builder, not an export permission.""" return [copy.deepcopy(self._inventory[key]) for key in sorted(self._inventory)] + @boundary + def diagnostics(self, records, omitted): + """Capture an opted-in diagnostic index after complete scientific evidence. + + Callers supply every retained record and all optional omission ranges. + This does not infer numerical acceptance or repair invalid evidence. + """ + require(self._finished, 'Finish complete scientific evidence before diagnostics.') + from .diagnostics import write + return write(self, records, omitted) + @boundary def validate_projection(value, context_raw, blobs): From 653a6ed707e04935ad21d809f16c1fa5698ebe23 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 22:15:38 +0000 Subject: [PATCH 2/7] test: qualify indexed diagnostic publication recovery --- .../owner-roundtrip-final.log | 10 +++ .../owner-roundtrip-first.log | 35 ++++++++ .../owner-roundtrip-receipt.json | 83 +++++++++++++++++++ .../owner-roundtrip-second.log | 25 ++++++ .../pure-tests-final.log | 4 + .../reports/m1-indexed-diagnostics/report.md | 70 +++++++++++++++- .../m1-indexed-diagnostics/source-sha256.txt | 9 ++ tests/test_indexed_diagnostics.py | 12 +++ tests/test_indexed_diagnostics_publication.py | 33 ++++++-- 9 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-final.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-first.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-receipt.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-second.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests-final.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/source-sha256.txt diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-final.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-final.log new file mode 100644 index 00000000..23c23759 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-final.log @@ -0,0 +1,10 @@ +test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) ... ok +test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 16.580s + +OK +Removed exact recorded fresh schema: catalog_8e3d372a170f48a2944f0bd37e90fb95 +INDEXED_DIAGNOSTIC_QUALIFICATION {"accounting_and_events_unchanged": true, "artifacts": [{"bytes": 948, "media_type": "application/octet-stream", "name": "diagnostic-index.json", "role": "diagnostic_index_v1", "sha256": "aa3b3b1046dd25c173914e9c8264a99825f50b8cc5fca7017aefd4c9592b21f7"}, {"bytes": 983, "media_type": "application/octet-stream", "name": "diagnostic-0.json", "role": "diagnostic_shard_v1", "sha256": "cc944ee001168eb8be360d65eacd0185155ab04815c2d953d612038bc93ff51b"}, {"bytes": 443, "media_type": "application/octet-stream", "name": "diagnostic-1.json", "role": "diagnostic_shard_v1", "sha256": "24d33075cc30512a60e018dbc0a068a17cac3f299035d9bf0a3d59d396b1e076"}], "captured_python_producer_executed": false, "counts_after": {"access_execution_consumptions": 1, "access_execution_grants": 1, "model_attempt_reservations": 1, "workspace_graphs": 1, "workspace_runs": 1}, "counts_before": {"access_execution_consumptions": 1, "access_execution_grants": 1, "model_attempt_reservations": 1, "workspace_graphs": 1, "workspace_runs": 1}, "coverage": {"omitted": [{"count": 1, "reason": "capture_budget", "start": 2}], "omitted_count": 1, "optional_count": 3, "required_count": 2, "retained_optional_count": 2, "retained_required_count": 2, "total_count": 5}, "custody_outcome": "completed", "numerical_acceptance": "not_evaluated", "original_state": "interrupted", "result_ref": {"catalog_id": "9173b8d7d0114555ab6798fb4ff4c523", "custody_receipt_sha256": "dc9b18287cbf3fa0d1b9349210c6591b626a307520d703d1458c87e20fcced81", "inventory_sha256": "e539861befdf104b58d74d37c492f41cc33948d190fe8b47a17fb2176cd19a41", "kind": "recovered_publication_result", "material_sha256": "7f90fbc249f1fa641867c9e3318e1527dd7e75290a8e3d1072a989897bac87b8", "run_id": "98d665737bdf44bfbb3fa0af44efa61b", "schema_version": 1, "sha256": "b3c7859fe9f8bad7288adc09bb66b996a769c8545770dbd9389a1fb2c5184143"}, "run_id": "98d665737bdf44bfbb3fa0af44efa61b", "runtime_packages": {"numpy": "0.0", "torch": "0.0"}, "runtime_snapshot_sha256": "cd69d4b1de535e6c59efc66b6b376991e7c9afdc2d3e7cefcc8f0ea18523d164", "schema": "catalog_9173b8d7d0114555ab6798fb4ff4c523", "scope": "harness SDK serialization; actual no-op metadata runtime custody and semantic recovery", "training_executed": false, "verification_scope": "source_destination_and_cold_exact_ref_reader"} +Removed exact recorded fresh schema: catalog_9173b8d7d0114555ab6798fb4ff4c523 diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-first.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-first.log new file mode 100644 index 00000000..307d13c1 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-first.log @@ -0,0 +1,35 @@ +test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) ... ERROR +test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay) ... ERROR + +====================================================================== +ERROR: test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/tests/test_indexed_diagnostics_publication.py", line 74, in setUp + self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/app.py", line 158, in __init__ + self.projects = projects.Projects(self, project_roots) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/projects.py", line 126, in __init__ + raise WorkbenchError('Cannot open configured project roots; check aliases and local directories.') from None +workbench.WorkbenchError: Cannot open configured project roots; check aliases and local directories. + +====================================================================== +ERROR: test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/tests/test_indexed_diagnostics_publication.py", line 74, in setUp + self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/app.py", line 158, in __init__ + self.projects = projects.Projects(self, project_roots) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/projects.py", line 126, in __init__ + raise WorkbenchError('Cannot open configured project roots; check aliases and local directories.') from None +workbench.WorkbenchError: Cannot open configured project roots; check aliases and local directories. + +---------------------------------------------------------------------- +Ran 2 tests in 1.744s + +FAILED (errors=2) diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-receipt.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-receipt.json new file mode 100644 index 00000000..f4dea27b --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-receipt.json @@ -0,0 +1,83 @@ +{ + "accounting_and_events_unchanged": true, + "artifacts": [ + { + "bytes": 948, + "media_type": "application/octet-stream", + "name": "diagnostic-index.json", + "role": "diagnostic_index_v1", + "sha256": "aa3b3b1046dd25c173914e9c8264a99825f50b8cc5fca7017aefd4c9592b21f7" + }, + { + "bytes": 983, + "media_type": "application/octet-stream", + "name": "diagnostic-0.json", + "role": "diagnostic_shard_v1", + "sha256": "cc944ee001168eb8be360d65eacd0185155ab04815c2d953d612038bc93ff51b" + }, + { + "bytes": 443, + "media_type": "application/octet-stream", + "name": "diagnostic-1.json", + "role": "diagnostic_shard_v1", + "sha256": "24d33075cc30512a60e018dbc0a068a17cac3f299035d9bf0a3d59d396b1e076" + } + ], + "captured_python_producer_executed": false, + "counts_after": { + "access_execution_consumptions": 1, + "access_execution_grants": 1, + "model_attempt_reservations": 1, + "workspace_graphs": 1, + "workspace_runs": 1 + }, + "counts_before": { + "access_execution_consumptions": 1, + "access_execution_grants": 1, + "model_attempt_reservations": 1, + "workspace_graphs": 1, + "workspace_runs": 1 + }, + "coverage": { + "omitted": [ + { + "count": 1, + "reason": "capture_budget", + "start": 2 + } + ], + "omitted_count": 1, + "optional_count": 3, + "required_count": 2, + "retained_optional_count": 2, + "retained_required_count": 2, + "total_count": 5 + }, + "custody_outcome": "completed", + "exact_schema_removed": true, + "numerical_acceptance": "not_evaluated", + "original_state": "interrupted", + "result_ref": { + "catalog_id": "9173b8d7d0114555ab6798fb4ff4c523", + "custody_receipt_sha256": "dc9b18287cbf3fa0d1b9349210c6591b626a307520d703d1458c87e20fcced81", + "inventory_sha256": "e539861befdf104b58d74d37c492f41cc33948d190fe8b47a17fb2176cd19a41", + "kind": "recovered_publication_result", + "material_sha256": "7f90fbc249f1fa641867c9e3318e1527dd7e75290a8e3d1072a989897bac87b8", + "run_id": "98d665737bdf44bfbb3fa0af44efa61b", + "schema_version": 1, + "sha256": "b3c7859fe9f8bad7288adc09bb66b996a769c8545770dbd9389a1fb2c5184143" + }, + "run_id": "98d665737bdf44bfbb3fa0af44efa61b", + "runtime": "Python 3.12.14 harness; captured /usr/bin/true with torch/numpy 0.0 metadata", + "runtime_packages": { + "numpy": "0.0", + "torch": "0.0" + }, + "runtime_snapshot_sha256": "cd69d4b1de535e6c59efc66b6b376991e7c9afdc2d3e7cefcc8f0ea18523d164", + "schema": "catalog_9173b8d7d0114555ab6798fb4ff4c523", + "scope": "harness SDK serialization; actual no-op metadata runtime custody and semantic recovery", + "source_composed_head": "f052fa7d93ee2df35f73afd28b565e1bac42183b", + "state": "passed", + "training_executed": false, + "verification_scope": "source_destination_and_cold_exact_ref_reader" +} diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-second.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-second.log new file mode 100644 index 00000000..98ab1657 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/owner-roundtrip-second.log @@ -0,0 +1,25 @@ +test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) ... ERROR +test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay) ... ok + +====================================================================== +ERROR: test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/tests/test_indexed_diagnostics_publication.py", line 160, in test_impossible_population_refuses_before_graph_run_grant_or_reservation + self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/access.py", line 73, in guarded + return function(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/projects.py", line 230, in open_snapshot + return self._open(body, True) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/workspace/Tuldok-lab-indexed-diagnostics-m1/projects.py", line 258, in _open + raise WorkbenchError('This project ID already names a different snapshot or root. Keep the original or use a new project ID.', 'conflict', 409) +workbench.WorkbenchError: This project ID already names a different snapshot or root. Keep the original or use a new project ID. + +---------------------------------------------------------------------- +Ran 2 tests in 17.778s + +FAILED (errors=1) +Removed exact recorded fresh schema: catalog_bcd1d0b17e6340cdbccea964073b7d43 +Removed exact recorded fresh schema: catalog_9821a7d198e846369a511147fe6a982f diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests-final.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests-final.log new file mode 100644 index 00000000..00c832f9 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/pure-tests-final.log @@ -0,0 +1,4 @@ +---------------------------------------------------------------------- +Ran 54 tests in 1.786s + +OK (skipped=2) diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md index 62fae0a9..a151f64d 100644 --- a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/report.md @@ -42,8 +42,7 @@ public contract validation. One intermediate test-only failure used an invalid `selectors` argument for `runs.model_evidence/2`; the test was corrected to the existing flat request grammar. No production contract was changed to pass it. -`git diff --check` passed. Actual PostgreSQL/custodian qualification is pending -the coordinated benchmark quiet-window release. Its executable fixture is +`git diff --check` passed. The executable PostgreSQL/custodian fixture is `tests/test_indexed_diagnostics_publication.py`: a fresh tiny metadata runtime (`/usr/bin/true`, dummy 0.0 packages), captured SDK source serialized by the harness before launch, real no-op child/custodian, injected publication copy @@ -54,3 +53,70 @@ failure bytes/path remain unavailable, so PUB-1 classification is blocked. No captured Python producer execution, training, CUDA, physical operator, scientific criterion or numerical acceptance is claimed. Full M1/T1/T2 acceptance remains the root integration owner's decision after review and qualification. + +## Completed owner qualification + +After M4's quiet-window release, the complete latest PR104 source +`1c9f01cb7e4adafd169dd56e95e02f0f5025b35d` was merged normally with the initial +implementation commit `28f61e8`, producing composed source +`f052fa7d93ee2df35f73afd28b565e1bac42183b`. Production diagnostic source did not +change during the qualifier fixture corrections. + +`owner-roundtrip-final.log`: both actual PostgreSQL tests passed in 16.580 s. +Command (using the explicitly nominated private task-cluster DSN): + +```sh +TULDOK_DATABASE_URL='' PYTHONPATH=tests:. \ +PATH=/workspace/tuldok-runtime/bin:$PATH /workspace/tuldok-runtime/bin/python \ + -m unittest test_indexed_diagnostics_publication -v +``` + +The negative test refuses an impossible population before graph creation, +process launch, Run, grant consumption or reservation. The positive test uses +actual Project/Model/Study/grant admission, harness SDK serialization, a real +drained/completed no-op child, frozen inventory and an injected output copy +failure. After reopen, real source, destination and cold exact-Ref semantic +verification pass. The original Run remains interrupted. Recovery is idempotent +under the same Ref; Run/grant/consumption/reservation counts, the reservation +document, conservative Study charge and original events remain unchanged. +An incorrect shard hash refuses. Public publication evidence still uses the +original closed projection. + +`owner-roundtrip-receipt.json` retains actual result identity, exact index/shard +inventory hashes, coverage and accounting facts. The index is 948 bytes, shards +are 983 and 443 bytes, and all five declared ordinal positions are accounted for: +two mandatory and two optional records retained, one optional record explicitly +omitted for `capture_budget`. The successful Run was +`98d665737bdf44bfbb3fa0af44efa61b` in the fresh +`catalog_9173b8d7d0114555ab6798fb4ff4c523` schema. That exact recorded schema and +the negative-test schema `catalog_8e3d372a170f48a2944f0bd37e90fb95` were removed +after closing their catalog owners; the log records both removals. + +`pure-tests-final.log`: 54 tests run, 2 expected database-dependent skips, +remaining tests passed. This adds a valid bound recipe addon/field-target and +host-closure witness to the initial group. No production source changed after +the successful owner qualification. `source-sha256.txt` records the final +production/test/contract file identities. + +## Retained failed attempts and cleanup boundary + +`owner-roundtrip-first.log` records two fixture setup errors: the configured +Project directory did not exist before Dataset construction. Dataset closed its +connection, but exact fresh schema identities were not registered before the +temporary manifests were removed. A read-only `pg_namespace` query afterward +observed newest candidate names +`catalog_9b3d1d36c0a2473d9a4e6a7a416653ec` (`xmin=9145`) and +`catalog_b423b373d5424501af8802943b4f7893` (`xmin=9146`), following an older +`xmin=9103` namespace. Temporal adjacency does not prove ownership. These +candidates were not opened, read or dropped. Cleanup is **not attempted and +blocked on exact ownership evidence**; root explicitly preserved this boundary. +No uncertain cleanup action was retried. + +The corrected fixture records a fresh schema through Metadata before constructing +Dataset and registers exact cleanup immediately. `owner-roundtrip-second.log` +records a passing positive round trip plus a negative fixture error at the +existing immutable Project identity guard. It was corrected using a new +explicit Project ID. Both exact recorded second-attempt schemas were removed: +`catalog_bcd1d0b17e6340cdbccea964073b7d43` and +`catalog_9821a7d198e846369a511147fe6a982f`. The final run above passed both +tests. These were fixture mistakes; no production rule was weakened. diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/source-sha256.txt b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/source-sha256.txt new file mode 100644 index 00000000..0e211196 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/source-sha256.txt @@ -0,0 +1,9 @@ +3a05ac3e4e005b4b2943594bf82b7f97afb5eaa853572b5fe296d3349c0515e6 extension_evidence.py +dc57a657dd59cee5297cbdaf2426a2d147424a1e737cc0dad799b6084b873646 recipe_contract.py +b7e0b3167de54dd1ce8bf5209af92ee0bcab9ba6eb6fc9f8c7db4e3fea96fc18 tuldok_extension_sdk/__init__.py +53027c44b78fde2db62e736d9b8ed66915b17bfca3fefe36eee30cae59af0a70 tuldok_extension_sdk/evidence.py +d2439f8ab20dc7a3b829b547745fd7fa84d320ac4df2930b600cd36014689699 tuldok_extension_sdk/diagnostics.py +af764dc4c1dbb1e7127293968adabbbcf7d77b80ba843a2b51d50a882785a439 tests/test_indexed_diagnostics.py +12a6b3c5a75c741a7d4c7b7bbf58e53ed3f554e6c1f39f8aca59a0a3700d8800 tests/test_indexed_diagnostics_publication.py +7504ff0bc36c6906b465d7643eb310596f7d737fbec9412b75d196d70203404b docs/contracts/indexed-diagnostics.md +4330ea53433ed7cf8c2a44e2d4feef7e7114e8d9f613893441ae8349c277b0d2 docs/contracts/run-preflight-publication.md diff --git a/tests/test_indexed_diagnostics.py b/tests/test_indexed_diagnostics.py index 841241a8..89a22a37 100644 --- a/tests/test_indexed_diagnostics.py +++ b/tests/test_indexed_diagnostics.py @@ -122,6 +122,18 @@ def test_addon_requires_explicit_tag_and_exact_output_roles(self): change() with self.assertRaises(WorkbenchError): self.decode() + def test_bound_recipe_selects_the_same_addon_with_its_existing_target_contract(self): + self.definition = json.loads((Path(__file__).resolve().parents[1] / + 'examples/extensions/field-surrogate-b-independent/v1/recipe.json').read_bytes()) + self.capture = opt_in(self.definition) + self.capture['coverage'][0]['checkpoint_point'] = 32 + self.capture['coverage'][1]['checkpoint_point'] = 64 + self.assertEqual(self.decode(), self.definition) + self.assertTrue(recipe_contract.is_bound_evidence(self.definition)) + self.assertIn('field_targets', self.definition['model_evidence']) + self.assertEqual(set(recipe_contract.recipe_host_identity(self.definition)), + set(recipe_contract.BOUND_EVIDENCE_HOST_NAMES) | set(recipe_contract.DIAGNOSTIC_HOST_NAMES)) + class DiagnosticRoundTripTests(unittest.TestCase): def setUp(self): diff --git a/tests/test_indexed_diagnostics_publication.py b/tests/test_indexed_diagnostics_publication.py index f33fb1a5..da2d842f 100644 --- a/tests/test_indexed_diagnostics_publication.py +++ b/tests/test_indexed_diagnostics_publication.py @@ -17,6 +17,7 @@ import sys import tempfile import unittest +import uuid from unittest.mock import patch from workbench import WorkbenchError @@ -68,12 +69,17 @@ class IndexedDiagnosticPublicationTests(unittest.TestCase): def setUp(self): from app import Dataset from auth_helpers import ExplicitOwnerFixture + from metadata import Metadata from runtime_snapshots import RuntimeSnapshots self.temp = tempfile.TemporaryDirectory(prefix='indexed-diagnostics-'); self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) - self.schema = self.dataset.db.schema + (self.root / 'project').mkdir() + metadata = Metadata(self.root / 'catalog') + self.schema = metadata.schema + metadata.close() + self.dataset = None self.addCleanup(self.cleanup_catalog) + self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) self.owner = ExplicitOwnerFixture.bootstrap(self.dataset) python = self.root / 'python'; site = self.root / 'packages' (python / 'bin').mkdir(parents=True); (python / 'lib/python3.12/lib-dynload').mkdir(parents=True) @@ -91,15 +97,21 @@ def setUp(self): snapshots = RuntimeSnapshots(self.dataset.db, self.dataset.access, self.dataset.path) identity = snapshots.inspect(config)['inventory_sha256'] self.runtime = snapshots.create(config, identity, quiesced=True, reserve_bytes=0) - self.definition, self.manifest, self.recipe_sha = project_fixture(self.root / 'project', identity) + # The directory must predate Dataset's configured-root descriptor. + # The established fixture builder creates its own new directory. + self.definition, self.manifest, self.recipe_sha = project_fixture(self.root / 'prepared-project', identity) + for source in (self.root / 'prepared-project').iterdir(): + shutil.move(str(source), self.root / 'project' / source.name) + (self.root / 'prepared-project').rmdir() self.project = self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) def cleanup_catalog(self): import psycopg from psycopg import sql - self.dataset.close() + if self.dataset is not None: self.dataset.close() with psycopg.connect(os.environ['TULDOK_DATABASE_URL'], autocommit=True) as connection: connection.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(self.schema))) + print('Removed exact recorded fresh schema: ' + self.schema) def graph_body(self): return dict(name='Indexed synthetic diagnostics; metadata runtime only', project_id=self.project['id'], @@ -143,10 +155,11 @@ def test_impossible_population_refuses_before_graph_run_grant_or_reservation(sel self.definition['model_evidence']['diagnostic_capture']['coverage'][0]['required_count'] = 7 raw = extension_evidence.canonical(self.definition); (self.root / 'project/recipe.json').write_bytes(raw) self.recipe_sha = hashlib.sha256(raw).hexdigest() + self.manifest['id'] = uuid.uuid4().hex self.manifest['recipes'][0]['sha256'] = self.recipe_sha (self.root / 'project/project.json').write_bytes(extension_evidence.canonical(self.manifest)) with self.owner.bound(self.dataset): - self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) + self.project = self.dataset.projects.open_snapshot({'root': 'synthetic', 'manifest_path': 'project.json'}) before = self.counts() with patch('subprocess.Popen', side_effect=AssertionError('Invalid declaration launched compute')): with self.assertRaises(WorkbenchError) as caught: self.dataset.pipelines.create_external(self.graph_body()) @@ -215,3 +228,13 @@ def semantic(directory, value, **kwargs): self.assertEqual(self.dataset.runs._load(run['id'])['events'], events) self.assertTrue(any('.external-' in path for path in checked), checked) self.assertTrue(any('.external-' not in path for path in checked), checked) + print('INDEXED_DIAGNOSTIC_QUALIFICATION ' + json.dumps(dict( + scope='harness SDK serialization; actual no-op metadata runtime custody and semantic recovery', + captured_python_producer_executed=False, training_executed=False, numerical_acceptance='not_evaluated', + schema=self.schema, run_id=run['id'], runtime_snapshot_sha256=self.runtime['manifest_sha256'], + runtime_packages=self.runtime['packages'], custody_outcome=custody['outcome'], + result_ref=recovered['result_ref'], artifacts=[indexed[capture['name']] for capture in + [self.definition['model_evidence']['diagnostic_capture']['index'], + *self.definition['model_evidence']['diagnostic_capture']['shards']]], coverage=index['coverage'], + verification_scope='source_destination_and_cold_exact_ref_reader', counts_before=before_counts, + counts_after=self.counts(), accounting_and_events_unchanged=True, original_state=saved['state']), sort_keys=True)) From 49559c8426232f963ecafd76d02b01e51d052431 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 23:00:54 +0000 Subject: [PATCH 3/7] test: exercise indexed evidence through composed batch and scientific readers --- tests/test_indexed_diagnostics_publication.py | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/test_indexed_diagnostics_publication.py b/tests/test_indexed_diagnostics_publication.py index da2d842f..1e499a0d 100644 --- a/tests/test_indexed_diagnostics_publication.py +++ b/tests/test_indexed_diagnostics_publication.py @@ -122,6 +122,62 @@ def counts(self): return {table: self.dataset.db.execute('SELECT count(*) FROM ' + table).fetchone()[0] for table in ('workspace_graphs', 'workspace_runs', 'access_execution_grants', 'access_execution_consumptions', 'model_attempt_reservations')} + def check_composed_readers(self, ref, public, inventory): + """Read the same admitted indexed evidence through composed public clients.""" + from http.server import ThreadingHTTPServer + import threading + import agent_api + from app import make_handler + from extension_evidence import digest + from tuldok_client import WorkflowClient + agent = self.dataset.access.issue({'label': 'Indexed composition inspection', 'scopes': ['inspect']}) + server = ThreadingHTTPServer(('127.0.0.1', 0), make_handler(self.dataset)) + thread = threading.Thread(target=server.serve_forever); thread.start() + try: + client = agent_api.AgentClient('http://127.0.0.1:' + str(server.server_port), + agent['token'], self.dataset.db.dataset_id, timeout=60) + workflow = WorkflowClient(client) + capture = self.definition['model_evidence']['diagnostic_capture'] + names = [capture['index']['name'], *[row['name'] for row in capture['shards']]] + manifest = workflow.collection_manifest(names=names, result_ref=ref) + output = self.root / 'composed-indexed-collection' + with patch.object(client, 'call', wraps=client.call) as dispatch: + collected = workflow.collect_batch(manifest, output) + operations = [call.args[0] for call in dispatch.call_args_list] + self.assertEqual(operations.count('runs.artifact_segments'), 1) + self.assertNotIn('runs.artifact_chunk', operations) + self.assertNotIn('runs.publication_artifact_chunk', operations) + self.assertEqual(collected['state'], 'complete', collected) + self.assertEqual({row['name'] for row in collected['items']}, set(names)) + self.assertEqual({path.name for path in output.iterdir()}, set(names)) + for name in names: + raw = (output / name).read_bytes() + self.assertEqual((len(raw), hashlib.sha256(raw).hexdigest()), + (inventory[name]['bytes'], inventory[name]['sha256'])) + self.assertEqual((output / name).stat().st_mode & 0o777, 0o600) + selection = {'metric_id': public['metric_definitions'][0]['id'], 'mode': 'paired'} + for prefix in ('', 'other_'): + selection.update({ + prefix + 'checkpoint_sha256': public['checkpoint_index']['entries'][0]['checkpoint']['sha256'], + prefix + 'evaluation_sha256': inventory['model-evidence.json']['sha256'], + prefix + 'metric_definition_sha256': digest(public['metric_definitions'][0]), + }) + operand = {'kind': 'publication_result', 'ref': ref} + arguments = {'operands': [copy.deepcopy(operand), copy.deepcopy(operand)], + 'selection': selection, + 'point': {'field_id': public['fields'][0]['field_id'], 'x': 1, 'y': 2, 'z': 3}} + compared = client.call('runs.compare_scientific_evidence', arguments) + self.assertEqual(compared['status'], 200, compared) + self.assertEqual([row['operand'] for row in compared['result']['operands']], arguments['operands']) + self.assertEqual(compared['result']['point'], arguments['point']) + self.assertEqual(compared['result']['cases'][0]['prediction_delta']['value'], 0) + self.assertEqual(compared['result']['criterion']['state'], 'unknown') + return {'batch_collected_exact_artifacts': names, 'batch_calls': 1, + 'individual_segment_calls': 0, 'comparison': 'same exact Ref diagnostic delta zero; no scientific acceptance'} + finally: + server.shutdown(); server.server_close(); thread.join(5) + self.assertFalse(thread.is_alive()) + def admit(self): import extension_evidence dataset = self.dataset; policy = self.definition['model_evidence'] @@ -218,6 +274,7 @@ def semantic(directory, value, **kwargs): public = self.dataset.runs.publication_model_evidence(recovered['result_ref'], {})['evidence'] self.assertEqual(public['state'], 'verified'); self.assertNotIn('_diagnostics', public) self.assertNotIn('diagnostics', public) + composed_readers = self.check_composed_readers(recovered['result_ref'], public, indexed) with self.assertRaises(WorkbenchError): self.dataset.runs.publication_artifact(recovered['result_ref'], 'diagnostic-0.json', '0' * 64) self.assertEqual(self.dataset.runs.recover_publication(body), recovered) @@ -236,5 +293,5 @@ def semantic(directory, value, **kwargs): result_ref=recovered['result_ref'], artifacts=[indexed[capture['name']] for capture in [self.definition['model_evidence']['diagnostic_capture']['index'], *self.definition['model_evidence']['diagnostic_capture']['shards']]], coverage=index['coverage'], - verification_scope='source_destination_and_cold_exact_ref_reader', counts_before=before_counts, + verification_scope='source_destination_and_cold_exact_ref_reader', composed_readers=composed_readers, counts_before=before_counts, counts_after=self.counts(), accounting_and_events_unchanged=True, original_state=saved['state']), sort_keys=True)) From 70dcaeb0382a97d051275319881acf589bfb931d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 23:01:28 +0000 Subject: [PATCH 4/7] test: create the explicit private collection destination --- tests/test_indexed_diagnostics_publication.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_indexed_diagnostics_publication.py b/tests/test_indexed_diagnostics_publication.py index 1e499a0d..26e70a53 100644 --- a/tests/test_indexed_diagnostics_publication.py +++ b/tests/test_indexed_diagnostics_publication.py @@ -141,6 +141,7 @@ def check_composed_readers(self, ref, public, inventory): names = [capture['index']['name'], *[row['name'] for row in capture['shards']]] manifest = workflow.collection_manifest(names=names, result_ref=ref) output = self.root / 'composed-indexed-collection' + output.mkdir(mode=0o700) with patch.object(client, 'call', wraps=client.call) as dispatch: collected = workflow.collect_batch(manifest, output) operations = [call.args[0] for call in dispatch.call_args_list] From 9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 23:02:35 +0000 Subject: [PATCH 5/7] test: register exact disposable schema before construction --- tests/test_indexed_diagnostics_publication.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_indexed_diagnostics_publication.py b/tests/test_indexed_diagnostics_publication.py index 26e70a53..b65cef2c 100644 --- a/tests/test_indexed_diagnostics_publication.py +++ b/tests/test_indexed_diagnostics_publication.py @@ -74,11 +74,14 @@ def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix='indexed-diagnostics-'); self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) (self.root / 'project').mkdir() - metadata = Metadata(self.root / 'catalog') - self.schema = metadata.schema - metadata.close() + identity = uuid.uuid4().hex + self.schema = 'catalog_' + identity self.dataset = None self.addCleanup(self.cleanup_catalog) + print('Reserved exact fresh schema before construction: ' + self.schema, flush=True) + metadata = Metadata(self.root / 'catalog', dataset_id=identity) + self.assertEqual(metadata.schema, self.schema) + metadata.close() self.dataset = Dataset(self.root / 'catalog', project_roots={'synthetic': self.root / 'project'}) self.owner = ExplicitOwnerFixture.bootstrap(self.dataset) python = self.root / 'python'; site = self.root / 'packages' @@ -110,7 +113,7 @@ def cleanup_catalog(self): from psycopg import sql if self.dataset is not None: self.dataset.close() with psycopg.connect(os.environ['TULDOK_DATABASE_URL'], autocommit=True) as connection: - connection.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(self.schema))) + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(self.schema))) print('Removed exact recorded fresh schema: ' + self.schema) def graph_body(self): From 7e287342332d6a918671b5ed79b9725fdb916c86 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 19 Oct 2026 23:15:55 +0000 Subject: [PATCH 6/7] test: retain actual indexed reader and collection composition evidence --- ...xed-final-9d563ceb-independent-review.json | 32 +++++ .../composed-readers/qualification.md | 9 ++ .../tuldok-indexed-composed-actual.log | 12 ++ ...tuldok-indexed-composed-qualification.json | 130 ++++++++++++++++++ ...ed-readers-70dcaeb-independent-review.json | 47 +++++++ .../tuldok-indexed-composed-source.json | 17 +++ ...k-refined-indexed-cleanup-independent.json | 22 +++ .../composed-readers/validation.json | 41 ++++++ 8 files changed, 310 insertions(+) create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/indexed-final-9d563ceb-independent-review.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/qualification.md create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-actual.log create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-qualification.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-readers-70dcaeb-independent-review.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-source.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-refined-indexed-cleanup-independent.json create mode 100644 docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/validation.json diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/indexed-final-9d563ceb-independent-review.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/indexed-final-9d563ceb-independent-review.json new file mode 100644 index 00000000..f4fa8233 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/indexed-final-9d563ceb-independent-review.json @@ -0,0 +1,32 @@ +{ + "checks_run": [ + "Read-only frozen source/Metadata inspection", + "Built-in compile and stdlib AST/order checks only" + ], + "format": "indexed_final_fixture_ownership_independent_review_v1", + "limits": [ + "No imports/application/DB/heavy test executed during M4 quiet window.", + "Actual2 qualification on9d is still root-owned and pending quiet release.", + "Flushed stdout ledger is runner provenance, not a power-loss durability claim." + ], + "metadata_source_sha256": "893d489f8e5d323ac1eed61350d206bbc1c0375801402f52cc586ac8423d62e4", + "prior_review": { + "receipt_path": "/tmp/tuldok-indexed-composed-readers-70dcaeb-independent-review.json", + "receipt_sha256": "be25642992e49f5a0d89cb5a34a7185470a5ce321443103bf4597ed81c577a04", + "source": "70dcaeb0382a97d051275319881acf589bfb931d" + }, + "result": "Clear: exact fresh schema ownership is registered and flushed before Metadata allocation; original composed helper/accounting review remains applicable", + "reviewed_at_utc": "2026-10-19T23:03:47.661032+00:00", + "reviewed_file": "tests/test_indexed_diagnostics_publication.py", + "reviewed_file_sha256": "7e14d96a4d2a5d6e2be5aa884770a8b57a32653e009c890f68291f125551fa11", + "reviewed_source": "9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3", + "verified": [ + "UUID identity and catalog_ schema assigned before construction; self.dataset=None and catalog cleanup registered before Metadata(...dataset_id=identity).", + "Ledger print uses flush=True before constructor; runner can attribute a later constructor failure to this exact fresh schema, without discovering or guessing older schema identities.", + "Metadata honors explicit dataset_id, validates UUID identity and derives exact catalog_ schema. Current owner implementation closes its connection on initialization exception.", + "Schema equality asserted and initial Metadata closed before Dataset reopen. Registered cleanup closes self.dataset when present and uses parameterized SQL Identifier for only self.schema.", + "DROP SCHEMA IF EXISTS handles failure before schema creation; cleanup does not enumerate, infer, adopt or delete older unknown schemas.", + "Cleanup registration follows temporary-directory cleanup registration, so unittest LIFO closes/drops catalog before removing its temporary catalog directory.", + "AST equality proves all methods except fixture setUp/cleanup_catalog unchanged from70; composed batch/scientific helper, no-replay and accounting assertions preserved." + ] +} diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/qualification.md b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/qualification.md new file mode 100644 index 00000000..3e88171c --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/qualification.md @@ -0,0 +1,9 @@ +# Composed indexed evidence qualification + +At exact source `9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3`, both actual PostgreSQL/custodian cases passed in 22.431 seconds with no skips. The original captured-SDK serialization, actual no-op child, copy interruption, cold recovery, mandatory/optional coverage, exact artifacts and accounting/event conservation assertions remain. The source manifest records the final production and test file hashes. + +The recovered indexed result was additionally inspected through a real cold inspect AgentClient: WorkflowClient collected its original index and two shards through one batch call, with no individual segment fallback, matching whole SHA256 and private 0600 files. The scientific comparison operation read the same exact Ref twice with exact point/operand correlation, diagnostic delta zero and unknown criterion. This tests composition; it does not compare different models or establish scientific acceptance. Serialization remains harness-authored before an actual /usr/bin/true no-op child, with dummy runtime packages. No captured Python producer execution, training or CUDA is claimed. + +Independent source review found a missing private destination directory before execution; commit70dcaeb fixed it. The final fixture at9d563ce additionally reserves and logs its exact fresh schema before Metadata construction. Both pre-registered schemas were independently queried absent after owner cleanup. The shared cleanup receipt also clearly labels two separate refined-performance schemas; those are not indexed test fixtures. Older unowned schema candidates remain untouched. + +Hosted lifecycle/browser acceptance and serial integration remain separate gates. The original lab diagnostic failure path/bytes and scientific criteria remain missing inputs. diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-actual.log b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-actual.log new file mode 100644 index 00000000..7836de36 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-actual.log @@ -0,0 +1,12 @@ +test_impossible_population_refuses_before_graph_run_grant_or_reservation (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_impossible_population_refuses_before_graph_run_grant_or_reservation) ... Reserved exact fresh schema before construction: catalog_b8674cd6642d40c59051fda95a7cb9b5 +ok +test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay (test_indexed_diagnostics_publication.IndexedDiagnosticPublicationTests.test_real_custody_cold_recovery_and_exact_inventory_read_with_no_replay) ... Removed exact recorded fresh schema: catalog_b8674cd6642d40c59051fda95a7cb9b5 +Reserved exact fresh schema before construction: catalog_fc607ffa4125402994f62c319f632faa +ok + +---------------------------------------------------------------------- +Ran 2 tests in 22.431s + +OK +INDEXED_DIAGNOSTIC_QUALIFICATION {"accounting_and_events_unchanged": true, "artifacts": [{"bytes": 948, "media_type": "application/octet-stream", "name": "diagnostic-index.json", "role": "diagnostic_index_v1", "sha256": "351f54de24ac9824a69a641674b1d63246bc2541c2d40f43bfc19bcbd94419d3"}, {"bytes": 983, "media_type": "application/octet-stream", "name": "diagnostic-0.json", "role": "diagnostic_shard_v1", "sha256": "bb0f9f8361a1380c02dfad134f2352e40f40da8dfa1b79347c0b3a47aba90543"}, {"bytes": 443, "media_type": "application/octet-stream", "name": "diagnostic-1.json", "role": "diagnostic_shard_v1", "sha256": "b98746f69577f240ee39204f4b6030b91c9e46b1a674d4164afb52afc06ef133"}], "captured_python_producer_executed": false, "composed_readers": {"batch_calls": 1, "batch_collected_exact_artifacts": ["diagnostic-index.json", "diagnostic-0.json", "diagnostic-1.json"], "comparison": "same exact Ref diagnostic delta zero; no scientific acceptance", "individual_segment_calls": 0}, "counts_after": {"access_execution_consumptions": 1, "access_execution_grants": 1, "model_attempt_reservations": 1, "workspace_graphs": 1, "workspace_runs": 1}, "counts_before": {"access_execution_consumptions": 1, "access_execution_grants": 1, "model_attempt_reservations": 1, "workspace_graphs": 1, "workspace_runs": 1}, "coverage": {"omitted": [{"count": 1, "reason": "capture_budget", "start": 2}], "omitted_count": 1, "optional_count": 3, "required_count": 2, "retained_optional_count": 2, "retained_required_count": 2, "total_count": 5}, "custody_outcome": "completed", "numerical_acceptance": "not_evaluated", "original_state": "interrupted", "result_ref": {"catalog_id": "fc607ffa4125402994f62c319f632faa", "custody_receipt_sha256": "66d76987a01903d9bcfdb06a991d411b729064c0a2a29461ae62d295e5c4edec", "inventory_sha256": "552f458cfd5e606c3f306bf4878946636e2e78bf6a53c6f648074a9aba7d9a1b", "kind": "recovered_publication_result", "material_sha256": "4c17d900d38acccd75a2bf299beda456e8693091b9bb5a8f77056732ae5f801d", "run_id": "d6a762d587d5475d82a56c0dbe4d2384", "schema_version": 1, "sha256": "ff56088bdaaa60497e06eeab578df457e5f1a24b085680f28daa5296e59970fd"}, "run_id": "d6a762d587d5475d82a56c0dbe4d2384", "runtime_packages": {"numpy": "0.0", "torch": "0.0"}, "runtime_snapshot_sha256": "cd69d4b1de535e6c59efc66b6b376991e7c9afdc2d3e7cefcc8f0ea18523d164", "schema": "catalog_fc607ffa4125402994f62c319f632faa", "scope": "harness SDK serialization; actual no-op metadata runtime custody and semantic recovery", "training_executed": false, "verification_scope": "source_destination_and_cold_exact_ref_reader"} +Removed exact recorded fresh schema: catalog_fc607ffa4125402994f62c319f632faa diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-qualification.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-qualification.json new file mode 100644 index 00000000..5f1239a1 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-qualification.json @@ -0,0 +1,130 @@ +{ + "source": { + "head": "9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3", + "tree": "7f64e85f6e8f84db4d9671fbb2cd06dc5d170311", + "scope": "Two actual indexed-diagnostic custody/admission cases including new composed SDK batch/scientific readers; explicit pre-child synthetic serialization and actual no-op child, no training/science acceptance", + "files": { + "tests/test_indexed_diagnostics_publication.py": "7e14d96a4d2a5d6e2be5aa884770a8b57a32653e009c890f68291f125551fa11", + "extension_evidence.py": "3265173c631453c2ec16a00a2d9c4a61cf0cb6f58f7b2413cd3b872d30ffecc2", + "recipe_contract.py": "dc57a657dd59cee5297cbdaf2426a2d147424a1e737cc0dad799b6084b873646", + "tuldok_extension_sdk/diagnostics.py": "d2439f8ab20dc7a3b829b547745fd7fa84d320ac4df2930b600cd36014689699", + "tuldok_extension_sdk/evidence.py": "53027c44b78fde2db62e736d9b8ed66915b17bfca3fefe36eee30cae59af0a70", + "agent_api.py": "f1937d17da3e4db9717443e76d5feb6cbde18ce462baf709ad45710883b77b89", + "runs.py": "b701532d0bf5c77ee87676fa3bbfbf9759ae45370c28d45765b2eac538a4547e", + "run_publication.py": "fe41cdd7892afd6116a5f948069f009348652e25b788d42fd17a037db3cb816c", + "tuldok_client.py": "8ca08eb0a5fea9017dd56be0487b6c608e0375f10fcb83e1b26f5ac915c2ca03", + "app.py": "e59785cbc6c0f071baa1f0a78d544184dc786a67bca3ebf51289c09ed34fc7a4" + } + }, + "actual_tests": 2, + "seconds": 22.431, + "skipped": 0, + "state": "passed", + "source_unchanged": true, + "positive": { + "accounting_and_events_unchanged": true, + "artifacts": [ + { + "bytes": 948, + "media_type": "application/octet-stream", + "name": "diagnostic-index.json", + "role": "diagnostic_index_v1", + "sha256": "351f54de24ac9824a69a641674b1d63246bc2541c2d40f43bfc19bcbd94419d3" + }, + { + "bytes": 983, + "media_type": "application/octet-stream", + "name": "diagnostic-0.json", + "role": "diagnostic_shard_v1", + "sha256": "bb0f9f8361a1380c02dfad134f2352e40f40da8dfa1b79347c0b3a47aba90543" + }, + { + "bytes": 443, + "media_type": "application/octet-stream", + "name": "diagnostic-1.json", + "role": "diagnostic_shard_v1", + "sha256": "b98746f69577f240ee39204f4b6030b91c9e46b1a674d4164afb52afc06ef133" + } + ], + "captured_python_producer_executed": false, + "composed_readers": { + "batch_calls": 1, + "batch_collected_exact_artifacts": [ + "diagnostic-index.json", + "diagnostic-0.json", + "diagnostic-1.json" + ], + "comparison": "same exact Ref diagnostic delta zero; no scientific acceptance", + "individual_segment_calls": 0 + }, + "counts_after": { + "access_execution_consumptions": 1, + "access_execution_grants": 1, + "model_attempt_reservations": 1, + "workspace_graphs": 1, + "workspace_runs": 1 + }, + "counts_before": { + "access_execution_consumptions": 1, + "access_execution_grants": 1, + "model_attempt_reservations": 1, + "workspace_graphs": 1, + "workspace_runs": 1 + }, + "coverage": { + "omitted": [ + { + "count": 1, + "reason": "capture_budget", + "start": 2 + } + ], + "omitted_count": 1, + "optional_count": 3, + "required_count": 2, + "retained_optional_count": 2, + "retained_required_count": 2, + "total_count": 5 + }, + "custody_outcome": "completed", + "numerical_acceptance": "not_evaluated", + "original_state": "interrupted", + "result_ref": { + "catalog_id": "fc607ffa4125402994f62c319f632faa", + "custody_receipt_sha256": "66d76987a01903d9bcfdb06a991d411b729064c0a2a29461ae62d295e5c4edec", + "inventory_sha256": "552f458cfd5e606c3f306bf4878946636e2e78bf6a53c6f648074a9aba7d9a1b", + "kind": "recovered_publication_result", + "material_sha256": "4c17d900d38acccd75a2bf299beda456e8693091b9bb5a8f77056732ae5f801d", + "run_id": "d6a762d587d5475d82a56c0dbe4d2384", + "schema_version": 1, + "sha256": "ff56088bdaaa60497e06eeab578df457e5f1a24b085680f28daa5296e59970fd" + }, + "run_id": "d6a762d587d5475d82a56c0dbe4d2384", + "runtime_packages": { + "numpy": "0.0", + "torch": "0.0" + }, + "runtime_snapshot_sha256": "cd69d4b1de535e6c59efc66b6b376991e7c9afdc2d3e7cefcc8f0ea18523d164", + "schema": "catalog_fc607ffa4125402994f62c319f632faa", + "scope": "harness SDK serialization; actual no-op metadata runtime custody and semantic recovery", + "training_executed": false, + "verification_scope": "source_destination_and_cold_exact_ref_reader" + }, + "cleanup": [ + { + "scope": "indexed", + "schema": "catalog_b8674cd6642d40c59051fda95a7cb9b5", + "independently_absent": true + }, + { + "scope": "indexed", + "schema": "catalog_fc607ffa4125402994f62c319f632faa", + "independently_absent": true + } + ], + "limits": [ + "Harness-authored captured SDK serialization before actual no-op child, not a captured Python producer/training witness.", + "Scientific comparison uses the same exact Ref twice; diagnostic delta zero is not a differing-model scientific acceptance result.", + "One SDK batch operation and no individual segment operation are asserted; transport is real HTTP, separate raw HTTP count is covered in prior batch consumer tests." + ] +} diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-readers-70dcaeb-independent-review.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-readers-70dcaeb-independent-review.json new file mode 100644 index 00000000..46252998 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-readers-70dcaeb-independent-review.json @@ -0,0 +1,47 @@ +{ + "checks_run": [ + "Read frozen git source and fixture/auth/owner implementations only", + "Built-in compile(source) and small AST/source ordering/assertion checks; no imports/application/runtime/DB/test execution" + ], + "evidence_hashes": { + "access.py": "50681dd5b2aaba01a311a7465b15161ca264fa4ca9812b95f15914adb5d808f4", + "agent_api.py": "f1937d17da3e4db9717443e76d5feb6cbde18ce462baf709ad45710883b77b89", + "tests/auth_helpers.py": "27a40542686a8a56df04601d765b0d4bf815d65d327b49931696a0ed161b5a7e", + "tests/qualify_publication_synthetic.py": "8756816e6434b3860bdb6027df52f43eb2344bcaed15b518f93061a38f59ee18", + "tests/test_indexed_diagnostics.py": "af764dc4c1dbb1e7127293968adabbbcf7d77b80ba843a2b51d50a882785a439", + "tests/test_indexed_diagnostics_publication.py": "480000ff8e7e4d4d7d97793eb9700170935a7ccf4150fb6b675af07f8759af57", + "tuldok_client.py": "8ca08eb0a5fea9017dd56be0487b6c608e0375f10fcb83e1b26f5ac915c2ca03" + }, + "finding_and_resolution": { + "original_finding": "49559c8 selected an absent destination directory; canonical collect_batch requires an existing regular directory, so the new helper would fail before any batch request.", + "root_fix": "70dcaeb0382a97d051275319881acf589bfb931d", + "verified_resolution": "output.mkdir(mode=0o700) now precedes collect_batch; no reviewer author-file edits." + }, + "format": "indexed_composed_reader_independent_source_review_v1", + "limits": [ + "Actual two C2 cases are pending root execution after M4 quiet release; no actual pass inferred from source review.", + "Batch count assertion observes AgentClient.call operations, not raw HTTP requests; canonical transport source has no automatic retries/fallback and server owners are separately qualified.", + "Scientific witness compares identical operands and one case point; it does not validate differing operands, numerical quality, training, runtime performance or a scientific winner.", + "Synthetic producer serialization occurs in harness before /usr/bin/true child; dummy0.0 package metadata is not an admitted scientific/training runtime witness.", + "No hard-real-time/cancellation latency or whole-plan acceptance claim from this helper review." + ], + "meaningful_assertions": [ + "Cold reopened Dataset from existing C2 setup is served through real loopback HTTP; cold AgentClient negotiates with a newly issued actual inspect-only agent bearer, not an owner token or fake client.", + "Helper is called inside explicit owner.bind scope to issue credential; Access.bind releases lock before yielding its ContextVar context. No main-thread dataset lock held across blocking HTTP requests was found.", + "Batch manifest pins exact recovered Ref and declared diagnostic index plus two shard names. One batch SDK operation and zero ordinary/publication single-segment operations asserted; complete receipt/item and directory inventories required.", + "Every locally published index/shard is checked against original publication inventory whole bytes/SHA256 and mode0600. No segment hash is mistaken for whole verification.", + "Scientific call uses the same exact recovered Ref twice, explicit first checkpoint, evaluation inventory SHA, canonical owner metric-definition digest and ordered operands/point echo assertions. Synthetic arrays have shape4x4x4, so point1,2,3 is valid.", + "Expected prediction delta0 is a same-evidence protocol diagnostic only; criterion must remain unknown. No scientific criterion or fitted model claim is introduced.", + "Existing no-replay recipe_process.launch guard spans helper. Copy fault, retained source/destination semantic checks and cold exact-Ref reader remain in original test.", + "Original post-helper charged/reservations, full reservation document, Run/graph/grant/consumption/reservation counts and original Run events assertions remain. Credential/audit creation is expected and outside those compute-accounting counts.", + "Finally shuts down/closes HTTP server and joins/asserts serving thread stopped before fresh catalog close/DROP in registered test cleanups. Only the exact schema recorded during fresh setup is dropped; temp directory cleanup follows." + ], + "original_helper_source": "49559c8426232f963ecafd76d02b01e51d052431", + "result": "Clear for source/test design after explicit directory repair; actual two-case qualification still pending", + "reviewed_at_utc": "2026-10-19T23:02:38.402398+00:00", + "reviewed_file": "tests/test_indexed_diagnostics_publication.py", + "reviewed_file_sha256": "480000ff8e7e4d4d7d97793eb9700170935a7ccf4150fb6b675af07f8759af57", + "reviewed_source": "70dcaeb0382a97d051275319881acf589bfb931d", + "reviewed_tree": "e249e29649d35cb9b7e33b951023bf0219c5afc2", + "reviewer": "M5 consumer author independent review" +} diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-source.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-source.json new file mode 100644 index 00000000..ddb13667 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-indexed-composed-source.json @@ -0,0 +1,17 @@ +{ + "head": "9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3", + "tree": "7f64e85f6e8f84db4d9671fbb2cd06dc5d170311", + "scope": "Two actual indexed-diagnostic custody/admission cases including new composed SDK batch/scientific readers; explicit pre-child synthetic serialization and actual no-op child, no training/science acceptance", + "files": { + "tests/test_indexed_diagnostics_publication.py": "7e14d96a4d2a5d6e2be5aa884770a8b57a32653e009c890f68291f125551fa11", + "extension_evidence.py": "3265173c631453c2ec16a00a2d9c4a61cf0cb6f58f7b2413cd3b872d30ffecc2", + "recipe_contract.py": "dc57a657dd59cee5297cbdaf2426a2d147424a1e737cc0dad799b6084b873646", + "tuldok_extension_sdk/diagnostics.py": "d2439f8ab20dc7a3b829b547745fd7fa84d320ac4df2930b600cd36014689699", + "tuldok_extension_sdk/evidence.py": "53027c44b78fde2db62e736d9b8ed66915b17bfca3fefe36eee30cae59af0a70", + "agent_api.py": "f1937d17da3e4db9717443e76d5feb6cbde18ce462baf709ad45710883b77b89", + "runs.py": "b701532d0bf5c77ee87676fa3bbfbf9759ae45370c28d45765b2eac538a4547e", + "run_publication.py": "fe41cdd7892afd6116a5f948069f009348652e25b788d42fd17a037db3cb816c", + "tuldok_client.py": "8ca08eb0a5fea9017dd56be0487b6c608e0375f10fcb83e1b26f5ac915c2ca03", + "app.py": "e59785cbc6c0f071baa1f0a78d544184dc786a67bca3ebf51289c09ed34fc7a4" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-refined-indexed-cleanup-independent.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-refined-indexed-cleanup-independent.json new file mode 100644 index 00000000..e8a07b77 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/tuldok-refined-indexed-cleanup-independent.json @@ -0,0 +1,22 @@ +[ + { + "scope": "indexed", + "schema": "catalog_b8674cd6642d40c59051fda95a7cb9b5", + "independently_absent": true + }, + { + "scope": "indexed", + "schema": "catalog_fc607ffa4125402994f62c319f632faa", + "independently_absent": true + }, + { + "scope": "m4_refined", + "schema": "catalog_a160667f63dc4c29b5f53f49aa5a838b", + "independently_absent": true + }, + { + "scope": "m4_refined", + "schema": "catalog_799f18cb732548b185f1cc1d7ec01a82", + "independently_absent": true + } +] diff --git a/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/validation.json b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/validation.json new file mode 100644 index 00000000..66111a0f --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/m1-indexed-diagnostics/composed-readers/validation.json @@ -0,0 +1,41 @@ +{ + "source": "9d563ceb72fdfa7cb3ce520540b4b002cbe3fba3", + "state": "2actualpassed22.431seconds0skips", + "files": [ + { + "name": "indexed-final-9d563ceb-independent-review.json", + "bytes": 2527, + "sha256": "e2156d07caec2034bf61ee36df820656948442f4383d1360956cdeb000ec072e" + }, + { + "name": "qualification.md", + "bytes": 1799, + "sha256": "b161fd1f6e56aa78ec1122665b76e49f25103150fcefc11e12a397aed1ae0193" + }, + { + "name": "tuldok-indexed-composed-actual.log", + "bytes": 3482, + "sha256": "efb4fd6d332c9a316b567892dd0930bd03c907f2f9b067e78596b3d2a83d8b42" + }, + { + "name": "tuldok-indexed-composed-qualification.json", + "bytes": 5345, + "sha256": "b9cd773ea3e2cc412ab45571daf8b4dc2c7c46ca04b42fbf26040dc04d95c2b3" + }, + { + "name": "tuldok-indexed-composed-readers-70dcaeb-independent-review.json", + "bytes": 4801, + "sha256": "be25642992e49f5a0d89cb5a34a7185470a5ce321443103bf4597ed81c577a04" + }, + { + "name": "tuldok-indexed-composed-source.json", + "bytes": 1317, + "sha256": "c39835e804b9500b770e0d6f7cb6a7dd44bee8ec386778a47b2d37130f734588" + }, + { + "name": "tuldok-refined-indexed-cleanup-independent.json", + "bytes": 505, + "sha256": "74e2e410c87085a81efe8fe50c2e7effe3f2b41c1a977eb6b4a4ffa8bbd84222" + } + ] +} From 9f08427029d8b48c8f9b50927a1c835afb573276 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 20 Oct 2026 02:33:04 +0000 Subject: [PATCH 7/7] test: preserve frozen canonical SDK source authority --- tests/test_extension_canonical_author.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_extension_canonical_author.py b/tests/test_extension_canonical_author.py index bb049a67..19fb4546 100644 --- a/tests/test_extension_canonical_author.py +++ b/tests/test_extension_canonical_author.py @@ -75,8 +75,14 @@ def test_new_project_exact_sources_full_catalog_and_model_controls_are_inert(sel definition=policy[kind]['definition'] if kind=='task' else policy[kind] raw=json.dumps(definition,sort_keys=True,separators=(',',':'),ensure_ascii=False).encode() self.assertEqual(hashlib.sha256(raw).hexdigest(),contract['policy'][kind]['definition_sha256']) - for name in ('evidence.py','__init__.py'): - self.assertEqual((PACKAGE/'tuldok_extension_sdk'/name).read_bytes(),(Path(__file__).resolve().parents[1]/'tuldok_extension_sdk'/name).read_bytes()) + # This captured SDK is pinned by the canonical inference authority; + # additive APIs in the live SDK must not rewrite historical source. + self.assertEqual(set(authority['sdk_sources']), + {'tuldok_extension_sdk/evidence.py','tuldok_extension_sdk/__init__.py'}) + for name,digest in authority['sdk_sources'].items(): + raw=(PACKAGE/name).read_bytes() + self.assertEqual(hashlib.sha256(raw).hexdigest(),digest,name) + self.assertEqual((self.root/'authored'/name).read_bytes(),raw,name) self.assertEqual(self.dataset.db.execute('SELECT count(*) FROM workspace_runs').fetchone()[0],0) def test_new_package_validates_from_copied_kit_without_app_database_or_numerical_startup(self):