diff --git a/docs/plans/lab-workflow-reliability/execution-ledger.md b/docs/plans/lab-workflow-reliability/execution-ledger.md index 966467bd..0c7681e4 100644 --- a/docs/plans/lab-workflow-reliability/execution-ledger.md +++ b/docs/plans/lab-workflow-reliability/execution-ledger.md @@ -1,5 +1,38 @@ # Lab workflow reliability execution ledger +## 2026-10-21 — Populated Study/recovered-result transfer admission + +The lead selects one composed T2/T6/T8 acceptance gap after the current-version +interrupted-transfer witness. The original Study case already executes one +neutral attempt, retains its publication copy fault, settles the original +conservative reservation and recovers the same Run. Its extension carries that +populated state through stopped backup, a separate database/directory restore, +raw preactivation snapshots and cold current SDK/CLI result collection. + +The source branch is `test/study-publication-restore-20261021` from main +`8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`; normally compose the reviewed +interrupted-transfer source/evidence before the one affected qualification. +No production, schema, cache, runtime installation, old-version compatibility, +automatic migration or new long benchmark is admitted. PUB-1/SCI-1 remain blocked +on their genuine missing lab inputs. + +The normal source composition `55fe1c6` passed only the affected Study case once: +15.550 seconds unittest / 15.582514518 seconds wrapper, zero failures/errors/skips. +One actual neutral attempt launch and zero restore/consumer replay calls were +observed. Raw 43-table/sequences/file/mode transfer, one settled reservation and +the original charge, exact cold SDK/CLI result collection and source/backup +preservation passed. Exact owned schema/database cleanup was queried and the +task PostgreSQL container stopped afterward. Raw source/environment/result/domain +receipts are sealed under `reports/study-recovered-result-transfer-20261021/`. +Root independently passed that exact frozen source from a detached worktree: +one test in 15.924 seconds unittest / 15.951664041 seconds wrapper, zero +failures/errors/skips, independently absent exact owned resources and unchanged +database inventory. Source/reproduction review is clear; the same test bytes +remain sealed. Final report/source review and published-head hosted checks +remain gates. This is one clean current-version transfer with Project roots +available; no duplicate interruption matrix, new performance or science proof +is implied. + ## 2026-10-21 — Current-version interrupted restore admission PR115 and PR116 are merged; main is diff --git a/docs/plans/lab-workflow-reliability/plan.md b/docs/plans/lab-workflow-reliability/plan.md index ea002de9..4eedf2ef 100644 --- a/docs/plans/lab-workflow-reliability/plan.md +++ b/docs/plans/lab-workflow-reliability/plan.md @@ -429,6 +429,34 @@ arguments are not substituted for old source execution. No production, schema, workflow or README change is admitted. Actual qualification and independent review remain gates; this does not complete unrelated T8 or lab acceptance. +### Populated Study and recovered-result transfer admission — 2026-10-21 + +The integration lead admits the remaining composed T2/T6/T8 boundary on isolated +branch `test/study-publication-restore-20261021`, based on exact main +`8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`. Its write set is the existing +`tests/test_run_publication_recovery.py` Study accounting case, this plan, +execution ledger and scoped `reports/study-recovered-result-transfer-20261021/`. +Normally compose the separately reviewed interrupted-transfer slice before +qualification, preserving both source/evidence histories. + +Existing tests separately prove live Study settlement through same-Run recovery, +queued Study restore, and direct-Run recovered Ref restore. Extend the actual +Study case to carry its one settled reservation/conservative charge, original +interrupted Run/fault/events, Model/Trial/material and exact recovered Ref through +stopped backup and an independent database/directory restore. Compare raw tables, +sequences, file bytes and POSIX modes before activation, preserve the original +source/backup, and use current authenticated SDK and separate CLI collection. +Repeated reconciliation and same-key recovery must retain exactly one Run, +execution grant/consumption and reservation, without a second charge or launch. + +Only the affected focused case and independent serial reproduction are needed +locally; no long benchmark or duplicate generic interruption matrix is admitted. +Published-head hosted checks and independent source/evidence review remain gates. +No production change is admitted absent an observed correctness failure. This +does not add backward adapters or automatic migration, qualify installed-runtime +mutation/preparation reuse, complete all M1–M6/T1–T8, or supply missing PUB-1/SCI-1 +lab inputs. Their real provenance/criterion requirements remain explicit. + ### Current-version interrupted-transfer acceptance admission — 2026-10-21 After PR116 merged as `8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`, the diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/acceptance.md b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/acceptance.md new file mode 100644 index 00000000..0c1ad2c1 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/acceptance.md @@ -0,0 +1,102 @@ +# Settled Study and recovered result through stopped restore + +This extends the existing real Study publication-recovery case through one clean +current-version stopped transfer and cold public consumers. No production repair +or new feature implementation was needed. The existing T2/T6/T8 plan admission +owns the scope; this is not a new product plan. + +## Exact source and execution + +The source checkpoint is `55fe1c6e6c214f71b00d00ea3cdd80030a6751da`, tree +`96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68`. It normally composes the Study +source `79fcbe6a84ca4d22b9a3d1039fa16d7f237fe6a4` and the reviewed interruption +candidate `a28034e06772d337200f4529e3545bf96bdc524c`; both complete plan/ledger +admissions were retained when their adjacent insertions conflicted. The inherited +interruption test and its sealed reports are unchanged. + +The only affected case, +`PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery`, +passed once in 15.550 seconds unittest / 15.582514518 seconds wrapper, with zero +failures, errors or skips. Its test file SHA256 is +`e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3`. +These are test execution durations, not a performance comparison. + +The serial author run used Python 3.12.14, psycopg 3.3.6 and PostgreSQL 17.11. +The [manifest](evidence-manifest.json) seals the exact raw runners, logs, +environment, source, result/domain receipts and resource inventories under +`author/` and `independent/`. Root independently reproduced the same frozen +source from a separate detached worktree: one test in 15.924 seconds unittest / +15.951664041 seconds wrapper, with zero failures, errors or skips. Its +source/reproduction review is clear, receipt SHA256 +`4e98c56e3a90b4129acb653ef93f1dbafe516b889db7ff6df29bdc4b271eef54`. +Final report/source review and published-head hosted checks remain required; +neither local result is relabeled as a run of an unexecuted publication SHA. + +## Composed integrity boundary + +The existing neutral fixture creates one captured Model, Study, Trial and exact +execution grant. It admits and launches the actual external byte-copy attempt +once. Only original publication copying is faulted after valid process drainage. +The original Run remains interrupted with its recorded `unavailable` publication +fault. Explicit accounting reconciliation settles the original reservation, then +owner-authorized recovery attaches the exact same-Run publication-result Ref. + +The four observed business counts remain one each: Run, execution grant, +consumption and Study reservation. The reservation document and original events +are unchanged by recovery. Its conservative charge equals the original admitted +envelope: one attempt slot, 37 admitted wall seconds, 2,802,248 retained bytes and +3,210,449 peak staging bytes. These are the existing reservation policy bounds, +not measured execution time or physical disk usage. + +After all source writers stop, ordinary backup retains raw schema 10 tables and +file inventories. Restore targets a new directory and a separately created owned +PostgreSQL database, leaving the original source intact. Before application +activation, all 43 raw tables, all identity sequences, ordinary file bytes and +POSIX permission modes match the stopped source. This includes a populated +Study/Trial, its settled reservation and captured material alongside the exact +recovered publication Ref, rather than the zero-Study direct-Run fixture. + +Cold activation then verifies the exact Trial/material, full post-recovery Run +document and result Ref. Repeated explicit Study reconciliation returns the same +reservation and charge; same-key publication recovery returns the same result. +The ordinary interrupted-Run artifact reader still refuses without substituting +the attached result. No new execution request key, Run, grant, consumption, +reservation or charge is introduced. + +An inspect-only current client reads the actual Study, Trial and recipe material +through authenticated HTTP and collects the exact recovered Ref through the SDK. +A separate CLI process reads Study state and the publication result, exports the +same collection manifest and collects identical complete artifact bytes. The SDK +also exercises seven-byte segment reads; both consumers publish mode 0600 files. +After all calls, every raw table and sequence still equals the source snapshot, +including the exact serialized reservation and complete post-recovery Run. + +The original source and backup retain all bytes and POSIX modes, and the source +raw tables/sequences remain unchanged. The execution observer records one real +attempt launch; the restore/activation/consumer launch tripwire records zero. +Both runs have empty before/after schema inventories and matching complete +database-name inventories. Exact emitted source-schema and restored-database +names were separately queried and confirmed absent after each cleanup. Each +serial author/independent window stopped the task-owned PostgreSQL container +after completion; no additional affected-case run was performed. + +## Preserved limits + +This is one clean current-version Study transfer. The separately qualified three +controlled restore interruptions remain attributed to their own source/reports; +they were not rerun here. Original Project roots remain available. This case does +not qualify root removal, new corruption seams, host crash/power loss, device/fsync +durability, ACL/ownership/xattr/timestamp preservation or arbitrary job-sequence +rollback. It does not add a new no-recopy I/O measurement. + +No new benchmark, preparation cache, runtime installation/mutation, classifier +training, scientific selection or E4 qualification was run. Existing paired +collection evidence retains its original subjects and scoped retention decision; +no broad workflow speedup or full M1–M6/T1–T8 acceptance follows from this case. + +Owners migrate older projects themselves once; no backward adapter or automatic +migration system is introduced. Current schema validation, honest version errors, +accounting, original failure evidence and no compute replay remain required. +PUB-1 still lacks the original real failed Run/custody/profile/output inventory. +SCI-1 still lacks a real comparison pair, criterion authority and environment. +This neutral byte-custody result cannot supply either missing lab input. diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/domain.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/domain.json new file mode 100644 index 00000000..4128c9ff --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/domain.json @@ -0,0 +1,714 @@ +{ + "format": "study_publication_restore_acceptance_v1", + "scope": "Real neutral Study recovery, stopped raw transfer and cold current clients; no science/performance claim", + "schema_version": 10, + "catalog_id": "d0d14aad4a754a42b274f919dd9f484d", + "study_id": "b3dabe7fe8ee44928c05d661dbc595a6", + "trial_id": "bcff5def0f754573b2cc8adb4aeb7362", + "run_id": "e29b681480db4fd18e749c65d92139d1", + "result_ref": { + "kind": "recovered_publication_result", + "schema_version": 1, + "catalog_id": "d0d14aad4a754a42b274f919dd9f484d", + "run_id": "e29b681480db4fd18e749c65d92139d1", + "material_sha256": "f26a4d9539c8a1af3e8ada840d61523afb0ace76b87e6e5032c2abf44e5e778a", + "inventory_sha256": "84afb1364c2ecd99dbc7b34258f2d331b576f8e49b96aebd980cd73dd1450d84", + "custody_receipt_sha256": "ea32867553288928096f43bca5e61fd2ee4397d9da6e3c165873943d43b9437a", + "sha256": "34a9d2b69c99df8afff999f129b2dd382230644ff35129a5bc60821260dce000" + }, + "original_run_state": "interrupted", + "original_fault": { + "code": "unavailable", + "format": "external_publication_fault_v1", + "message": "Recipe output changed or could not be published.", + "recorded_at": "2026-10-21T01:47:36.877661+00:00", + "source_sha256": "4430f74ec039484162001c0371fe8a3e9c9848280731532cd4065e076b53c8cd" + }, + "custody_phase": "drained", + "reservation_document_sha256": "d010e50c981e300095a91fc7cabfc8c011abe91a21a84320cd3b4e13d84cfc61", + "settled_reservation": { + "actor_id": "fa2d94d90d194d0f8f9339b0716cd029", + "admission_epoch": 1, + "envelope": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "id": "9f186c488a314eadacc38f9cef976d4e", + "material_sha256": "51282dd9b1d2423b1b93e40f99d1618783f03016ec65ba8388bf06597897dddd", + "parent_run_id": null, + "request_key": "attempt-0bd55233e9a84b7d92a9b37a72b0edb8", + "request_sha256": "dcaf5347f7d4506fb9bf6005b16617dda54532a4dd63edcfe6ec23ffb25edb99", + "run_id": "e29b681480db4fd18e749c65d92139d1", + "settled": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "state": "settled", + "study_id": "b3dabe7fe8ee44928c05d661dbc595a6", + "trial_id": "bcff5def0f754573b2cc8adb4aeb7362" + }, + "charged_before": { + "attempt_slots": 1, + "admitted_wall_seconds": 37, + "retained_bytes": 2802248, + "peak_staging_bytes": 3210449 + }, + "charged_after": { + "attempt_slots": 1, + "admitted_wall_seconds": 37, + "retained_bytes": 2802248, + "peak_staging_bytes": 3210449 + }, + "original_envelope": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "counts_before": { + "workspace_runs": 1, + "access_execution_grants": 1, + "access_execution_consumptions": 1, + "model_attempt_reservations": 1 + }, + "counts_after": { + "workspace_runs": 1, + "access_execution_grants": 1, + "access_execution_consumptions": 1, + "model_attempt_reservations": 1 + }, + "table_summaries": { + "access_actors": { + "columns": [ + "id", + "kind", + "label", + "created_at" + ], + "count": 4, + "row_sha256": [ + "02b3a5dc37073f2cce5492185dfe9d4b279d46b73135a87e319931ff5bd08ba1", + "84614e428ef051cad5f45adc4b5f4f78d2ca1ae1d8857f01ba13544c40b5bbce", + "fbe1616671a3a769f63b4947ad1950bc59e1e35906af449d5f94589096f9e120", + "1567e85601bc6733d5f37240658e4f0027d52ff8fbc4ce1d05fc60628b72eb8c" + ], + "sha256": "a7facf4699cde261d07c4a8a41835338c688fafd48d76e57d251190809a0fafb" + }, + "access_admissions": { + "columns": [ + "id", + "catalog_id", + "actor_id", + "credential_id", + "kind", + "job_id", + "request_digest", + "effects_json", + "created_at" + ], + "count": 1, + "row_sha256": [ + "27216c7b4ce64f0ef4cdda28bd6e8c1975f766b5c3e761a6043cf94b00d16dfa" + ], + "sha256": "007f385cd1e492874fe186492ef8050ed4eb2ebeb382295be713965977d2f27d" + }, + "access_audit": { + "columns": [ + "id", + "actor_id", + "credential_id", + "action", + "target_json", + "details_json", + "result", + "created_at" + ], + "count": 28, + "row_sha256": [ + "c89e0fe3895241a523ca1b7b8018d96e81e8b2a743fde277af35e7fd398c522a", + "0f8f8899e3c6437846e556ac79707067352629d787c2a5a04b57104ac687ebea", + "87ac3cb8ec4fd27a6e7171cf202bb4f2c5d1cc3411dcb7ac462eb937a8b0dab3", + "da98d1dcb0034a24a54e5fb29494e3005381fe9be38ea1c1f27d04c3c5a1d4ad", + "3b5c95601d8bc61c748993618d81208532beb73e731f715ea304d3d84cd08161", + "95f944937a7dc0b583d4097fa30058f942128f44e5bc63be38c8a92fb15290bf", + "1cb39f1be95e645e32f42d9ae384e0567aff13cf651920d3982f1f43e77947c2", + "e192070e5539a6bc1b339ceb3fc9474597c5f8cfe2ef7745181e6658b7bef585", + "1b9b9e5f910b1d444c6b3cb7365186cc59f7fdb345b8c69f76d3bfe925de98b6", + "31acc85b01f107e57a6a8d4dae4a3632dd9ea6898482f0ad327971b3848ad223", + "7d8a7fb3d0f79de4ee480f086b06c13e2aae1ce4b7075eb2bdb8a10dac0137e1", + "eb22036265927a001554baae8731fc0091b3ec3a38c1e3d561fd5d764e4fe565", + "8b64403dec358f1e8b04c19fb0987eaf0bbbcd4e86e8d5577cd13a10bb18c35c", + "ad55610e7c6cc407b3f4f9c779b915c7d6355e7d10f51fe51e41c66e387972f9", + "ba4523acc32f77694c16c2b884d78fe09604467fd9d927e09219e1348238b65f", + "bcd73c9dcd5bc525550beb0facc4452c917f3e29900eb779f5fd9f298623a674", + "681665589bf7eb5990dc604f8ac2d76ae456c2851340f7663d0f314aabafef7a", + "ea68a62299fbf2994f112d4eb9b0cea241d9e2e02b795267fda248ba04c50cf6", + "75b87cbb1cb7871f7e9da47dd05c3d0406aef762f5f7425d587224ca601b82b5", + "b9dcde6f037ecdb075d4fd57329b2e59ee35095b407ca870a5b6a8fa4d4509df", + "371bf5d7a503464cc6026b6d50f09b11f4b546b08bf412165084bb0cd9a965c7", + "2b1ffff2497cc63eb2725cce96566d96f2c69849fe9200ea64ae1cd2e14720f9", + "68ee33b2715d111407272efe13a2d2ec249e48b5f9161cb6f01e958089d65248", + "3d8ad56d527fc1fbbba7170d5bd2c44f350efd44496f922c81e71153ce77d00e", + "88e1e701673f5b373801eeee20917b91977a7737b0407d5cd9d58f05275f7d85", + "98a97bcf5cbc81359d140256581bc478044eace30803577954e26637c80b8573", + "8df1df593c3a2cb2d1b6aa11d3e4d687801f9bca8acaa4e1d4653954a501eded", + "9677960d550e606ef7cb06864fd7ce74767dff0c4c2e88b4297553254b64af3f" + ], + "sha256": "3b5011a2a95ed594291d9f182149775e0a8b565ca1d25c9dbbc72a611edff8b5" + }, + "access_credentials": { + "columns": [ + "id", + "actor_id", + "token_sha256", + "kind", + "scopes_json", + "expires_at", + "revoked", + "consumed", + "created_at" + ], + "count": 5, + "row_sha256": [ + "91bdce2628eb9ae59320fad4b20eaafac0c69d289a02c88944ee910b9769983c", + "e93a19c6008cc55d77593d5b0cb7e14395d04676cf5ae1a517014f8c9050bbe4", + "f30e2befc7ef66f283e36d8414ae217b0e7c0e003f1fcb0f09ba6b5db2760954", + "b06c6f533667e681cb407038328fefc5d664677f561ef962cb9773d3dffffa15", + "e7cdc936af57733f4b7a647ef0b475806ccf5bd61ef6162323f056251be1a005" + ], + "sha256": "0057d5d4ae41d4e41473fd47bad0952217a397789ffd5f3a9d00dea586c42524" + }, + "access_execution_consumptions": { + "columns": [ + "grant_id", + "run_id", + "admission_id", + "consumer_credential_id", + "request_sha256", + "consumed_at" + ], + "count": 1, + "row_sha256": [ + "566d626e83dfd9d7d05114e78e37bd1c267eeb9deb1e66b7d420486a0a108bdc" + ], + "sha256": "b67b48e1c148ffdbf7344d18b0e9b14e073fa03abaa51ab399e9f692a68d1e01" + }, + "access_execution_grants": { + "columns": [ + "id", + "catalog_id", + "approver_actor_id", + "approver_credential_id", + "consumer_actor_id", + "material_sha256", + "created_at", + "expires_at", + "revoked_at" + ], + "count": 1, + "row_sha256": [ + "92182ed5addc4c9de8648373c6d49961542ee9a2ffde27ebb9831d3113440e0c" + ], + "sha256": "3747ceb317c91f69ba9196e3da125874ee05250633ec511853c520f18bfaad92" + }, + "caption_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "generation_entries": { + "columns": [ + "id", + "job_id", + "ordinal", + "data", + "storage_order" + ], + "count": 0, + "row_sha256": [], + "sha256": "530faef7a9e5031599c72af9332f8c2472584ffd0d4110ab97eb445c3a9680dd" + }, + "generation_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "grounded_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "model_attempt_reservations": { + "columns": [ + "id", + "study_id", + "actor_id", + "request_key", + "request_sha256", + "trial_id", + "run_id", + "document" + ], + "count": 1, + "row_sha256": [ + "8f433744017d689b5bf2a6499212435ea054fa794755070448fdc4e47963eaab" + ], + "sha256": "6660517d89f2b32262bf18e2399028ecb510a2e843f676bf47527f3f1e2afe05" + }, + "model_candidates": { + "columns": [ + "id", + "study_id", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "8e150598074027645a884f8c83471a910abaabfaca5017c5154bed5b5f6ea480" + }, + "model_definition_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "1bce4cb61599f3c7aab04f4d999b7d255a38c35ad91d1c240885a286a1de44c3" + ], + "sha256": "b05a077b3bae48b1b3338b05791d6cd66371ded18c4cb5f51dae2b907d426517" + }, + "model_materials": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "0bd0f1e447ed1af675842a22b016a30f5904c0d6f67d5cb00a9b4d1c439c6307" + ], + "sha256": "467e8e12933732f2f54afa4e1fb539f2aba62014ed728408a65cfe5c33314696" + }, + "model_operation_receipts": { + "columns": [ + "actor_id", + "operation_id", + "request_key", + "request_sha256", + "document" + ], + "count": 2, + "row_sha256": [ + "3d33ca34560bf402fc6c38aee3846d04e96e3e0850b41547d490665138d525c6", + "2cbac16d553709fcf21bd4a6b39cda1b2836bda1dfd509a9568e3b80905269af" + ], + "sha256": "8a595802c4618f9c7e256d405c0eb74cd109965892f63a99f9bca1fe55ccf3d2" + }, + "model_selection_policies": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "dd2465d1922f54a3f7ac8fff7dbc5a9bc9917475f962ad60f7a63fcee71b4f89" + ], + "sha256": "85cef6a73c5303cdcf10d60f9edda7c51d03c299471b13fad0ff67870dc2e969" + }, + "model_studies": { + "columns": [ + "id", + "revision", + "admission_epoch", + "document" + ], + "count": 1, + "row_sha256": [ + "f9941320bc47476cd5deadbcfe7e7b5eab444c8be2dd9edf877ae2c4bedfc45e" + ], + "sha256": "e4ed2d862f796b4cd230c97fdae63403e36b13405c2a05f6919e345ee0ae952d" + }, + "model_study_memberships": { + "columns": [ + "study_id", + "revision", + "document" + ], + "count": 2, + "row_sha256": [ + "ff706f6668d3fdfe1b85b690a951baeebcf84adb3d6aff08c13131cd984b5682", + "1f193a7380d5997865c36ae508fd940253b9e7cd6cf4df3799193134d9444c05" + ], + "sha256": "29b05ccfb99b1089a03bcd50124ef3bda67596410e731c009948d0bd46c5061f" + }, + "model_trial_lineage": { + "columns": [ + "trial_id", + "parent_trial_id", + "baseline_trial_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ce519910146db157ed46ee02a83bc6add785136de22e6b9cd48581c756299ec0" + }, + "model_trials": { + "columns": [ + "id", + "study_id", + "material_sha256", + "replicate", + "document" + ], + "count": 1, + "row_sha256": [ + "a6eeedbab7f17289adfea28c83b5e195136405177f0038729424b4cb4f965dc6" + ], + "sha256": "17425504a64efc0d6358eeffba545dac709410b62b4bfc29bb811616520632ee" + }, + "samples": { + "columns": [ + "id", + "sha256", + "filename", + "width", + "height", + "book_id", + "session_id", + "split", + "created_at", + "updated_at", + "revision", + "annotation" + ], + "count": 0, + "row_sha256": [], + "sha256": "45cba19d248a0b126119bc05246fdfe6459c95d5cce245ec3a4cd47d1b5e2995" + }, + "saved_searches": { + "columns": [ + "id", + "name", + "revision", + "criteria_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "caa62ec987589481b04e93502d8f340a1a41703997b68a0c94df642ee8e5a8a6" + }, + "saved_selections": { + "columns": [ + "id", + "name", + "revision", + "members_json", + "member_count", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "ef015a7aae83417e2ee6b9aa54c0f3f618358296ed201682901be8c0841b7e11" + }, + "text_classification_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "workbench_deleted_sources": { + "columns": [ + "id", + "book_id", + "session_id", + "split" + ], + "count": 0, + "row_sha256": [], + "sha256": "c1c95154abd259591b21272e5324695db12a227e849c3fbd46d96ea2e554a7c2" + }, + "workbench_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_mesh_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_pointcloud_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_preference_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_preferences": { + "columns": [ + "id", + "prompt_id", + "revision", + "data" + ], + "count": 0, + "row_sha256": [], + "sha256": "67b197c3ca40fbcbe1baf7d841be28c4e9c98b3ef3f22178c7732560f66e16c3" + }, + "workbench_records": { + "columns": [ + "id", + "kind", + "name", + "text", + "original_text", + "content_hash", + "pixel_hash", + "groups_json", + "parents_json", + "provenance_json", + "task", + "annotation_json", + "review", + "revision", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "e2e5e5e9cef6974f035aa0b76e15e13932e11b2f3fdb93574cf2791aacb19259" + }, + "workbench_response_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_responses": { + "columns": [ + "id", + "prompt_id", + "revision", + "completion", + "review", + "provenance_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "4eb68800cc161c30f33a81c52c233a59559a12f1a3744545088bf3cbe9930d67" + }, + "workbench_rights_notes": { + "columns": [ + "id", + "note", + "revision" + ], + "count": 0, + "row_sha256": [], + "sha256": "894e92255b0040027f355105f88c51cff087e8471008b30731a6e0e849c3cdc3" + }, + "workbench_sequence_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_target_proposals": { + "columns": [ + "id", + "evidence" + ], + "count": 0, + "row_sha256": [], + "sha256": "c610988ec09c1a996fba5c2d51051fd38b37a1b51d1defc6ea9647664742838e" + }, + "workbench_volume_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workspace_exposures": { + "columns": [ + "id", + "record_id", + "source_hash", + "reason", + "run_id", + "release_id", + "created_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "5bf95a047db8503a858f55e8095cb28322d6329af09b545dc4f67a3cb83e8599" + }, + "workspace_graphs": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "80639fc527d3ae9ce3cea1652dcccdba789cf71cc30babf3a131056b945a1d23" + ], + "sha256": "7e78b070b662b6f1d05a183ebc50889736c5b5e1efa2a35d90ce7517621e81e1" + }, + "workspace_projects": { + "columns": [ + "id", + "document" + ], + "count": 1, + "row_sha256": [ + "634bd97ec5f97406f2878aff02eb67daf9e9a6182932c9d4139169aecbf8325d" + ], + "sha256": "cc68fbe6116f7d6a56c191531b968f46a60f9148caabbd2e75ddb809e458a9ad" + }, + "workspace_recipe_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "workspace_recipe_materials": { + "columns": [ + "sha256", + "recipe_id", + "recipe_revision", + "document" + ], + "count": 1, + "row_sha256": [ + "2b5bf382ed80055fc591a31bdd15628bc3809352733aa372c47cabfe7dec411c" + ], + "sha256": "b93bd7a206370a3197f675e325ffd1bae6241254c5d6c680b90da42aabca1c32" + }, + "workspace_runs": { + "columns": [ + "id", + "request_key", + "request_sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "66c1b0a9796df9898f5b60a43e5389be29bb954fca6f7c15846b4979761e9577" + ], + "sha256": "707a9ebf9ccfdfa1ee7b1696a3d3d5dc43a5b7b76a041c06cd73cdbe75676bab" + } + }, + "sequences": { + "generation_jobs": { + "last_value": 1, + "is_called": false + }, + "grounded_jobs": { + "last_value": 1, + "is_called": false + }, + "caption_proposals": { + "last_value": 1, + "is_called": false + }, + "text_classification_proposals": { + "last_value": 1, + "is_called": false + }, + "generation_entries": { + "last_value": 1, + "is_called": false + } + }, + "source_tree_sha256": "14757d42082e60d13f23f8b652be4532cc57e3497f2114a3227e451346a746e0", + "source_modes_sha256": "14fae5e444ef8e6a796844da35d9c82c165596e78728ebf8f1575d4391ce3568", + "backup_tree_sha256": "20d0ce064fd3d170b8280bea6bb7c3ba9b86d5cc6a13fd244f6829199f417644", + "source_and_backup_unchanged": true, + "raw_transfer_equal_before_activation": true, + "cold_tables_and_sequences_unchanged": true, + "artifact_sha256": "54ae9355f214d827afd4ec287ff813d3af22683a52c10e9ed1ea62507cfa4d10", + "artifact_bytes": 55, + "sdk_and_cli_collection_complete": true, + "same_key_result_unchanged": true, + "actual_attempt_launch_calls": 1, + "restore_and_consumer_replay_calls": 0, + "cleanup": { + "restored_database_removed": "study_publication_restore_a140970193ae4b75bc3f8292aaa3b3bf", + "source_schema_removed": "catalog_d0d14aad4a754a42b274f919dd9f484d" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/result.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/result.json new file mode 100644 index 00000000..84ae698d --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/result.json @@ -0,0 +1,60 @@ +{ + "format": "study_publication_restore_focused_run_v1", + "case": "test_run_publication_recovery.PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery", + "source": { + "commit": "55fe1c6e6c214f71b00d00ea3cdd80030a6751da", + "tree": "96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68", + "parents": [ + "79fcbe6a84ca4d22b9a3d1039fa16d7f237fe6a4", + "a28034e06772d337200f4529e3545bf96bdc524c" + ], + "status": "", + "test_sha256": "e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3" + }, + "environment": { + "python": "3.12.14", + "platform": "Linux-6.18.55-x86_64-with-glibc2.41", + "machine": "x86_64", + "executable": "/workspace/tuldok-qa-venv/bin/python", + "packages": { + "psycopg": "3.3.6", + "jsonschema": "4.26.0", + "Pillow": "12.3.0" + }, + "container_image": "postgres:17", + "container_image_id": "sha256:327daa8fae7178d61f93142f146b098467b345e10997b9eb79f63bd58e5c8f3c" + }, + "wrapper_seconds": 15.582514518000607, + "tests_run": 1, + "successful": true, + "failures": 0, + "errors": 0, + "skips": 0, + "resources_before": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "resources_after": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "source_unchanged": true, + "log_sha256": "cf836500fe4cbfca274a4bfba000e8af86083c518d7b14f1cedfd98cef093c81", + "domain_sha256": "e65fd1648af837095512f9cc57c754bff1f22c34ffe5f2d3decea8cd2348cbda", + "exact_cleanup_queries": { + "source_schema_absent": true, + "restored_database_absent": true + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/runner.py b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/runner.py new file mode 100644 index 00000000..55a96d23 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/runner.py @@ -0,0 +1,77 @@ +import contextlib +import hashlib +from importlib.metadata import version +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +import unittest + +import psycopg +from psycopg.conninfo import make_conninfo + +REPO = Path('/workspace/Tuldok-study-restore-20261021') +EXPECTED = '55fe1c6e6c214f71b00d00ea3cdd80030a6751da' +OUT = Path(__file__).parent/'attempt01' +def git(*arguments): + return subprocess.check_output(['git', *arguments], cwd=REPO, text=True).strip() +assert git('rev-parse', 'HEAD') == EXPECTED +assert not git('status', '--porcelain=v1') +OUT.mkdir(exist_ok=False, mode=0o700) +container = json.loads(subprocess.check_output(['docker', 'inspect', 'tuldok-qa-20261020']))[0] +assert container['State']['Running'] +settings = dict(item.split('=', 1) for item in container['Config']['Env'] if '=' in item) +dsn = make_conninfo(host='127.0.0.1', port=55432, + dbname=settings.get('POSTGRES_DB', 'postgres'), user=settings.get('POSTGRES_USER', 'postgres'), + password=settings['POSTGRES_PASSWORD']) +os.environ['TULDOK_DATABASE_URL'] = dsn +os.environ['TULDOK_STUDY_RESTORE_ACCEPTANCE_REPORT'] = str(OUT/'domain.json') + +def resources(): + with psycopg.connect(dsn, autocommit=True) as db: + return {'schemas': [r[0] for r in db.execute("SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname NOT IN ('public','information_schema') ORDER BY nspname")], + 'databases': [r[0] for r in db.execute('SELECT datname FROM pg_database ORDER BY datname')], + 'server_version': db.execute('SHOW server_version').fetchone()[0]} + +source = {'commit': git('rev-parse','HEAD'), 'tree': git('rev-parse','HEAD^{tree}'), + 'parents': git('show','-s','--format=%P','HEAD').split(), 'status': git('status','--porcelain=v1'), + 'test_sha256': hashlib.sha256((REPO/'tests/test_run_publication_recovery.py').read_bytes()).hexdigest()} +environment = {'python': platform.python_version(), 'platform': platform.platform(), + 'machine': platform.machine(), 'executable': sys.executable, + 'packages': {name: version(name) for name in ('psycopg','jsonschema','Pillow')}, + 'container_image': container['Config']['Image'], 'container_image_id': container['Image']} +before = resources(); assert before['schemas'] == [], before +os.chdir(REPO); sys.path[:0] = [str(REPO/'tests'), str(REPO)] +case = 'test_run_publication_recovery.PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery' +with (OUT/'unittest.log').open('x') as stream: + start = time.perf_counter() + with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): + suite = unittest.defaultTestLoader.loadTestsFromName(case) + result = unittest.TextTestRunner(stream=stream, verbosity=2).run(suite) + seconds = time.perf_counter()-start +after = resources() +record = {'format':'study_publication_restore_focused_run_v1', 'case':case, + 'source':source, 'environment':environment, 'wrapper_seconds':seconds, + 'tests_run':result.testsRun, 'successful':result.wasSuccessful(), + 'failures':len(result.failures), 'errors':len(result.errors), 'skips':len(result.skipped), + 'resources_before':before, 'resources_after':after, + 'source_unchanged':git('rev-parse','HEAD')==EXPECTED and not git('status','--porcelain=v1'), + 'log_sha256':hashlib.sha256((OUT/'unittest.log').read_bytes()).hexdigest()} +if (OUT/'domain.json').exists(): + domain = json.loads((OUT/'domain.json').read_text()) + record['domain_sha256'] = hashlib.sha256((OUT/'domain.json').read_bytes()).hexdigest() + names = domain['cleanup'] + with psycopg.connect(dsn, autocommit=True) as db: + record['exact_cleanup_queries'] = { + 'source_schema_absent': db.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', (names['source_schema_removed'],)).fetchone() is None, + 'restored_database_absent': db.execute('SELECT 1 FROM pg_database WHERE datname=%s', (names['restored_database_removed'],)).fetchone() is None} +with (OUT/'result.json').open('x') as stream: + json.dump(record, stream, indent=2); stream.write('\n') +print(json.dumps(record)) +assert before==after +assert record['source_unchanged'] +assert result.testsRun==1 and result.wasSuccessful() and not result.skipped +assert record['exact_cleanup_queries']=={'source_schema_absent':True,'restored_database_absent':True} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/unittest.log b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/unittest.log new file mode 100644 index 00000000..649ffed2 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/author/unittest.log @@ -0,0 +1,6 @@ +test_existing_study_reservation_and_conservative_charge_survive_recovery (test_run_publication_recovery.PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 15.550s + +OK diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/evidence-manifest.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/evidence-manifest.json new file mode 100644 index 00000000..45e841e8 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/evidence-manifest.json @@ -0,0 +1,70 @@ +{ + "format": "study_recovered_result_transfer_evidence_manifest_v1", + "baseline_main": "8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e", + "qualified_source": { + "commit": "55fe1c6e6c214f71b00d00ea3cdd80030a6751da", + "tree": "96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68", + "parents": [ + "79fcbe6a84ca4d22b9a3d1039fa16d7f237fe6a4", + "a28034e06772d337200f4529e3545bf96bdc524c" + ], + "status": "", + "test_sha256": "e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3" + }, + "inherited_interruption_source": "a28034e06772d337200f4529e3545bf96bdc524c", + "inherited_interruption_tree": "6fa29f84c442a126eea20c8484cef69855a59a97", + "inherited_interruption_test_and_reports_unchanged": true, + "evidence": { + "author/runner.py": { + "bytes": 4397, + "sha256": "ee1cfd4f25d815ccf6b9e1851525ecfeb3db52cdb8a04e16becb1b140c471d37" + }, + "author/unittest.log": { + "bytes": 308, + "sha256": "cf836500fe4cbfca274a4bfba000e8af86083c518d7b14f1cedfd98cef093c81" + }, + "author/result.json": { + "bytes": 1829, + "sha256": "fced28ffdae32884aa22a7ef24e25bc75629fd849a5a6194f45d3b2a865361a2" + }, + "author/domain.json": { + "bytes": 21095, + "sha256": "e65fd1648af837095512f9cc57c754bff1f22c34ffe5f2d3decea8cd2348cbda" + }, + "independent/runner.py": { + "bytes": 3921, + "sha256": "7418038e94ba0f5ca23b2a6a20caf75a6a290514dc55e4291ec20791499924c5" + }, + "independent/unittest.log": { + "bytes": 308, + "sha256": "a3d9e7b10707c2073bc5056d60e7a57d8e104eaffe7fdb8998399488a53c0138" + }, + "independent/result.json": { + "bytes": 1004, + "sha256": "7d6049d68de9e0845aa700256dcb929f34086dc6346830b7d9393fa53d6da573" + }, + "independent/domain.json": { + "bytes": 21095, + "sha256": "3e42b9126b10e4a8609f650f60f53f96a5d9e6e62d374ccaba02f171d9d993af" + }, + "independent/source-and-reproduction-review.json": { + "bytes": 2686, + "sha256": "4e98c56e3a90b4129acb653ef93f1dbafe516b889db7ff6df29bdc4b271eef54" + } + }, + "independent_reproduction": { + "source": { + "head": "55fe1c6e6c214f71b00d00ea3cdd80030a6751da", + "tree": "96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68", + "status": "", + "test_sha256": "e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3" + }, + "tests_run": 1, + "failures": 0, + "errors": 0, + "skips": 0, + "unittest_seconds": 15.924, + "wrapper_seconds": 15.951664041000186, + "source_and_reproduction_review_sha256": "4e98c56e3a90b4129acb653ef93f1dbafe516b889db7ff6df29bdc4b271eef54" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/domain.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/domain.json new file mode 100644 index 00000000..d733022e --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/domain.json @@ -0,0 +1,714 @@ +{ + "format": "study_publication_restore_acceptance_v1", + "scope": "Real neutral Study recovery, stopped raw transfer and cold current clients; no science/performance claim", + "schema_version": 10, + "catalog_id": "bf2f31fc9bec426ba21d1e1780e22997", + "study_id": "5e51ccd42e534bdf91d249bfae9f63a3", + "trial_id": "88343a4c577d4bc88efd733ea25b0e93", + "run_id": "2ba748f74f334f47b247c57accffcf1c", + "result_ref": { + "kind": "recovered_publication_result", + "schema_version": 1, + "catalog_id": "bf2f31fc9bec426ba21d1e1780e22997", + "run_id": "2ba748f74f334f47b247c57accffcf1c", + "material_sha256": "9b5f8aad6dfce5fe80820c6fab32a2dc5a5463462af6feb8bbf4eb25b849993d", + "inventory_sha256": "84afb1364c2ecd99dbc7b34258f2d331b576f8e49b96aebd980cd73dd1450d84", + "custody_receipt_sha256": "0e7ca2c7d7282bd69aad8840a107eeee20bee8db4e81eb10d3b30c7ad231ad8c", + "sha256": "e8d6213fe3982eefae92fb71709648c1613a7f7edcdcfff3a10745bbaaecae06" + }, + "original_run_state": "interrupted", + "original_fault": { + "code": "unavailable", + "format": "external_publication_fault_v1", + "message": "Recipe output changed or could not be published.", + "recorded_at": "2026-10-21T01:50:04.411314+00:00", + "source_sha256": "d0add4b13fc49ac8d633981014482b764a2687c14583a22ffff5022e3ba88aac" + }, + "custody_phase": "drained", + "reservation_document_sha256": "cdd462e99d7b4117af9b2526203b776ee8df7549650be123fe4cfffc7893972d", + "settled_reservation": { + "actor_id": "a3d3839fd85e44bdb9b0bdabde61db8c", + "admission_epoch": 1, + "envelope": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "id": "be53d3c630cb42b288c000320fa7241f", + "material_sha256": "a41af8797d2f2981145264b34c4349bc436b57afed9cda8ff6f9293429594487", + "parent_run_id": null, + "request_key": "attempt-6fe8e769e1f443e89dccd2a4a99a8144", + "request_sha256": "ae761d48812950406105c753f1a0e942f84d865e49ef62d26fb3907660f68602", + "run_id": "2ba748f74f334f47b247c57accffcf1c", + "settled": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "state": "settled", + "study_id": "5e51ccd42e534bdf91d249bfae9f63a3", + "trial_id": "88343a4c577d4bc88efd733ea25b0e93" + }, + "charged_before": { + "attempt_slots": 1, + "admitted_wall_seconds": 37, + "retained_bytes": 2802248, + "peak_staging_bytes": 3210449 + }, + "charged_after": { + "attempt_slots": 1, + "admitted_wall_seconds": 37, + "retained_bytes": 2802248, + "peak_staging_bytes": 3210449 + }, + "original_envelope": { + "admitted_wall_seconds": 37, + "attempt_slots": 1, + "peak_staging_bytes": 3210449, + "retained_bytes": 2802248 + }, + "counts_before": { + "workspace_runs": 1, + "access_execution_grants": 1, + "access_execution_consumptions": 1, + "model_attempt_reservations": 1 + }, + "counts_after": { + "workspace_runs": 1, + "access_execution_grants": 1, + "access_execution_consumptions": 1, + "model_attempt_reservations": 1 + }, + "table_summaries": { + "access_actors": { + "columns": [ + "id", + "kind", + "label", + "created_at" + ], + "count": 4, + "row_sha256": [ + "08ea3e02e63256c6882706ad2643dee0141f01df42a41abeff9e371c31449aef", + "e515fded2e7ba5dfe83c8620a71a980ef5ff15f043ab667190920e76ea95f0a6", + "fc7b5c9df689901b37e0af063bbf88298200322bb054e6a62685bcb877e82993", + "2384bc61eb4dcd0658d83e9692befc30347f2ce7814fc3e672ee2ec32dde3158" + ], + "sha256": "822c22c66f06674ca29c3e5235268f8d1cb58cdb3ecc05dc1d69990cb520697b" + }, + "access_admissions": { + "columns": [ + "id", + "catalog_id", + "actor_id", + "credential_id", + "kind", + "job_id", + "request_digest", + "effects_json", + "created_at" + ], + "count": 1, + "row_sha256": [ + "170c820c39398c488de5db3d10ded7e12de76ec584625b11fd5b1dba703e85c6" + ], + "sha256": "a0bb99c2b16cb1c65fdcaf122d012441287bdca5ef5c8307d476c6713dda378e" + }, + "access_audit": { + "columns": [ + "id", + "actor_id", + "credential_id", + "action", + "target_json", + "details_json", + "result", + "created_at" + ], + "count": 28, + "row_sha256": [ + "823c0dbd926135251a7788b3330074d06d10b2ebca5479c6e7559749a564450a", + "5a071e3e7396cef91b1f3dd32d17d84e1e1e99dee1e636b94d5bbd386c82ea14", + "97d6c4b366fbf49e66115488a6ec196f631f462251e0e685f82f057838c2eb29", + "13f3eaeed4a2e8da5328f552dd1f4bfb444876f5eebd553f228d3a9355e24c2b", + "a403a66f86eedb8b0b618f20beebdb9f7e8091ee0df8e4fc7718ee2283da6359", + "88c1aa5a482f29e43f7cd232e93649e18d06344a1cba2c6729e1e0c0e9b0a951", + "f489bedb96ea3c8aa1ea61827dccea2403832ccd2ccc2517de99b8caf097015a", + "30df66755aa3930775f71f3816b5af975b56fbfd64d74311f720eb52772049d3", + "ca7dd3c3657c61c1293c7091e1dc57ad2af45e6897135faa5254754306daf34a", + "d4eafeed359a1c6e9eeb0fa7abaafa8c1064a32efc6f6eed7f32d47b2abbb665", + "c965049244fd20047e1a5dd44c000368da463bb237d1e4194a2609f217981530", + "6273f70afa5e2279c25cfb775b8377e9a9ed054a7932802f9a74610d8bba8f87", + "5a0076a95b66191963f0eb170ca8300bf8ecd1bdc3ca09e4aaa24d543f652b4d", + "04ba9432a9817a33f477046548befc07fbba148e00cb7e7c3f1a3290da5d5278", + "e7fbf8164881257a6462232ff41d9b56b07d04f05414113534322b952c585249", + "466cae7771e839d2eb87beaa5813b9a4e382bf2b3163eefa711ec358fdaf0b2e", + "6500790a984bb98745039e27f1a19c4ee114184107b4d02935093e383b62199c", + "8bc841e02a1cedf426aa06adb600e43905022629fb4b904d0f5fe60e856e8a9a", + "552a2dc3df1395ead71424f55fc8f4ee116f76222d74e7d20fa0f169e137efcc", + "a1c478add97af3bcf4c453bb96bfcb8299a954cf47315697ed2960c7aa971fe5", + "cf3bacd41d4be9ea2c3b044f341d724304d4c8accf1a18cd792e1a569a28f1b9", + "a455df9d204fbf277c8c1594f9c61376aa0024b006af4bd3952813869b0e4511", + "225affef30e076f2580b9be48d77103a3e2d74979a5230a9d22a8dbdb0ed9cdd", + "6245f65f4026a6150118f26ac132ea66ef70ddf10971dde8b62b431a7b2e2315", + "f0c6c3b783d195174463e165d5df110826abdad5fc06ad8070c87615e82805f0", + "1644eb79291a514d4d9b13f7d4de6501366a1883a17d3e7a4ef3f0318e37a038", + "2848bcc114d05c2cbbb5c4f253a5c0c490ad80b20ae36719b16b5d107b0c9828", + "30ae3ba094b8fcade67f1bbbcb1f3d2b28716a156fac96e6f868fd4be714211e" + ], + "sha256": "a214516f2f1e0896bbc0aeec4afc91096519cd8ddd00e9a09cd3c10ff421f1e3" + }, + "access_credentials": { + "columns": [ + "id", + "actor_id", + "token_sha256", + "kind", + "scopes_json", + "expires_at", + "revoked", + "consumed", + "created_at" + ], + "count": 5, + "row_sha256": [ + "d5d41ca6584de6c22ca2930f048700eda3b0ba80f0a1c48853e09d1573b369bc", + "338ee6b0761eceb2615dba69cb810a553559ad0715387f3074da168a61dda183", + "90ac8429d384d5dccb488df4be76cc92b533918e5d057e466ac805d3e815ec5f", + "fdd3208ff08434854d6cb79dab62a5cb0672db3d01fc7afce1ad1b22ed274dd1", + "bdbe3018a92de0588e01fdd35cf492ee91faf007188192205ba909e1e4623ca8" + ], + "sha256": "4b3394ad09dd280779f3f8d9c29ea4287bab4fa9faebcf5c7301364a26186057" + }, + "access_execution_consumptions": { + "columns": [ + "grant_id", + "run_id", + "admission_id", + "consumer_credential_id", + "request_sha256", + "consumed_at" + ], + "count": 1, + "row_sha256": [ + "b0264f6708d6a5b6a8d553fd86fd8f06c877525e3c33221a92ddf97bf9dafbef" + ], + "sha256": "da290b3cfdb49ea80f1f3e6cfbae56c4523cd63faa83370a1d25231e5dbabaa9" + }, + "access_execution_grants": { + "columns": [ + "id", + "catalog_id", + "approver_actor_id", + "approver_credential_id", + "consumer_actor_id", + "material_sha256", + "created_at", + "expires_at", + "revoked_at" + ], + "count": 1, + "row_sha256": [ + "32a6aebaea16c7feec8caf4dcbc91c8a00d3177ad945693f5219b780fb1dc780" + ], + "sha256": "e939c04bb6dd04333c986580b9330b9af77b30033b76b05a549a2a26e7e08401" + }, + "caption_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "generation_entries": { + "columns": [ + "id", + "job_id", + "ordinal", + "data", + "storage_order" + ], + "count": 0, + "row_sha256": [], + "sha256": "530faef7a9e5031599c72af9332f8c2472584ffd0d4110ab97eb445c3a9680dd" + }, + "generation_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "grounded_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "model_attempt_reservations": { + "columns": [ + "id", + "study_id", + "actor_id", + "request_key", + "request_sha256", + "trial_id", + "run_id", + "document" + ], + "count": 1, + "row_sha256": [ + "c9a4d35230f57b9540da388357870a8435e246152ee50ac7527044b2331f7884" + ], + "sha256": "29d551d1568abeab173fc14ac85b538ccd4216d1ab80f4e908a1741ece92c329" + }, + "model_candidates": { + "columns": [ + "id", + "study_id", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "8e150598074027645a884f8c83471a910abaabfaca5017c5154bed5b5f6ea480" + }, + "model_definition_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "51464d0c922b9e4c2f7b899be69fcc9fd650a5dd737b324cd8830bb6e3f8e97b" + ], + "sha256": "f93671720839afe3af55e8364038aba5371287e91612ae6e4ab620a5b210dfe8" + }, + "model_materials": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "e12f77287e70ef91e9008bb5584083b61948d8a8d38d10610dbfde1f7cd34d37" + ], + "sha256": "04baef7a11d21eb0dae4d877eeec83efc5ea83beb008c0f45e2551694be788e7" + }, + "model_operation_receipts": { + "columns": [ + "actor_id", + "operation_id", + "request_key", + "request_sha256", + "document" + ], + "count": 2, + "row_sha256": [ + "e8bc44eb4d903d946c5247d7623e07733e1206581b51e73224283a83d88f932f", + "6c437780a04b530a7bc8d58536e23b067ae4572efb6fb768e2f170fbd41c41fa" + ], + "sha256": "09527342fd5e395c3dee46699aef655f803e192731671e9fab204d5fdd10f0b6" + }, + "model_selection_policies": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "27ebd952cc89b5a535c11d8cf9a741d7599e748d2aee9c84896b5a660324c3b4" + ], + "sha256": "5d9ea281c6911cac2dbe66cdccb8290f74490e3f86854f2f8f5bedbcb53eb193" + }, + "model_studies": { + "columns": [ + "id", + "revision", + "admission_epoch", + "document" + ], + "count": 1, + "row_sha256": [ + "5927c84d13137b61d6dd38b343651f3e586cb01ca67d654c82ca4bdc8daf1392" + ], + "sha256": "d3e97a6713edecb442e99c63b0fa13443138b3c207a90807aff6db0d19444b09" + }, + "model_study_memberships": { + "columns": [ + "study_id", + "revision", + "document" + ], + "count": 2, + "row_sha256": [ + "0583f61af6f44e85185af740df5d1a83f8d3da7fd3a1e3ba78dab211985b15f6", + "4b1db9300a711c23101df4b0a7e41a8d478a3e9723cbc9fa1d29981e5c91c80c" + ], + "sha256": "231e8bc95f994599bd795fa522ecc1894f3372963d70f84bd0e0aac65d1a5451" + }, + "model_trial_lineage": { + "columns": [ + "trial_id", + "parent_trial_id", + "baseline_trial_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ce519910146db157ed46ee02a83bc6add785136de22e6b9cd48581c756299ec0" + }, + "model_trials": { + "columns": [ + "id", + "study_id", + "material_sha256", + "replicate", + "document" + ], + "count": 1, + "row_sha256": [ + "38e922c3c56b11382427c1b8666588463c0aea06f3f2f8bebf3fd8c6e625601b" + ], + "sha256": "86d6e719b2426630c964beda6ecf3c1685e7f8bdd029dd0bac7b5cba7c13467b" + }, + "samples": { + "columns": [ + "id", + "sha256", + "filename", + "width", + "height", + "book_id", + "session_id", + "split", + "created_at", + "updated_at", + "revision", + "annotation" + ], + "count": 0, + "row_sha256": [], + "sha256": "45cba19d248a0b126119bc05246fdfe6459c95d5cce245ec3a4cd47d1b5e2995" + }, + "saved_searches": { + "columns": [ + "id", + "name", + "revision", + "criteria_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "caa62ec987589481b04e93502d8f340a1a41703997b68a0c94df642ee8e5a8a6" + }, + "saved_selections": { + "columns": [ + "id", + "name", + "revision", + "members_json", + "member_count", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "ef015a7aae83417e2ee6b9aa54c0f3f618358296ed201682901be8c0841b7e11" + }, + "text_classification_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "workbench_deleted_sources": { + "columns": [ + "id", + "book_id", + "session_id", + "split" + ], + "count": 0, + "row_sha256": [], + "sha256": "c1c95154abd259591b21272e5324695db12a227e849c3fbd46d96ea2e554a7c2" + }, + "workbench_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_mesh_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_pointcloud_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_preference_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_preferences": { + "columns": [ + "id", + "prompt_id", + "revision", + "data" + ], + "count": 0, + "row_sha256": [], + "sha256": "67b197c3ca40fbcbe1baf7d841be28c4e9c98b3ef3f22178c7732560f66e16c3" + }, + "workbench_records": { + "columns": [ + "id", + "kind", + "name", + "text", + "original_text", + "content_hash", + "pixel_hash", + "groups_json", + "parents_json", + "provenance_json", + "task", + "annotation_json", + "review", + "revision", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "e2e5e5e9cef6974f035aa0b76e15e13932e11b2f3fdb93574cf2791aacb19259" + }, + "workbench_response_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_responses": { + "columns": [ + "id", + "prompt_id", + "revision", + "completion", + "review", + "provenance_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "4eb68800cc161c30f33a81c52c233a59559a12f1a3744545088bf3cbe9930d67" + }, + "workbench_rights_notes": { + "columns": [ + "id", + "note", + "revision" + ], + "count": 0, + "row_sha256": [], + "sha256": "894e92255b0040027f355105f88c51cff087e8471008b30731a6e0e849c3cdc3" + }, + "workbench_sequence_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_target_proposals": { + "columns": [ + "id", + "evidence" + ], + "count": 0, + "row_sha256": [], + "sha256": "c610988ec09c1a996fba5c2d51051fd38b37a1b51d1defc6ea9647664742838e" + }, + "workbench_volume_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workspace_exposures": { + "columns": [ + "id", + "record_id", + "source_hash", + "reason", + "run_id", + "release_id", + "created_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "5bf95a047db8503a858f55e8095cb28322d6329af09b545dc4f67a3cb83e8599" + }, + "workspace_graphs": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "d4a39085c541a37b8b6f518933148e957ce2e468cbcdbf08b12421bf0ae48356" + ], + "sha256": "de9abb97a112af8f72328b2dfbe815e187fa1023527c30d04e573c4d81f43f01" + }, + "workspace_projects": { + "columns": [ + "id", + "document" + ], + "count": 1, + "row_sha256": [ + "680b011e178afae794d35b7fdcbccd60ee8732c2dc37dcc98292568c0884ff33" + ], + "sha256": "dea63a5b93bc5c4fdcc294f07a8543b8e587b271946000a858912c74a2e184d0" + }, + "workspace_recipe_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "workspace_recipe_materials": { + "columns": [ + "sha256", + "recipe_id", + "recipe_revision", + "document" + ], + "count": 1, + "row_sha256": [ + "66c1e9f1de9a5916dfba058288456fc711f3b6ff64c157fec8ac3a914fd5c7d0" + ], + "sha256": "ac42b30e477a7ae8e1b1e019fe5c9cc2ce6678c6c3a627e96f6dad8f98b5c4ad" + }, + "workspace_runs": { + "columns": [ + "id", + "request_key", + "request_sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "9630988e750ca157b2a3c4e8ea00451c32e007deac53532821bb314eeb7a2ac6" + ], + "sha256": "28394d2f161667d7283f493d91bfd5b1907d9a4a7f86075514b91115e14744af" + } + }, + "sequences": { + "generation_jobs": { + "last_value": 1, + "is_called": false + }, + "grounded_jobs": { + "last_value": 1, + "is_called": false + }, + "caption_proposals": { + "last_value": 1, + "is_called": false + }, + "text_classification_proposals": { + "last_value": 1, + "is_called": false + }, + "generation_entries": { + "last_value": 1, + "is_called": false + } + }, + "source_tree_sha256": "1ebfb239f38867b38ed65cd292ab3da05d676e27ef4cc7f704e71bf8705073a1", + "source_modes_sha256": "efcd6a25fed979ef3d9a77e39d553f72479f5a20f59edb89c5a5d3b0b60ee0dc", + "backup_tree_sha256": "8245f4f737f47cf58c82d65555b7ca051fad135f45a75b78eb5e904d94dffaf2", + "source_and_backup_unchanged": true, + "raw_transfer_equal_before_activation": true, + "cold_tables_and_sequences_unchanged": true, + "artifact_sha256": "54ae9355f214d827afd4ec287ff813d3af22683a52c10e9ed1ea62507cfa4d10", + "artifact_bytes": 55, + "sdk_and_cli_collection_complete": true, + "same_key_result_unchanged": true, + "actual_attempt_launch_calls": 1, + "restore_and_consumer_replay_calls": 0, + "cleanup": { + "restored_database_removed": "study_publication_restore_05a9bbba2e0e48c8abdacc2ef4d8b87b", + "source_schema_removed": "catalog_bf2f31fc9bec426ba21d1e1780e22997" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/result.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/result.json new file mode 100644 index 00000000..8e6cffeb --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/result.json @@ -0,0 +1,40 @@ +{ + "format": "independent_study_restore_qualification_v1", + "source": { + "head": "55fe1c6e6c214f71b00d00ea3cdd80030a6751da", + "tree": "96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68", + "status": "", + "test_sha256": "e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3" + }, + "seconds": 15.951664041000186, + "tests_run": 1, + "failures": 0, + "errors": 0, + "skips": 0, + "successful": true, + "resources_before": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ] + }, + "resources_after": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ] + }, + "final_status": "", + "log_sha256": "a3d9e7b10707c2073bc5056d60e7a57d8e104eaffe7fdb8998399488a53c0138", + "independent_exact_cleanup_queries": { + "source_schema": true, + "restored_database": true + }, + "domain_sha256": "3e42b9126b10e4a8609f650f60f53f96a5d9e6e62d374ccaba02f171d9d993af" +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/runner.py b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/runner.py new file mode 100644 index 00000000..59d7b3e7 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/runner.py @@ -0,0 +1,51 @@ +import contextlib, hashlib, io, json, os, pathlib, subprocess, sys, time, unittest +import psycopg +from psycopg.conninfo import make_conninfo + +repo = pathlib.Path('/workspace/Tuldok-study-restore-independent-20261021') +out = pathlib.Path(sys.argv[1]); out.mkdir(parents=True, exist_ok=False, mode=0o700) +container = json.loads(subprocess.check_output(['docker', 'inspect', 'tuldok-qa-20261020']))[0] +env = dict(item.split('=', 1) for item in container['Config']['Env'] if '=' in item) +dsn = make_conninfo(host='127.0.0.1', port=55432, user=env.get('POSTGRES_USER', 'postgres'), + dbname=env.get('POSTGRES_DB', 'postgres'), password=env['POSTGRES_PASSWORD']) +os.environ['TULDOK_DATABASE_URL'] = dsn +os.environ['TULDOK_STUDY_RESTORE_ACCEPTANCE_REPORT'] = str(out/'domain.json') +os.chdir(repo); sys.path[:0] = [str(repo/'tests'), str(repo)] +def git(*args): return subprocess.check_output(['git', *args]).decode().strip() +def resources(): + with psycopg.connect(dsn, autocommit=True) as db: + return {'schemas': [r[0] for r in db.execute("SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname NOT IN ('public','information_schema') ORDER BY nspname")], + 'databases': [r[0] for r in db.execute('SELECT datname FROM pg_database ORDER BY datname')]} +before = resources(); assert not before['schemas'], before +source = {'head': git('rev-parse','HEAD'), 'tree': git('rev-parse','HEAD^{tree}'), + 'status': git('status','--porcelain=v1'), 'test_sha256': hashlib.sha256((repo/'tests/test_run_publication_recovery.py').read_bytes()).hexdigest()} +assert not source['status'], source +with (out/'unittest.log').open('x') as log: + start = time.perf_counter() + with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log): + suite = unittest.defaultTestLoader.loadTestsFromName('test_run_publication_recovery.PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery') + result = unittest.TextTestRunner(stream=log, verbosity=2).run(suite) + elapsed = time.perf_counter()-start +after = resources() +receipt = {'format':'independent_study_restore_qualification_v1', 'source':source, + 'seconds':elapsed, 'tests_run':result.testsRun, 'failures':len(result.failures), + 'errors':len(result.errors), 'skips':len(result.skipped), 'successful':result.wasSuccessful(), + 'resources_before':before, 'resources_after':after, 'final_status':git('status','--porcelain=v1'), + 'log_sha256':hashlib.sha256((out/'unittest.log').read_bytes()).hexdigest()} +if (out/'domain.json').exists(): + domain = json.loads((out/'domain.json').read_text()) + assert len(domain['table_summaries']) == 43 + assert domain['counts_before'] == domain['counts_after'] == dict(workspace_runs=1, access_execution_grants=1, access_execution_consumptions=1, model_attempt_reservations=1) + assert domain['charged_before'] == domain['charged_after'] == domain['original_envelope'] + assert domain['actual_attempt_launch_calls'] == 1 and domain['restore_and_consumer_replay_calls'] == 0 + with psycopg.connect(dsn, autocommit=True) as db: + absent = {'source_schema': db.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', (domain['cleanup']['source_schema_removed'],)).fetchone() is None, + 'restored_database': db.execute('SELECT 1 FROM pg_database WHERE datname=%s', (domain['cleanup']['restored_database_removed'],)).fetchone() is None} + assert all(absent.values()), absent + receipt['independent_exact_cleanup_queries'] = absent + receipt['domain_sha256'] = hashlib.sha256((out/'domain.json').read_bytes()).hexdigest() +(out/'result.json').write_text(json.dumps(receipt,indent=2)+'\n') +print(json.dumps(receipt)) +assert before==after, receipt +assert not receipt['final_status'], receipt +raise SystemExit(0 if result.wasSuccessful() and result.testsRun==1 and not result.skipped else 1) diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/source-and-reproduction-review.json b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/source-and-reproduction-review.json new file mode 100644 index 00000000..a0c69552 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/source-and-reproduction-review.json @@ -0,0 +1,39 @@ +{ + "format": "independent_study_publication_restore_review_v1", + "disposition": "clear_focused_subject_hosted_checks_pending", + "blocking_findings": [], + "reviewed_commit": "55fe1c6e6c214f71b00d00ea3cdd80030a6751da", + "reviewed_tree": "96672fb93f5b2bf3f8ca6e8cbf1e537d2fa4da68", + "test_sha256": "e08eb764056b209aead37b6491cbced9f9adb9cea7813ec05ccc4ea3c147c8f3", + "reproduction": { + "tests_run": 1, + "seconds": 15.951664041000186, + "failures": 0, + "errors": 0, + "skips": 0, + "source_clean_before_and_after": true, + "schemas_before_and_after": [], + "database_inventory_unchanged": true, + "exact_source_schema_and_restored_database_independently_absent": true, + "log_sha256": "a3d9e7b10707c2073bc5056d60e7a57d8e104eaffe7fdb8998399488a53c0138", + "domain_sha256": "3e42b9126b10e4a8609f650f60f53f96a5d9e6e62d374ccaba02f171d9d993af" + }, + "review": [ + "Extends the existing real Study recovery case instead of a duplicate feature or synthetic scientific workload.", + "One actual neutral recipe launch has terminal drained custody and retains its original interrupted Run, events and publication fault.", + "One populated Study reservation is settled once; its exact document and charged envelope survive repeated recovery/reconciliation and stopped transfer.", + "A separate restored database has all 43 raw table states, sequences, complete file hashes and POSIX modes equal before application activation.", + "Cold owner reads and same-key recovery preserve exact Trial/material/result Ref, reservation, charge, Run and counts.", + "Authenticated current SDK and separate CLI read the Study and collect exact recovered Ref bytes into mode0600 files with no extra compute.", + "Post-read raw tables/sequences remain equal; source and backup files/modes/raw state remain unchanged; exact owned resources are cleaned.", + "Existing PR117 test and sealed evidence are composed unchanged; no production/schema/workflow/README change is required." + ], + "limits": [ + "One clean stopped current-version transfer; PR117 separately owns the three controlled interruption seams.", + "Project roots remain available; no new root-removal, scientific-training, power-loss, torn-payload or installed-runtime mutation proof.", + "No new performance samples, cache/admission policy or automatic migration system.", + "Owners migrate old projects themselves; current-version schema/integrity and no replay remain required.", + "No full M1-M6/T8 or PUB1/SCI1 acceptance; genuine original failure and comparison inputs remain absent.", + "Final report/source binding and required published-head hosted checks remain merge gates." + ] +} diff --git a/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/unittest.log b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/unittest.log new file mode 100644 index 00000000..bdafca45 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/study-recovered-result-transfer-20261021/independent/unittest.log @@ -0,0 +1,6 @@ +test_existing_study_reservation_and_conservative_charge_survive_recovery (test_run_publication_recovery.PublicationRecoveryTests.test_existing_study_reservation_and_conservative_charge_survive_recovery) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 15.924s + +OK diff --git a/tests/test_run_publication_recovery.py b/tests/test_run_publication_recovery.py index 36c5e32a..02ab387a 100644 --- a/tests/test_run_publication_recovery.py +++ b/tests/test_run_publication_recovery.py @@ -95,20 +95,55 @@ def stop(runs, directory, *args, **kwargs): self.assertEqual(dataset.runs._load(saved['id'])['publication_continuation']['state'], 'paused') def test_existing_study_reservation_and_conservative_charge_survive_recovery(self): + from http.server import ThreadingHTTPServer + import json + from pathlib import Path + import subprocess + import sys + import uuid + + import psycopg + from psycopg import sql + from psycopg.conninfo import make_conninfo + + import agent_api + from app import Dataset, make_handler + import backup_metadata as transfer + from metadata import Metadata + import recipe_process import test_experiments as fixtures + from tuldok_client import WorkflowClient + fixture = fixtures.StudyTests('runTest') self.addCleanup(fixture.doCleanups); fixture.setUp() dataset = fixture.dataset + source_dsn, source_schema = dataset.db.dsn, dataset.db.schema + cleanup = {} + + def drop_source(): + if cleanup.get('source_schema_removed'): + return + dataset.close() + with psycopg.connect(source_dsn, autocommit=True) as admin: + admin.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(source_schema))) + self.assertIsNone(admin.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', + (source_schema,)).fetchone()) + cleanup['source_schema_removed'] = source_schema + self.addCleanup(drop_source) + trial, _ = fixture.prepare(); fixture.open() body = fixture.admission_body(trial, fixture.approve()) with dataset.access.bind(fixture.agent['token']), patch.object(dataset.runs, '_start'): admitted = fixture.studies.admit(fixture.study['id'], body) run = dataset.runs._load(admitted['run']['id']) - with patch.object(dataset.runs, '_copy_verified', side_effect=OSError('Authored interrupted Study publication')): + with patch.object(dataset.runs, '_copy_verified', side_effect=OSError('Authored interrupted Study publication')), \ + patch('recipe_process.launch', wraps=recipe_process.launch) as executed: dataset.runs._run_attempt(run, (dataset.runs._capacity.cores[0],), dataset.runs._capacity.claim(run)) + self.assertEqual(executed.call_count, 1) saved = dataset.runs._load(run['id']) self.assertEqual(saved['state'], 'interrupted') + self.assertEqual(saved['custody_phase'], 'drained') self.assertEqual(saved['publication_fault']['code'], 'unavailable') with fixture.owner.bound(dataset): before = fixture.studies.reconcile(fixture.study['id']) @@ -131,6 +166,177 @@ def test_existing_study_reservation_and_conservative_charge_survive_recovery(sel self.assertEqual(current['state'], 'interrupted') self.assertEqual(current['events'], saved['events']) + # Transfer this populated, already settled Study and its exact recovered + # result together. Separate queued-Study and direct-Run restore cases do + # not prove that this reservation/charge/ref composition survives. + self.assertEqual(counts, {'workspace_runs': 1, 'access_execution_grants': 1, + 'access_execution_consumptions': 1, 'model_attempt_reservations': 1}) + ref = first['result_ref'] + with fixture.owner.bound(dataset): + expected_trial = copy.deepcopy(dataset.studies.trial(trial['id'])) + expected_material = copy.deepcopy(dataset.pipelines.recipe_material(trial['material'], verify=True)) + expected_result = copy.deepcopy(dataset.runs.publication_result(ref)) + reader = dataset.access.issue({'label': 'Cold restored Study inspector', 'scopes': ['inspect']}) + dataset.close() + source_tree = transfer.tree_manifest(dataset.path) + source_modes = transfer.tree_modes(dataset.path, source_tree) + original = Metadata(dataset.path, dsn=source_dsn, create=False) + try: + source_tables, source_sequences = transfer.read_catalog(original), transfer.read_sequences(original) + source_version = original.schema_version + finally: + original.close() + backup = fixture.path/'settled-study-backup' + receipt = transfer.backup_catalog(dataset.path, backup, dsn=source_dsn, quiesced=True) + backup_tree = transfer.tree_manifest(backup) + self.assertEqual(receipt['files'], source_tree) + self.assertEqual(receipt['tables'], transfer.summaries(source_tables)) + self.assertEqual(transfer.tree_modes(backup/'catalog', source_tree), source_modes) + + database = 'study_publication_restore_'+uuid.uuid4().hex + restored_dsn = make_conninfo(source_dsn, dbname=database) + with psycopg.connect(source_dsn, autocommit=True) as admin: + admin.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(database))) + + def drop_restored_database(): + if cleanup.get('restored_database_removed'): + return + with psycopg.connect(source_dsn, autocommit=True) as admin: + admin.execute(sql.SQL('DROP DATABASE {}').format(sql.Identifier(database))) + self.assertIsNone(admin.execute('SELECT 1 FROM pg_database WHERE datname=%s', + (database,)).fetchone()) + cleanup['restored_database_removed'] = database + self.addCleanup(drop_restored_database) + + restored_path = fixture.path/'restored-settled-study' + with patch('recipe_process.launch', side_effect=AssertionError('Study transfer replayed compute')) as replay: + marker = transfer.restore_catalog(backup, restored_path, dsn=restored_dsn) + marker_name = '.restore-receipt-'+transfer.digest(receipt)+'.json' + self.assertEqual(marker['dataset_id'], dataset.db.dataset_id) + self.assertEqual(transfer.tree_manifest(restored_path, (marker_name,)), source_tree) + self.assertEqual(transfer.tree_modes(restored_path, source_tree), source_modes) + held = Metadata(restored_path, dsn=restored_dsn, create=False) + try: + self.assertEqual(held.schema_version, source_version) + self.assertEqual(transfer.read_catalog(held), source_tables) + self.assertEqual(transfer.read_sequences(held), source_sequences) + finally: + held.close() + + cold = Dataset(restored_path, dsn=restored_dsn, project_roots={'fixture': fixture.project_path}) + self.addCleanup(cold.close) + with fixture.owner.bound(cold): + self.assertEqual(cold.studies.trial(trial['id']), expected_trial) + self.assertEqual(cold.pipelines.recipe_material(trial['material'], verify=True), expected_material) + self.assertEqual(cold.studies.reconcile(fixture.study['id']), after) + self.assertEqual(cold.studies.reconcile(fixture.study['id']), after) + self.assertEqual(cold.runs._load(run['id']), current) + self.assertEqual(cold.runs.publication_result(ref), expected_result) + self.assertEqual(cold.runs.recover_publication(self.body(saved)), first) + self.assertEqual(cold.db.execute('SELECT document FROM model_attempt_reservations WHERE run_id=%s', + (run['id'],)).fetchone()[0], reservation) + self.assertEqual(self.counts(cold), counts) + with self.assertRaises(WorkbenchError): + cold.runs.artifact(run['id'], 'result.txt') + self.assertIsNone(cold.runs._thread) + + server = ThreadingHTTPServer(('127.0.0.1', 0), make_handler(cold)) + worker = threading.Thread(target=server.serve_forever) + worker.start() + stopped = [] + + def stop_server(): + if not stopped: + server.shutdown(); server.server_close(); worker.join(5) + self.assertFalse(worker.is_alive()) + stopped.append(True) + self.addCleanup(stop_server) + + endpoint = 'http://127.0.0.1:'+str(server.server_port) + client = agent_api.AgentClient(endpoint, reader['token'], cold.db.dataset_id) + self.assertEqual(client.call('studies.get', {'study_id': fixture.study['id']})['result'], after) + self.assertEqual(client.call('studies.trial', {'trial_id': trial['id']})['result'], expected_trial) + self.assertEqual(client.call('recipes.material', {'ref': trial['material']})['result'], expected_material) + workflow = WorkflowClient(client) + manifest = workflow.collection_manifest(result_ref=ref, names=['result.txt']) + self.assertEqual(manifest['source'], {'ref': ref}) + sdk_directory = fixture.path/'study-sdk-collected'; sdk_directory.mkdir(mode=0o700) + sdk_receipt = workflow.collect(manifest, sdk_directory, chunk_bytes=7) + self.assertEqual(sdk_receipt['state'], 'complete', sdk_receipt) + self.assertEqual(sdk_receipt['items'][0]['receipt']['result_ref'], ref) + self.assertEqual((sdk_directory/'result.txt').read_bytes(), fixture.input) + + token = fixture.path/'study-reader.token'; token.write_text(reader['token']); token.chmod(0o600) + config = fixture.path/'study-reader.json' + config.write_text(json.dumps({'endpoint': endpoint, 'catalog_id': cold.db.dataset_id, + 'token_file': str(token), 'timeout': 20})); config.chmod(0o600) + ref_file = fixture.path/'study-result-ref.json'; ref_file.write_text(json.dumps(ref)) + arguments = fixture.path/'study-read.json' + arguments.write_text(json.dumps({'study_id': fixture.study['id']})) + + def cli(*arguments): + called = subprocess.run([sys.executable, str(Path(__file__).resolve().parents[1]/'project_cli.py'), + '--config', str(config), *map(str, arguments)], capture_output=True, text=True, timeout=90) + self.assertEqual(called.returncode, 0, called.stderr+called.stdout) + self.assertNotIn(reader['token'], called.stderr+called.stdout) + return json.loads(called.stdout) + + self.assertEqual(cli('call', '--operation', 'studies.get', '--arguments', arguments), after) + self.assertEqual(cli('publication-result', '--ref', ref_file), expected_result) + cli_manifest = cli('collection-manifest', '--ref', ref_file, '--name', 'result.txt') + self.assertEqual(cli_manifest, manifest) + manifest_file = fixture.path/'study-collection.json'; manifest_file.write_text(json.dumps(cli_manifest)) + cli_directory = fixture.path/'study-cli-collected'; cli_directory.mkdir(mode=0o700) + cli_receipt = cli('collect', '--manifest', manifest_file, '--directory', cli_directory) + self.assertEqual(cli_receipt['state'], 'complete', cli_receipt) + self.assertEqual(cli_receipt['items'][0]['receipt']['result_ref'], ref) + self.assertEqual((cli_directory/'result.txt').read_bytes(), fixture.input) + for path in (sdk_directory/'result.txt', cli_directory/'result.txt'): + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + stop_server() + self.assertEqual(transfer.read_catalog(cold.db), source_tables) + self.assertEqual(transfer.read_sequences(cold.db), source_sequences) + self.assertEqual(cold.runs._load(run['id']), current) + cold_counts = self.counts(cold) + self.assertEqual(cold_counts, counts) + self.assertIsNone(cold.runs._thread) + replay.assert_not_called() + cold.close() + + self.assertEqual(transfer.tree_manifest(dataset.path), source_tree) + self.assertEqual(transfer.tree_modes(dataset.path, source_tree), source_modes) + self.assertEqual(transfer.tree_manifest(backup), backup_tree) + self.assertEqual(transfer.tree_modes(backup/'catalog', source_tree), source_modes) + original = Metadata(dataset.path, dsn=source_dsn, create=False) + try: + self.assertEqual(transfer.read_catalog(original), source_tables) + self.assertEqual(transfer.read_sequences(original), source_sequences) + finally: + original.close() + drop_restored_database() + drop_source() + if report_path := os.environ.get('TULDOK_STUDY_RESTORE_ACCEPTANCE_REPORT'): + report = {'format': 'study_publication_restore_acceptance_v1', + 'scope': 'Real neutral Study recovery, stopped raw transfer and cold current clients; no science/performance claim', + 'schema_version': source_version, 'catalog_id': ref['catalog_id'], + 'study_id': fixture.study['id'], 'trial_id': trial['id'], 'run_id': run['id'], + 'result_ref': ref, 'original_run_state': saved['state'], + 'original_fault': saved['publication_fault'], 'custody_phase': saved['custody_phase'], + 'reservation_document_sha256': hashlib.sha256(reservation.encode()).hexdigest(), + 'settled_reservation': after['reservations'][0], 'charged_before': before['charged'], + 'charged_after': after['charged'], 'original_envelope': admitted['reservation']['envelope'], + 'counts_before': counts, 'counts_after': cold_counts, + 'table_summaries': transfer.summaries(source_tables), 'sequences': source_sequences, + 'source_tree_sha256': transfer.digest(source_tree), 'source_modes_sha256': transfer.digest(source_modes), + 'backup_tree_sha256': transfer.digest(backup_tree), 'source_and_backup_unchanged': True, + 'raw_transfer_equal_before_activation': True, 'cold_tables_and_sequences_unchanged': True, + 'artifact_sha256': hashlib.sha256(fixture.input).hexdigest(), 'artifact_bytes': len(fixture.input), + 'sdk_and_cli_collection_complete': True, 'same_key_result_unchanged': True, + 'actual_attempt_launch_calls': executed.call_count, 'restore_and_consumer_replay_calls': replay.call_count, + 'cleanup': cleanup} + with Path(report_path).open('x') as stream: + json.dump(report, stream, indent=2); stream.write('\n') + def test_partial_copy_capacity_counts_remaining_bytes_before_replacement(self): fixture, saved = self.failed_publication(); dataset = fixture.dataset payload = dataset.runs.path / saved['id'] / 'result.txt'