diff --git a/docs/plans/lab-workflow-reliability/execution-ledger.md b/docs/plans/lab-workflow-reliability/execution-ledger.md index 8b9f1464..966467bd 100644 --- a/docs/plans/lab-workflow-reliability/execution-ledger.md +++ b/docs/plans/lab-workflow-reliability/execution-ledger.md @@ -1,5 +1,31 @@ # Lab workflow reliability execution ledger +## 2026-10-21 — Current-version interrupted restore admission + +PR115 and PR116 are merged; main is +`8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`. All four applicable PR116 workflows +passed at exact `9d699cc154e9dde89fe26df16ad57c421e68d734`, with independent +source/policy review. Main's automatic push workflows are monitored separately; +no expired CLI credential or denied raw log is retried. + +The lead admits only the existing T8 interruption gap on the same combined +current-version fixture, with three real restore seams and raw independent +state comparisons. The original clean witness and all measurements stay sealed +at their original subjects. No benchmark, old-version witness, production cache, +automatic migration, training or scientific claim is added. PUB-1 and SCI-1 +retain their genuine missing-input boundaries. + +The initial source `0bbc6a7` passed one focused test in 17.614621813 seconds. +Independent review requested stronger backup permission/final-boundary checks, +then cleared the strengthened source `d02f034` and independently passed one test +in 17.012795836 seconds. Both runs had zero failures, errors or skips. Raw state +across 43 tables, sequences, files and modes, three controlled restore +interruptions, exact SDK/CLI result collection and zero replay passed. Exact +owned-schema/database cleanup was independently queried; the task PostgreSQL +container is stopped. See `reports/integrated-interrupted-restore-20261021/`. +Current published-head hosted checks remain a merge gate; no full T8, performance +or lab acceptance is inferred. + ## 2026-10-20 — User removes backward-compatibility requirement The user explicitly states that existing projects must perform their own one-time diff --git a/docs/plans/lab-workflow-reliability/plan.md b/docs/plans/lab-workflow-reliability/plan.md index 048696aa..ea002de9 100644 --- a/docs/plans/lab-workflow-reliability/plan.md +++ b/docs/plans/lab-workflow-reliability/plan.md @@ -429,6 +429,31 @@ arguments are not substituted for old source execution. No production, schema, workflow or README change is admitted. Actual qualification and independent review remain gates; this does not complete unrelated T8 or lab acceptance. +### Current-version interrupted-transfer acceptance admission — 2026-10-21 + +After PR116 merged as `8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`, the +integration lead selects the remaining interrupted-restore part of T8. Extend +the existing combined current-version witness, rather than duplicate its real +neutral attempt. The isolated branch is +`test/integrated-interrupted-restore-20261021`. Its write set is the existing +`tests/test_integrated_retention_restore.py`, this plan, execution ledger and +scoped evidence under `reports/integrated-interrupted-restore-20261021/`. + +Inject explicit failures after real catalog population before transaction commit, +after committed metadata and an actual partial file copy, and after the final +destination rename before its reply. At every safe checkpoint, reconcile raw +tables/sequences and the exact transfer marker; prove the source and backup +remain unchanged and the final destination is absent until publication. Retry +the same transfer and preserve all bytes, permissions, a nontrivial sequence +high-water mark, immutable coordinator acknowledgement, timing records and exact +publication Ref. The existing cold SDK/CLI reads must still launch no compute. +No production change is admitted without an observed correctness failure. + +Original and independent focused qualification, source review and current-head +hosted checks remain gates. This advances current-version T8 integrity only; it +does not revive backward-compatibility requirements, qualify preparation reuse, +close all M2/M3/M5/T8 boundaries, or supply missing PUB-1/SCI-1 lab evidence. + ### Integrated stopped-transfer acceptance admission — 2026-10-20 The fresh integration lead admits one narrow T8 witness from the exact reporting diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/acceptance.md b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/acceptance.md new file mode 100644 index 00000000..68bf4ace --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/acceptance.md @@ -0,0 +1,92 @@ +# Current-version interrupted retention restore + +The existing T8 witness now proves controlled interruption and retry for the +combined current-version coordinator, timing, publication and client state. No +production repair was needed. This is an extension of the existing plan, from +merged main `8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e`. + +## Exact qualification subjects + +The initial source `0bbc6a7c4cd58dc0451d6ba6c0983193763e1f91` passed one focused +test in 17.614621813 seconds, with zero failures, errors or skips. Independent +review then requested backup permission checks at each checkpoint and the final +source/backup boundary, plus explicit sequence-fixture provenance. These were +added without changing a restore owner or fault seam. + +The strengthened source `d02f034aa1bba6cd55dfd47ae44ebcb7014200a5`, tree +`68f147908523a65ce495f47392eada87273cbfaf`, passed independent reproduction: +one test in 17.012795836 seconds, zero failures, errors or skips. Its test file +SHA256 is `50ac2975b216052dc0d07248db9f21a8618f9cf096b22214c2c9a317c302d199`. +The independent source/reproduction review is clear; receipt SHA256 is +`92cc82aa677eeaca9496fe57af2997c8b45e95da58cba9fdb2778f5cb9df0700`. +The initial root pass is not relabeled as a run of the strengthened source. + +Raw runners, logs, result/domain receipts and the preliminary/final review are +preserved byte-for-byte under `root/` and `independent/`. The +[manifest](evidence-manifest.json) seals their sizes, hashes, exact source +subjects and the small root-to-final assertion diff. Report-only publication +must preserve the qualified test bytes; current published-head hosted checks +and final report/source review remain merge gates. + +## Real boundaries and independent state oracles + +The existing fixture executes one actual neutral Linux custodian attempt, +drains it, faults original publication copying, obtains a real PostgreSQL +advisory-lock coordinator fault and performs explicit same-Run recovery. It then +stops the source, backs up raw schema 10 state and restores it into a separate database. + +| Interruption | Observed safe checkpoint | Same-transfer continuation | +| --- | --- | --- | +| After real catalog population, before transaction commit | All 43 raw table states roll back to empty staging state; final destination absent; exact private restore marker retained. | Populate and restore original metadata/sequence state. | +| After metadata commits and the actual coordinator-journal file is copied and renamed | All raw tables/sequences equal source; copied inventory is a strict subset of complete original entries; final destination absent. | Verify existing entries and copy the remaining files. | +| After the actual final destination directory rename, before function return | Destination is published, staging absent, exact marker/tables/sequences/tree/POSIX modes retained. | Normal retry reconciles the completed result without overwriting it. | + +At every checkpoint and after final cold reads, source bytes, source POSIX modes, +backup bytes, backup/catalog POSIX modes, source raw tables and source sequences +remain unchanged. The source values and inventories were independently captured +before restore; expected raw state does not come from a restore result or the +production timing summarizer. + +The sequence high-water mark is explicit authored administrative fixture state: +`generation_jobs` starts at value 1/not-called and is set to value 20/called with +empty job-order rows. Its exact retained state survives committed transfer, +retry and cold reads. This is not a measurement of a real lab workload or proof +of arbitrary populated-sequence transactional rollback. + +After reconciliation, actual application activation preserves the immutable +completed coordinator acknowledgement, original interrupted Run and events, +exact recovered publication Ref, and measured/incomplete timing records. +Authenticated current SDK and separate CLI processes collect identical complete +artifact bytes into mode 0600 files. All raw tables/sequences remain unchanged; +the compute-launch tripwire records zero calls during restore and consumption. +The fixture has one Run, one grant/consumption and zero Study reservations; it +does not replace the separate populated-Study acceptance evidence. + +Both runs leave schema inventory empty and preserve the complete database-name +inventory. The independent reviewer also queried the exact emitted schema and +restored database names and confirmed both absent. The task-owned PostgreSQL 17.11 +container was stopped after the serial test windows. No benchmark, broad local +suite, workflow rerun or old-version witness was launched for this extension. + +## Retention decisions and remaining boundaries + +The earlier real collection measurements remain the performance evidence. No +new performance comparison was run or inferred here. Explicit twenty-item batch +collection retains its scoped benefit; individual collection stays the default, +including the measured slower single-item batch controls. Preparation reuse, +installed-runtime mutation, other batch sizes and general workflow speed are +not qualified by this restore witness. + +This qualifies controlled Python exceptions and retries, not host crashes, kill, +power loss or device/fsync durability. Partial copying here means a subset of +complete verified files, not a torn payload. POSIX mode bits are compared; +ownership, ACLs, xattrs and timestamps are not newly qualified. Full M2/M3/M5/T8 +acceptance is not implied by these three boundaries. + +The user's revised support policy remains authoritative: older project owners +perform their own one-time migration; no backward adapters or automatic +migration framework are added. Current-version schema validation, explicit +unsupported-version errors, backup/restore integrity and no replay remain +required. PUB-1 still needs its original real failed Run, custody, profile and +output inventory. SCI-1 still needs a real comparison pair, criterion authority +and environment/procedure. Neutral recovery evidence cannot supply those inputs. diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/evidence-manifest.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/evidence-manifest.json new file mode 100644 index 00000000..14563252 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/evidence-manifest.json @@ -0,0 +1,50 @@ +{ + "baseline": "8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e", + "original_root_source": "0bbc6a7c4cd58dc0451d6ba6c0983193763e1f91", + "reviewed_source": "d02f034aa1bba6cd55dfd47ae44ebcb7014200a5", + "reviewed_tree": "68f147908523a65ce495f47392eada87273cbfaf", + "test_sha256": "50ac2975b216052dc0d07248db9f21a8618f9cf096b22214c2c9a317c302d199", + "root_to_final_diff": "diff --git a/tests/test_integrated_retention_restore.py b/tests/test_integrated_retention_restore.py\nindex 62d81d8..d0023dd 100644\n--- a/tests/test_integrated_retention_restore.py\n+++ b/tests/test_integrated_retention_restore.py\n@@ -147,6 +147,7 @@ class IntegratedRetentionRestoreTests(unittest.TestCase):\n self.assertEqual(tree_manifest(source.path), original_tree)\n self.assertEqual(tree_modes(source.path, original_tree), original_modes)\n self.assertEqual(tree_manifest(backup), backup_tree)\n+ self.assertEqual(tree_modes(backup/'catalog', original_tree), original_modes)\n original = Metadata(source.path, dsn=original_dsn, create=False)\n try:\n self.assertEqual(read_catalog(original), original_tables)\n@@ -320,14 +321,7 @@ class IntegratedRetentionRestoreTests(unittest.TestCase):\n launch.assert_not_called()\n restored.close()\n \n- self.assertEqual(tree_manifest(source.path), original_tree)\n- self.assertEqual(tree_manifest(backup), backup_tree)\n- original = Metadata(source.path, dsn=original_dsn, create=False)\n- try:\n- self.assertEqual(read_catalog(original), original_tables)\n- self.assertEqual(read_sequences(original), original_sequences)\n- finally:\n- original.close()\n+ unchanged_originals()\n drop_restored_database()\n drop_source()\n if report_path := os.environ.get('TULDOK_RETENTION_ACCEPTANCE_REPORT'):\n@@ -342,6 +336,8 @@ class IntegratedRetentionRestoreTests(unittest.TestCase):\n 'artifact_sha256': hashlib.sha256(fixture.input).hexdigest(), 'artifact_bytes': len(fixture.input),\n 'raw_transfer_equal': True, 'source_and_backup_unchanged': True,\n 'restore_interruptions': interruptions, 'raw_permissions_equal': True,\n+ 'authored_sequence_fixture': {'table': 'generation_jobs', 'before': sequence,\n+ 'retained': original_sequences['generation_jobs']},\n 'cold_reads_and_collection_preserve_all_tables_and_sequences': True,\n 'compute_replay_calls': launch.call_count, 'cleanup': cleanup}\n with Path(report_path).open('x') as stream:\n", + "evidence": { + "root/runner.py": { + "bytes": 2768, + "sha256": "f68a94afd7be2985d2175fc65619933d387449b93dbd233901f39cd476128c21" + }, + "independent/runner.py": { + "bytes": 4159, + "sha256": "1f8ccc36f96c8e56857cfe1ac73891a7910f3d50c563ae93f247edc175df8618" + }, + "independent/preliminary-source-review.json": { + "bytes": 3198, + "sha256": "09c67142786156a91b8e9861c2331de2f21c7eb262397d5deb247f097d82b76c" + }, + "independent/source-and-reproduction-review.json": { + "bytes": 6631, + "sha256": "92cc82aa677eeaca9496fe57af2997c8b45e95da58cba9fdb2778f5cb9df0700" + }, + "root/result.json": { + "bytes": 816, + "sha256": "65176335b275a669885aa9dba91ca4ba1215aacbfc761d7060069fff68179c17" + }, + "independent/result.json": { + "bytes": 1190, + "sha256": "50b558e1c8138b77f7f4634f3202ed01ca15cc13345149c5231554da33ecce6d" + }, + "root/domain.json": { + "bytes": 18883, + "sha256": "c1f0fd6d54a8b0bbb7658e4def4836342bf005f76f5a7ac5fae758289e8de21d" + }, + "independent/domain.json": { + "bytes": 19096, + "sha256": "9b3f56f9fcfaee6c8934f1daa9abb7166b7b5c2c727019081a11b5037e9caf85" + }, + "root/unittest.log": { + "bytes": 329, + "sha256": "51f85c0c2f20c80697cf1843ba84c3fe9aebd8d78942440ef7f6c79540212955" + }, + "independent/unittest.log": { + "bytes": 329, + "sha256": "34169a1efd6154fe6f00c8ccb016b48c0e9a6097b8ecdbec54776f00a40d1cce" + } + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/domain.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/domain.json new file mode 100644 index 00000000..8247bda1 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/domain.json @@ -0,0 +1,665 @@ +{ + "format": "integrated_retention_restore_acceptance_v2", + "scope": "Real neutral custody, PostgreSQL stopped transfer and current HTTP/SDK/CLI; no science or performance claim", + "schema_version": 10, + "catalog_id": "58ccf61daf6b44b581e785b21295ff79", + "run_id": "77db03cbf9294906b5921388e6dd366f", + "result_ref": { + "kind": "recovered_publication_result", + "schema_version": 1, + "catalog_id": "58ccf61daf6b44b581e785b21295ff79", + "run_id": "77db03cbf9294906b5921388e6dd366f", + "material_sha256": "829e505e0135cb1f2cda19fa5a2a4597cd6fe586433c426ffa41f22500131a06", + "inventory_sha256": "84afb1364c2ecd99dbc7b34258f2d331b576f8e49b96aebd980cd73dd1450d84", + "custody_receipt_sha256": "ecdf4a97e4642a071c5928c666cb6190e3ef0bc07530b25a120571cb14248bd2", + "sha256": "ec2317d166bcd83a3b94f7336fe613759c35dcd9cefe3aea0c8f01b64cec729e" + }, + "table_summaries": { + "access_actors": { + "columns": [ + "id", + "kind", + "label", + "created_at" + ], + "count": 4, + "row_sha256": [ + "7349ad411c540158758c176f104cf98aa1d14ac56914bf467a3447a29c797c95", + "a04acd9f1ae27d96788d39b7eb6315dc10c6327859a3c69a1bb5037cf54b4a27", + "2db86771f7381fb276181be74c6969c20fa1efedd299eddf2a645f90738a6bc0", + "934f8a245d8e8c9906a0ae0de3cecc5962e2127e39a1a9317c6de60879fcba43" + ], + "sha256": "9bf2864091ef1c4aca1bf7417ca50c1c5d9933eb318d165291e12a0e2c197aab" + }, + "access_admissions": { + "columns": [ + "id", + "catalog_id", + "actor_id", + "credential_id", + "kind", + "job_id", + "request_digest", + "effects_json", + "created_at" + ], + "count": 1, + "row_sha256": [ + "937d6f9f16902b87b383906f69609bef564bbd1b633f7973f9cfc7cd18a7788a" + ], + "sha256": "69a8d3aef386920550a93c8bbd3805fd3b92f8b097411c641a4be3bce4912528" + }, + "access_audit": { + "columns": [ + "id", + "actor_id", + "credential_id", + "action", + "target_json", + "details_json", + "result", + "created_at" + ], + "count": 19, + "row_sha256": [ + "da342e0bd7423e66317ec43f594c22cb62deba00e60a0f7a4a170bcedd14aa79", + "7f5a7a35eb43e51809e37641f047f7241ade917ac33022360e044ee69763f816", + "2d5001a2291e8dd595b2db3d13ad95741a5650d385665a4b813a26df9eddd975", + "2939ab6ec732f6b0a974e4c996d9bb2f5e3a0d4a45c32905c73c7caeb342f667", + "eac1fe2a1d7acde67b468293c181cf20c946ec17ae1fc5111302d9844b3cd2e0", + "791b6b8f0dc4058297e63b9492d638ca3a7c230dac7f759b36d6692cdc4272a8", + "e8ca32187b7006ad8ef60fa21250396d3393cacc6ef69c046753c03422b8dc7c", + "0fb76bc01b3a69f8184e2081495e19b567162a3b47be8260750b993d9f191473", + "2fd56da5f4ec78d299521b62f3156aa33c5f8abc7690e78b8778be32872627f7", + "871fff37224508a807b856789bc87f2649202587dfa069cefd883d6e4e003cc5", + "e4ea9688e4c6723be909c23f13039613de63711cc70d8bcc4eb6418b9f434a63", + "802961e0cb9cf0ef756d6a1108b1b00b08c1da258fd5daae2dbfe7a2feea31d8", + "e4f34683548f90dede62bcc5e07bcf77bcd34ae6a4eaf984f54767db9107b195", + "4b91b103c4202cd28b7f268796b1f9328654f442347a5fa8197abc4a268b8dc5", + "1746e80d7b71254e6bba59920fef7f407bcd6d54eff67b0ee5f5afd289529886", + "a15eb3eaa8cf34c964cba720c8c0c29698fd08e05200c45db74ae53d62c42a21", + "65ccab1e4926ea8017d3793c5a69b1da2f938d1755c3781e4ad41e244cde3683", + "68b11f07c05c6ebbacf8d778b8bca539f6034850cd348b4b5d2f66a758a8eba4", + "423a66d64088b92805146095959075621cf01feb1160803142651e46e982217c" + ], + "sha256": "24e2bbbcc727e6ecd0158907ba2d54c596a8f95f3c90ae5623e477e853d55128" + }, + "access_credentials": { + "columns": [ + "id", + "actor_id", + "token_sha256", + "kind", + "scopes_json", + "expires_at", + "revoked", + "consumed", + "created_at" + ], + "count": 5, + "row_sha256": [ + "6662fe4cfb5efccf66278bf7f4fa1312cc62891f03d677a32895c9f32bd0bf89", + "8aae2d00a16113a7f67bada6ccbc9a74d5c20c76b10d360ebaff9803dfa3e112", + "f2b6cbc0695e2d520785091d2610bfb0d17f760b820aa8c13018bde8f223280b", + "8a6ebdd98b7ca700a185c87ee98c41ecf839ea4fc670ef523d00bfe184c2241e", + "8fe42128f460c437dd4e396bb77eb9f26f13472c13747edbd4f27a09394757e4" + ], + "sha256": "f2acae779925959cf6af22f4f76a314dc982f43f32eb9d03baf52f31112ec2bc" + }, + "access_execution_consumptions": { + "columns": [ + "grant_id", + "run_id", + "admission_id", + "consumer_credential_id", + "request_sha256", + "consumed_at" + ], + "count": 1, + "row_sha256": [ + "56871ec5a8d81792625df440280cc0cef435539ed4519a430eb02c448fdd9d83" + ], + "sha256": "650dd95344c8bb4b5b38dfc6158edc3dc19efd8c3b861e2bd3abbe8a77b437bc" + }, + "access_execution_grants": { + "columns": [ + "id", + "catalog_id", + "approver_actor_id", + "approver_credential_id", + "consumer_actor_id", + "material_sha256", + "created_at", + "expires_at", + "revoked_at" + ], + "count": 1, + "row_sha256": [ + "37d8af71d347afbf00e9a2ddbf1c4b057ead48ea238d9898ab4af3c093a5180e" + ], + "sha256": "c8c9c6b747fe4ac91891d23f3bfa94e058f7054df06859dccdcf163541d34b5d" + }, + "caption_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "generation_entries": { + "columns": [ + "id", + "job_id", + "ordinal", + "data", + "storage_order" + ], + "count": 0, + "row_sha256": [], + "sha256": "530faef7a9e5031599c72af9332f8c2472584ffd0d4110ab97eb445c3a9680dd" + }, + "generation_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "grounded_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "model_attempt_reservations": { + "columns": [ + "id", + "study_id", + "actor_id", + "request_key", + "request_sha256", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "952235730407a326f927ed027d547c534463ac174ce800ccd0f2630e7b02fe01" + }, + "model_candidates": { + "columns": [ + "id", + "study_id", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "8e150598074027645a884f8c83471a910abaabfaca5017c5154bed5b5f6ea480" + }, + "model_definition_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "model_materials": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a519823d3f1dcbcb57d09c23d8ecf7e7ceb4b534d649da6804084448530962c2" + }, + "model_operation_receipts": { + "columns": [ + "actor_id", + "operation_id", + "request_key", + "request_sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "58bf43af0b5efd8c06272e144beb72720bbdd6204e120fd4708e739555fb899c" + }, + "model_selection_policies": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a519823d3f1dcbcb57d09c23d8ecf7e7ceb4b534d649da6804084448530962c2" + }, + "model_studies": { + "columns": [ + "id", + "revision", + "admission_epoch", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a50d931d7ec23487169070a69653a1b933dcc1d1985a8f3b60ff1e4b7091f94c" + }, + "model_study_memberships": { + "columns": [ + "study_id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "4b114a32959ae2c599d50d18ebfeccf1c757d45c03518d95820aa463c6b85c98" + }, + "model_trial_lineage": { + "columns": [ + "trial_id", + "parent_trial_id", + "baseline_trial_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ce519910146db157ed46ee02a83bc6add785136de22e6b9cd48581c756299ec0" + }, + "model_trials": { + "columns": [ + "id", + "study_id", + "material_sha256", + "replicate", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "d790df0190b101dba64d6025d8c10072d436e400152aad4059d8de5665f0545c" + }, + "samples": { + "columns": [ + "id", + "sha256", + "filename", + "width", + "height", + "book_id", + "session_id", + "split", + "created_at", + "updated_at", + "revision", + "annotation" + ], + "count": 0, + "row_sha256": [], + "sha256": "45cba19d248a0b126119bc05246fdfe6459c95d5cce245ec3a4cd47d1b5e2995" + }, + "saved_searches": { + "columns": [ + "id", + "name", + "revision", + "criteria_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "caa62ec987589481b04e93502d8f340a1a41703997b68a0c94df642ee8e5a8a6" + }, + "saved_selections": { + "columns": [ + "id", + "name", + "revision", + "members_json", + "member_count", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "ef015a7aae83417e2ee6b9aa54c0f3f618358296ed201682901be8c0841b7e11" + }, + "text_classification_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "workbench_deleted_sources": { + "columns": [ + "id", + "book_id", + "session_id", + "split" + ], + "count": 0, + "row_sha256": [], + "sha256": "c1c95154abd259591b21272e5324695db12a227e849c3fbd46d96ea2e554a7c2" + }, + "workbench_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_mesh_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_pointcloud_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_preference_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_preferences": { + "columns": [ + "id", + "prompt_id", + "revision", + "data" + ], + "count": 0, + "row_sha256": [], + "sha256": "67b197c3ca40fbcbe1baf7d841be28c4e9c98b3ef3f22178c7732560f66e16c3" + }, + "workbench_records": { + "columns": [ + "id", + "kind", + "name", + "text", + "original_text", + "content_hash", + "pixel_hash", + "groups_json", + "parents_json", + "provenance_json", + "task", + "annotation_json", + "review", + "revision", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "e2e5e5e9cef6974f035aa0b76e15e13932e11b2f3fdb93574cf2791aacb19259" + }, + "workbench_response_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_responses": { + "columns": [ + "id", + "prompt_id", + "revision", + "completion", + "review", + "provenance_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "4eb68800cc161c30f33a81c52c233a59559a12f1a3744545088bf3cbe9930d67" + }, + "workbench_rights_notes": { + "columns": [ + "id", + "note", + "revision" + ], + "count": 0, + "row_sha256": [], + "sha256": "894e92255b0040027f355105f88c51cff087e8471008b30731a6e0e849c3cdc3" + }, + "workbench_sequence_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_target_proposals": { + "columns": [ + "id", + "evidence" + ], + "count": 0, + "row_sha256": [], + "sha256": "c610988ec09c1a996fba5c2d51051fd38b37a1b51d1defc6ea9647664742838e" + }, + "workbench_volume_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workspace_exposures": { + "columns": [ + "id", + "record_id", + "source_hash", + "reason", + "run_id", + "release_id", + "created_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "5bf95a047db8503a858f55e8095cb28322d6329af09b545dc4f67a3cb83e8599" + }, + "workspace_graphs": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "362309ffa0971284b206baefe87157df8a7cadfe57de19167c7bcfa60c158a73" + ], + "sha256": "6017ff1c1cfd07d4f3e7e0e130a38c20ddf880d1b788dbb8921839f2594a79d1" + }, + "workspace_projects": { + "columns": [ + "id", + "document" + ], + "count": 1, + "row_sha256": [ + "e4cead70471a5922596485e3c4aa89b0c3e61b7095f698bde5bbff68e7cc479b" + ], + "sha256": "b22e3acdb81513f135d1ef02a04e71a2d6db1379049122040d3b6fb204ecb697" + }, + "workspace_recipe_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "workspace_recipe_materials": { + "columns": [ + "sha256", + "recipe_id", + "recipe_revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "7b2b46c5091c5e07e25b8648e2161ed0253aae41384b82f7923aba99bb5f04d0" + }, + "workspace_runs": { + "columns": [ + "id", + "request_key", + "request_sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "cf340f870ad8715c188efec480597a9f5f4fe3f9f5fcceb7e25288ed8dc9f1fd" + ], + "sha256": "05d93df836be84537e05af58549c9f606d1c1072a1f8cf19f24dc2ab00e4f2e3" + } + }, + "sequences": { + "generation_jobs": { + "last_value": 20, + "is_called": true + }, + "grounded_jobs": { + "last_value": 1, + "is_called": false + }, + "caption_proposals": { + "last_value": 1, + "is_called": false + }, + "text_classification_proposals": { + "last_value": 1, + "is_called": false + }, + "generation_entries": { + "last_value": 1, + "is_called": false + } + }, + "catalog_tree_sha256": "1b2dba4f8fd97fd9a41238e04075e6b82f8d51cf9e6d1d1fbe050c98c0ae843f", + "backup_tree_sha256": "b679b08e3c1a798456041b315ed336e28ae17baf806a6f860331124dccc368eb", + "journal_sha256": "d0ff910942362c8260eb9418026cb3a36b3ba4a44678aece9f6354a6a1d21156", + "retained_timing_sha256": "460d9dc7b814d5dbc4d52bef7824803725cda37002978151cd77da9c1ad5cb39", + "recovery_request": { + "actor_id": "d65ec61a214f4b39899e9fd2b5f7910a", + "completed_generation": 2, + "expected_generation": 1, + "request_key": "transfer-coordinator-recovery-0001", + "request_sha256": "7e484717aefbb27ee9e23cc37e10d115b41464ff53f05a04177ee7610c1bdcd0" + }, + "stored_span_states": [ + "incomplete", + "measured" + ], + "artifact_sha256": "54ae9355f214d827afd4ec287ff813d3af22683a52c10e9ed1ea62507cfa4d10", + "artifact_bytes": 55, + "raw_transfer_equal": true, + "source_and_backup_unchanged": true, + "restore_interruptions": [ + { + "seam": "metadata_transaction_rollback", + "destination_published": false, + "table_sha256": "20ce31ed06a9ac732d86432973dfd4427d55a939c72b0ccdf1797979ae7322c5", + "sequence_sha256": "915979a8c675d1c5bb78aef6cc05730ce55215dc052fcc0a43c88459b5cbda19", + "tree_sha256": "bd9b3529c33398f26f5d2a0192556f92fea7d3b4fc559242db22e1b2e8639cbc", + "source_and_backup_unchanged": true + }, + { + "seam": "committed_metadata_partial_files", + "destination_published": false, + "table_sha256": "c1154a34b12ce88d578acd51ceef63f9e86300b83cd8956cd7988db1e9f18e2e", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "f27acd85aa73d33f5933cb919f884f976fb3de299a6460b5038f16935f53cd74", + "source_and_backup_unchanged": true + }, + { + "seam": "published_destination_lost_reply", + "destination_published": true, + "table_sha256": "c1154a34b12ce88d578acd51ceef63f9e86300b83cd8956cd7988db1e9f18e2e", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "1b2dba4f8fd97fd9a41238e04075e6b82f8d51cf9e6d1d1fbe050c98c0ae843f", + "source_and_backup_unchanged": true + } + ], + "raw_permissions_equal": true, + "authored_sequence_fixture": { + "table": "generation_jobs", + "before": { + "last_value": 1, + "is_called": false + }, + "retained": { + "last_value": 20, + "is_called": true + } + }, + "cold_reads_and_collection_preserve_all_tables_and_sequences": true, + "compute_replay_calls": 0, + "cleanup": { + "restored_database_removed": "retention_restore_1094643898f14f1495c439ccbf8a8f08", + "source_schema_removed": "catalog_58ccf61daf6b44b581e785b21295ff79" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/preliminary-source-review.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/preliminary-source-review.json new file mode 100644 index 00000000..a0fd11da --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/preliminary-source-review.json @@ -0,0 +1,34 @@ +{ + "format": "independent_interrupted_restore_source_review_v1", + "status": "preliminary_source_review_strengthening_agreed_pending_updated_head", + "reviewed_commit": "0bbc6a7c4cd58dc0451d6ba6c0983193763e1f91", + "reviewed_tree": "b2e7b8196a52b283fc79c93ff7fcdfdf00201f5a", + "baseline": "8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e", + "scope": "Read-only independent source review; no tests, PG queries, benchmarks or network writes performed by reviewer.", + "blocking_production_findings": [], + "requested_strengthening": [ + { + "file": "tests/test_integrated_retention_restore.py", + "at": "unchanged_originals() and post-cold-read boundary", + "issue": "Backup content equality is asserted at each checkpoint but backup/catalog mode equality is only checked once before restore. Final repeated source/backup comparison also omits source modes.", + "recommendation": "Assert backup/catalog tree_modes equals original_modes in unchanged_originals(); use unchanged_originals() at the final post-cold-read boundary.", + "root_response": "Agreed; preserve running original subject, update after termination, then independent reproduction at final exact subject." + } + ], + "reviewed_invariants": [ + "After real raw catalog population, authored exception leaves private staging and triggers SQL transaction rollback before final destination publication.", + "Partial-copy injection first performs the actual journal-file rename; strict subset tree and per-entry original equality prove an incomplete copied inventory.", + "Lost-return injection first performs the actual final destination rename; later exact marker, raw table/sequence/tree/mode comparison and idempotent restore check the published result.", + "Cold app activation may advance operational generation while immutable completed acknowledgement, original interrupted Run/events, exact attached result Ref, retained measured/incomplete timing records and current public SDK/CLI reads stay bound.", + "The independent raw source/backup tables, sequence states, bytes and mode snapshots are expected state; restored state is not accepted through a manufactured domain completion.", + "All source and destination resources have exact fixture identity and bounded cleanup callbacks; stopped server/app handles precede dropping the owned restored database.", + "No new old-version witness, migration framework, production/schema/README/workflow change, cache, science proof or whole-workflow speed claim." + ], + "limits": [ + "Only a controlled Python exception after final rename is exercised; no host crash, kill, power loss or storage-device durability result is claimed.", + "The partial copy is a proper subset of complete verified files, not a torn file payload.", + "The generation_jobs high-water mark is explicitly authored administrative fixture state; it is not workload/performance evidence.", + "The fixture has no populated job-order rows; raw sequence equality does not establish transaction rollback semantics for arbitrary nonempty sequences.", + "PUB-1/SCI-1 missing real lab inputs remain absent; no full T8/M2/M3/M5 acceptance is established by this witness." + ] +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/result.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/result.json new file mode 100644 index 00000000..354534eb --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/result.json @@ -0,0 +1,43 @@ +{ + "format": "independent_interrupted_restore_reproduction_v1", + "source": { + "commit": "d02f034aa1bba6cd55dfd47ae44ebcb7014200a5", + "tree": "68f147908523a65ce495f47392eada87273cbfaf", + "test_sha256": "50ac2975b216052dc0d07248db9f21a8618f9cf096b22214c2c9a317c302d199", + "python": "3.12.14", + "psycopg": "3.3.6" + }, + "tests_run": 1, + "successful": true, + "failures": 0, + "errors": 0, + "skips": 0, + "unittest_runner_seconds": 17.012795835998986, + "resources_before": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "resources_after": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "source_unchanged": true, + "log_sha256": "34169a1efd6154fe6f00c8ccb016b48c0e9a6097b8ecdbec54776f00a40d1cce", + "domain_sha256": "9b3f56f9fcfaee6c8934f1daa9abb7166b7b5c2c727019081a11b5037e9caf85", + "independent_exact_cleanup_query": { + "source_schema_absent": true, + "restored_database_absent": true + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/runner.py b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/runner.py new file mode 100644 index 00000000..2f9eb157 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/runner.py @@ -0,0 +1,76 @@ +import contextlib +import hashlib +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +import unittest + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path('/workspace/Tuldok-interrupted-restore-20261021') +EXPECTED_HEAD = 'd02f034aa1bba6cd55dfd47ae44ebcb7014200a5' +OUT = Path(__file__).parent/'reproduction-d02f034' + +def git(*arguments): + return subprocess.check_output(['git', *arguments], cwd=ROOT, text=True).strip() + +assert git('rev-parse', 'HEAD') == EXPECTED_HEAD +assert not git('status', '--porcelain=v1') +OUT.mkdir(mode=0o700, exist_ok=False) +container = json.loads(subprocess.check_output(['docker', 'inspect', 'tuldok-qa-20261020']))[0] +assert container['State']['Running'] +settings = dict(value.split('=', 1) for value in container['Config']['Env'] if '=' in value) +dsn = make_conninfo(host='127.0.0.1', port=55432, + dbname=settings.get('POSTGRES_DB', 'postgres'), + user=settings.get('POSTGRES_USER', 'postgres'), password=settings['POSTGRES_PASSWORD']) +os.environ['TULDOK_DATABASE_URL'] = dsn +os.environ['TULDOK_RETENTION_ACCEPTANCE_REPORT'] = str(OUT/'domain.json') + +def inventory(): + with psycopg.connect(dsn, autocommit=True) as connection: + return { + 'schemas': [row[0] for row in connection.execute("SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname NOT IN ('public','information_schema') ORDER BY nspname")], + 'databases': [row[0] for row in connection.execute('SELECT datname FROM pg_database ORDER BY datname')], + 'server_version': connection.execute('SHOW server_version').fetchone()[0]} + +before = inventory() +assert before['schemas'] == [], before +source = {'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'test_sha256': hashlib.sha256((ROOT/'tests/test_integrated_retention_restore.py').read_bytes()).hexdigest(), + 'python': platform.python_version(), 'psycopg': psycopg.__version__} +sys.path[:0] = [str(ROOT/'tests'), str(ROOT)] +os.chdir(ROOT) +with (OUT/'unittest.log').open('x') as stream: + with contextlib.redirect_stdout(stream), contextlib.redirect_stderr(stream): + suite = unittest.defaultTestLoader.loadTestsFromName('test_integrated_retention_restore.IntegratedRetentionRestoreTests.test_stopped_transfer_retains_recovery_timing_and_exact_publication_consumers') + start = time.perf_counter() + result = unittest.TextTestRunner(stream=stream, verbosity=2).run(suite) + seconds = time.perf_counter()-start +after = inventory() +record = {'format': 'independent_interrupted_restore_reproduction_v1', 'source': source, + 'tests_run': result.testsRun, 'successful': result.wasSuccessful(), + 'failures': len(result.failures), 'errors': len(result.errors), 'skips': len(result.skipped), + 'unittest_runner_seconds': seconds, 'resources_before': before, 'resources_after': after, + 'source_unchanged': git('rev-parse', 'HEAD') == EXPECTED_HEAD and not git('status', '--porcelain=v1'), + 'log_sha256': hashlib.sha256((OUT/'unittest.log').read_bytes()).hexdigest()} +if (OUT/'domain.json').exists(): + domain = json.loads((OUT/'domain.json').read_bytes()) + record['domain_sha256'] = hashlib.sha256((OUT/'domain.json').read_bytes()).hexdigest() + names = domain.get('cleanup', {}) + with psycopg.connect(dsn, autocommit=True) as connection: + record['independent_exact_cleanup_query'] = { + 'source_schema_absent': connection.execute('SELECT 1 FROM pg_namespace WHERE nspname=%s', (names.get('source_schema_removed'),)).fetchone() is None, + 'restored_database_absent': connection.execute('SELECT 1 FROM pg_database WHERE datname=%s', (names.get('restored_database_removed'),)).fetchone() is None} +with (OUT/'result.json').open('x') as stream: + json.dump(record, stream, indent=2) + stream.write('\n') +print(json.dumps(record)) +assert before == after +assert record['source_unchanged'] +assert result.testsRun == 1 and result.wasSuccessful() and not result.skipped +assert record['independent_exact_cleanup_query'] == {'source_schema_absent': True, 'restored_database_absent': True} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/source-and-reproduction-review.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/source-and-reproduction-review.json new file mode 100644 index 00000000..6131928e --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/source-and-reproduction-review.json @@ -0,0 +1,135 @@ +{ + "format": "independent_interrupted_restore_final_review_v1", + "disposition": "clear_focused_current_version_restore_subject_hosted_checks_pending", + "blocking_findings": [], + "reviewed_commit": "d02f034aa1bba6cd55dfd47ae44ebcb7014200a5", + "reviewed_tree": "68f147908523a65ce495f47392eada87273cbfaf", + "baseline": "8b1f78eb06c17e3dba2a73dc0acc9568e53ccc7e", + "test_sha256": "50ac2975b216052dc0d07248db9f21a8618f9cf096b22214c2c9a317c302d199", + "source_review": [ + "Only the existing combined current-version test, existing plan and ledger differ from baseline; no application/schema/workflow/README change.", + "Actual raw PostgreSQL population precedes the authored first failure; all 43 table states return to the initially empty private staging state, with no final destination.", + "Second authored failure follows a real journal-file rename after raw metadata/sequence commit; the copied inventory is a strict subset of the original with every copied entry equal.", + "Third authored failure follows the real final directory rename; exact marker/raw tables/sequences/tree/modes and a normal retry reconcile the completed destination.", + "Source and backup/catalog byte and POSIX-mode checks now run at every interruption and final post-cold-read boundary; preliminary strengthening is resolved.", + "Current app activation keeps immutable completed recovery acknowledgement, original interrupted Run/events, recovered publication Ref and measured/incomplete timing records; actual SDK and separate CLI collect the same exact bytes without compute replay.", + "The authored generation_jobs high-water fixture is explicit; its retained value20/is_calledtrue survives committed state, retry and cold consumer reads.", + "No backward-compatibility/automatic-migration witness, synthetic PUB1/SCI1 acceptance, preparation-cache claim or performance result is introduced." + ], + "reproduction": { + "format": "independent_interrupted_restore_reproduction_v1", + "source": { + "commit": "d02f034aa1bba6cd55dfd47ae44ebcb7014200a5", + "tree": "68f147908523a65ce495f47392eada87273cbfaf", + "test_sha256": "50ac2975b216052dc0d07248db9f21a8618f9cf096b22214c2c9a317c302d199", + "python": "3.12.14", + "psycopg": "3.3.6" + }, + "tests_run": 1, + "successful": true, + "failures": 0, + "errors": 0, + "skips": 0, + "unittest_runner_seconds": 17.012795835998986, + "resources_before": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "resources_after": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ], + "server_version": "17.11 (Debian 17.11-1.pgdg13+2)" + }, + "source_unchanged": true, + "log_sha256": "34169a1efd6154fe6f00c8ccb016b48c0e9a6097b8ecdbec54776f00a40d1cce", + "domain_sha256": "9b3f56f9fcfaee6c8934f1daa9abb7166b7b5c2c727019081a11b5037e9caf85", + "independent_exact_cleanup_query": { + "source_schema_absent": true, + "restored_database_absent": true + } + }, + "domain_inspection": { + "schema_version": 10, + "table_count": 43, + "seams": [ + { + "seam": "metadata_transaction_rollback", + "destination_published": false, + "table_sha256": "20ce31ed06a9ac732d86432973dfd4427d55a939c72b0ccdf1797979ae7322c5", + "sequence_sha256": "915979a8c675d1c5bb78aef6cc05730ce55215dc052fcc0a43c88459b5cbda19", + "tree_sha256": "bd9b3529c33398f26f5d2a0192556f92fea7d3b4fc559242db22e1b2e8639cbc", + "source_and_backup_unchanged": true + }, + { + "seam": "committed_metadata_partial_files", + "destination_published": false, + "table_sha256": "c1154a34b12ce88d578acd51ceef63f9e86300b83cd8956cd7988db1e9f18e2e", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "f27acd85aa73d33f5933cb919f884f976fb3de299a6460b5038f16935f53cd74", + "source_and_backup_unchanged": true + }, + { + "seam": "published_destination_lost_reply", + "destination_published": true, + "table_sha256": "c1154a34b12ce88d578acd51ceef63f9e86300b83cd8956cd7988db1e9f18e2e", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "1b2dba4f8fd97fd9a41238e04075e6b82f8d51cf9e6d1d1fbe050c98c0ae843f", + "source_and_backup_unchanged": true + } + ], + "authored_sequence_fixture": { + "table": "generation_jobs", + "before": { + "last_value": 1, + "is_called": false + }, + "retained": { + "last_value": 20, + "is_called": true + } + }, + "grant_count": 1, + "consumption_count": 1, + "run_count": 1, + "study_reservation_count": 0, + "stored_span_states": [ + "incomplete", + "measured" + ], + "compute_replay_calls": 0, + "raw_permissions_equal": true, + "cold_reads_preserve_all_tables_and_sequences": true, + "exact_cleanup_names": { + "restored_database_removed": "retention_restore_1094643898f14f1495c439ccbf8a8f08", + "source_schema_removed": "catalog_58ccf61daf6b44b581e785b21295ff79" + } + }, + "files_sha256": { + "independent_runner.py": "1f8ccc36f96c8e56857cfe1ac73891a7910f3d50c563ae93f247edc175df8618", + "reproduction-d02f034/unittest.log": "34169a1efd6154fe6f00c8ccb016b48c0e9a6097b8ecdbec54776f00a40d1cce", + "reproduction-d02f034/domain.json": "9b3f56f9fcfaee6c8934f1daa9abb7166b7b5c2c727019081a11b5037e9caf85", + "reproduction-d02f034/result.json": "50b558e1c8138b77f7f4634f3202ed01ca15cc13345149c5231554da33ecce6d" + }, + "remaining_gates": [ + "Current published-head required hosted checks and final report/source binding before authorized merge." + ], + "limits": [ + "Controlled Python exceptions/retry are qualified, not host crash, kill, power loss or storage-device durability.", + "Partial-copy proof covers a strict subset of complete verified files, not torn file payloads.", + "POSIX mode bits are compared; ownership/ACL/xattr/timestamp preservation is not newly qualified.", + "Job-order tables are empty; the high-water mark is explicit administrative fixture state, not arbitrary populated-sequence rollback proof.", + "Real neutral execution and PostgreSQL recovery/consumer integrity are covered, not scientific training acceptance or a performance comparison.", + "Full M2/M3/M5/T8 acceptance is not implied; real PUB1/SCI1 inputs remain missing." + ] +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/unittest.log b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/unittest.log new file mode 100644 index 00000000..b9a01f84 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/independent/unittest.log @@ -0,0 +1,6 @@ +test_stopped_transfer_retains_recovery_timing_and_exact_publication_consumers (test_integrated_retention_restore.IntegratedRetentionRestoreTests.test_stopped_transfer_retains_recovery_timing_and_exact_publication_consumers) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 17.013s + +OK diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/domain.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/domain.json new file mode 100644 index 00000000..a74edd5e --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/domain.json @@ -0,0 +1,654 @@ +{ + "format": "integrated_retention_restore_acceptance_v2", + "scope": "Real neutral custody, PostgreSQL stopped transfer and current HTTP/SDK/CLI; no science or performance claim", + "schema_version": 10, + "catalog_id": "f6300ed343a6493794da008ee4450194", + "run_id": "34db9d58c3be49ba8f54702534cb2bf9", + "result_ref": { + "kind": "recovered_publication_result", + "schema_version": 1, + "catalog_id": "f6300ed343a6493794da008ee4450194", + "run_id": "34db9d58c3be49ba8f54702534cb2bf9", + "material_sha256": "e9757435f5411aae4f9a886059d7687177992c6fe4f370a49bb13562b401c2da", + "inventory_sha256": "84afb1364c2ecd99dbc7b34258f2d331b576f8e49b96aebd980cd73dd1450d84", + "custody_receipt_sha256": "8230010c7923f1b8b1abce6866667419df539c7d7e1a2ab957343a1de7a52706", + "sha256": "f8269b389516c145e7c80f01d48dc56b428e5db822d8c07b5c98e4422afb09c6" + }, + "table_summaries": { + "access_actors": { + "columns": [ + "id", + "kind", + "label", + "created_at" + ], + "count": 4, + "row_sha256": [ + "f56d785807bff893692bdc2a97844d91fcefe7cf46b1287a8b44c4a12c8aa352", + "1e2b76314d3a62a5c27a43ecd50fa6dfc1e0aa2095fb33d9eedcf5f12536f3a3", + "aa433e96b013de22d0de22b71f666a7bb6a9e6817930810eb367f81bc2c9b530", + "e0ceced35f3a7c94fcfd8b3313dea575b6fecd7a892f9fb14c64becf223359fa" + ], + "sha256": "ff04178edfcd1ff1c9af1ef9e9c536df3e7253f395db2adf330a4e04cd85698e" + }, + "access_admissions": { + "columns": [ + "id", + "catalog_id", + "actor_id", + "credential_id", + "kind", + "job_id", + "request_digest", + "effects_json", + "created_at" + ], + "count": 1, + "row_sha256": [ + "d0d6ddc45bf485d34b9fb305226d286a084cd17a4a90cbe981190c79ed58f04b" + ], + "sha256": "5329d44343b03e4cdbf4ff543b428fbe31f793676bc069b16a2ea99c5edf403c" + }, + "access_audit": { + "columns": [ + "id", + "actor_id", + "credential_id", + "action", + "target_json", + "details_json", + "result", + "created_at" + ], + "count": 19, + "row_sha256": [ + "35a020f754a4c4f7577d14c096fc6af5b8b219c4d6059bc1cc2f8b30016c0126", + "093c1cbaeee413bd5290d625372c93536887c91878a1e69b02781d87b6f8302e", + "6c3349a701bf57015e27927990871cef048523578f1e9c2a984b2dc136afde32", + "41733457de328fa62edf034e16e7a33092bf4b43872e48cf080f960555d87928", + "f0dee759db0860a011deb817e36b201cd8f26eab2c8aec956bc90a4856910516", + "6822c74296525c8c67749b794137c90fe1c9d6c40fbfc7203583ebfff0b1fd4a", + "3266f62be9768bd120dccacf551da3b213307c15c7e93c4ded05356140150d83", + "c35648c9b523423c2e81607a6af06ac7f26a674525e3553071c68904db42e551", + "c9019599f2c01c752098bf4561504f906d81ab2ba4df0730cf9f8692324698ff", + "05bc969be1384afe3760840fb0502d290d8259dc5a561eaf602658dcf6291dec", + "3204b686bb7e1589d8761868c34f490d9ed44ccbd601914f61576902f958f1a4", + "19f7a9734e26b995f863c0331aa397fe9f13bde174dc730dd93758b393fb98d8", + "1ca6973ac8d256650c52fc5bd0baaf0e4f83d66213ba10fc48762805b4ce75de", + "7c8391fe23c3fdaedc517451d88b52f8bb0036664f2e7100d908727a5dfed7c0", + "ff853389f6a4fdd6d65b9c6126fc207a85535a4c47fba007e45de7d5380730a3", + "7d1ab36137ac25bd992b608c6c07f798c92d882a466c45eb8333748f795942f5", + "b565b08750cad79f450d8cc751174b56cbdc1a8ea6dc5819cc6ff18133f9c670", + "57574d4a52498ee471f7958088266ba72d529613f5df36f0fa19a616fdd1f111", + "4083ef56ca75a54b57bc07a537dd87f749f6be9f81219923779db901a66c2c02" + ], + "sha256": "2f43318668abe56597852204b21df4311b25ba8a4035e52fdebfb82cef99d383" + }, + "access_credentials": { + "columns": [ + "id", + "actor_id", + "token_sha256", + "kind", + "scopes_json", + "expires_at", + "revoked", + "consumed", + "created_at" + ], + "count": 5, + "row_sha256": [ + "6d4a8c61042a06492032aca2003fb4bb737ba80de9702075037272c2ab8024ce", + "2006e24df91284275e1f715e1e186da60779831421c5aa37037fedf53901823e", + "aa5154c2d10241fb5c815ecfd5a66befd35b19cf4a546bb1f8a4836e06f36ed9", + "9a8a8f045ab469bc9a3c7835060da1b6a15d21426ff62d96a6e2eeb98e43c983", + "85442b558d12bd829405e317b11f361eef6128292b69a93e19095b69ae079be2" + ], + "sha256": "780cf9789e7fa973f6b8b5ff298591756bb6e9e8960b44da1b5cc0950e45c122" + }, + "access_execution_consumptions": { + "columns": [ + "grant_id", + "run_id", + "admission_id", + "consumer_credential_id", + "request_sha256", + "consumed_at" + ], + "count": 1, + "row_sha256": [ + "3d602b2135f616c199eb2417b8bb5c4013f91bfce6b696b67b7c5a43c7c986f9" + ], + "sha256": "fb23166f0abb68c12524527a101ffb552a8973bb5bfa698e4369bfaf379a3caf" + }, + "access_execution_grants": { + "columns": [ + "id", + "catalog_id", + "approver_actor_id", + "approver_credential_id", + "consumer_actor_id", + "material_sha256", + "created_at", + "expires_at", + "revoked_at" + ], + "count": 1, + "row_sha256": [ + "930023ebb68ea6aca8e7ac58f8cc15e6ce7da535b5d51111de6d36e030e51a11" + ], + "sha256": "239b73a7a2da785f586fff7f0dd3685d16037b878d1a42f3ea92677be5cea21d" + }, + "caption_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "generation_entries": { + "columns": [ + "id", + "job_id", + "ordinal", + "data", + "storage_order" + ], + "count": 0, + "row_sha256": [], + "sha256": "530faef7a9e5031599c72af9332f8c2472584ffd0d4110ab97eb445c3a9680dd" + }, + "generation_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "grounded_jobs": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "model_attempt_reservations": { + "columns": [ + "id", + "study_id", + "actor_id", + "request_key", + "request_sha256", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "952235730407a326f927ed027d547c534463ac174ce800ccd0f2630e7b02fe01" + }, + "model_candidates": { + "columns": [ + "id", + "study_id", + "trial_id", + "run_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "8e150598074027645a884f8c83471a910abaabfaca5017c5154bed5b5f6ea480" + }, + "model_definition_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "model_materials": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a519823d3f1dcbcb57d09c23d8ecf7e7ceb4b534d649da6804084448530962c2" + }, + "model_operation_receipts": { + "columns": [ + "actor_id", + "operation_id", + "request_key", + "request_sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "58bf43af0b5efd8c06272e144beb72720bbdd6204e120fd4708e739555fb899c" + }, + "model_selection_policies": { + "columns": [ + "id", + "revision", + "sha256", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a519823d3f1dcbcb57d09c23d8ecf7e7ceb4b534d649da6804084448530962c2" + }, + "model_studies": { + "columns": [ + "id", + "revision", + "admission_epoch", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "a50d931d7ec23487169070a69653a1b933dcc1d1985a8f3b60ff1e4b7091f94c" + }, + "model_study_memberships": { + "columns": [ + "study_id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "4b114a32959ae2c599d50d18ebfeccf1c757d45c03518d95820aa463c6b85c98" + }, + "model_trial_lineage": { + "columns": [ + "trial_id", + "parent_trial_id", + "baseline_trial_id", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ce519910146db157ed46ee02a83bc6add785136de22e6b9cd48581c756299ec0" + }, + "model_trials": { + "columns": [ + "id", + "study_id", + "material_sha256", + "replicate", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "d790df0190b101dba64d6025d8c10072d436e400152aad4059d8de5665f0545c" + }, + "samples": { + "columns": [ + "id", + "sha256", + "filename", + "width", + "height", + "book_id", + "session_id", + "split", + "created_at", + "updated_at", + "revision", + "annotation" + ], + "count": 0, + "row_sha256": [], + "sha256": "45cba19d248a0b126119bc05246fdfe6459c95d5cce245ec3a4cd47d1b5e2995" + }, + "saved_searches": { + "columns": [ + "id", + "name", + "revision", + "criteria_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "caa62ec987589481b04e93502d8f340a1a41703997b68a0c94df642ee8e5a8a6" + }, + "saved_selections": { + "columns": [ + "id", + "name", + "revision", + "members_json", + "member_count", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "ef015a7aae83417e2ee6b9aa54c0f3f618358296ed201682901be8c0841b7e11" + }, + "text_classification_proposals": { + "columns": [ + "id", + "data", + "ordinal" + ], + "count": 0, + "row_sha256": [], + "sha256": "2239b351e3d06bc5293c9bba3811ccca010924f2050b60760c4b58ac8cb022a8" + }, + "workbench_deleted_sources": { + "columns": [ + "id", + "book_id", + "session_id", + "split" + ], + "count": 0, + "row_sha256": [], + "sha256": "c1c95154abd259591b21272e5324695db12a227e849c3fbd46d96ea2e554a7c2" + }, + "workbench_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_mesh_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_pointcloud_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_preference_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_preferences": { + "columns": [ + "id", + "prompt_id", + "revision", + "data" + ], + "count": 0, + "row_sha256": [], + "sha256": "67b197c3ca40fbcbe1baf7d841be28c4e9c98b3ef3f22178c7732560f66e16c3" + }, + "workbench_records": { + "columns": [ + "id", + "kind", + "name", + "text", + "original_text", + "content_hash", + "pixel_hash", + "groups_json", + "parents_json", + "provenance_json", + "task", + "annotation_json", + "review", + "revision", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "e2e5e5e9cef6974f035aa0b76e15e13932e11b2f3fdb93574cf2791aacb19259" + }, + "workbench_response_history": { + "columns": [ + "id", + "revision", + "snapshot" + ], + "count": 0, + "row_sha256": [], + "sha256": "8c767798a7512a72e56b40ecff0c92b8303bd7aa32fb65e1052d57238f08bfa6" + }, + "workbench_responses": { + "columns": [ + "id", + "prompt_id", + "revision", + "completion", + "review", + "provenance_json", + "created_at", + "updated_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "4eb68800cc161c30f33a81c52c233a59559a12f1a3744545088bf3cbe9930d67" + }, + "workbench_rights_notes": { + "columns": [ + "id", + "note", + "revision" + ], + "count": 0, + "row_sha256": [], + "sha256": "894e92255b0040027f355105f88c51cff087e8471008b30731a6e0e849c3cdc3" + }, + "workbench_sequence_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workbench_target_proposals": { + "columns": [ + "id", + "evidence" + ], + "count": 0, + "row_sha256": [], + "sha256": "c610988ec09c1a996fba5c2d51051fd38b37a1b51d1defc6ea9647664742838e" + }, + "workbench_volume_assets": { + "columns": [ + "id", + "bundle", + "metadata_json", + "bundle_bytes" + ], + "count": 0, + "row_sha256": [], + "sha256": "52372712cdd0cf503b97b2aa3a696f10ab52df562097e77dd96d9c0d6feb920d" + }, + "workspace_exposures": { + "columns": [ + "id", + "record_id", + "source_hash", + "reason", + "run_id", + "release_id", + "created_at" + ], + "count": 0, + "row_sha256": [], + "sha256": "5bf95a047db8503a858f55e8095cb28322d6329af09b545dc4f67a3cb83e8599" + }, + "workspace_graphs": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 1, + "row_sha256": [ + "5764095a62d18e14a05936b85f82cd6a000a4721edab635ddf911d4d6309299d" + ], + "sha256": "a5bae2dfe051d1e845dc697b230497a97d664c9a1b907e1ee335b22ee6c15844" + }, + "workspace_projects": { + "columns": [ + "id", + "document" + ], + "count": 1, + "row_sha256": [ + "bccb71f7199a9f5e9ae8b9b9e6d2db944dfd9cfe36df2263d2b87c5593523a02" + ], + "sha256": "699fcc377496aa57ca88102124037bd3aea9a5215fd0cb80ddba6e697ba0f058" + }, + "workspace_recipe_drafts": { + "columns": [ + "id", + "revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "ffb3a23ecc7b51f988108891d577a0c846811cfcfddbb1427421287a201f347a" + }, + "workspace_recipe_materials": { + "columns": [ + "sha256", + "recipe_id", + "recipe_revision", + "document" + ], + "count": 0, + "row_sha256": [], + "sha256": "7b2b46c5091c5e07e25b8648e2161ed0253aae41384b82f7923aba99bb5f04d0" + }, + "workspace_runs": { + "columns": [ + "id", + "request_key", + "request_sha256", + "document" + ], + "count": 1, + "row_sha256": [ + "c01a081e1ba56faf18cf37820cd735eb0ab55e6d65758c66cce089184217c50e" + ], + "sha256": "f992c675510126f43758c62a468ac244f63a2499421b5e03e930693329eac2f1" + } + }, + "sequences": { + "generation_jobs": { + "last_value": 20, + "is_called": true + }, + "grounded_jobs": { + "last_value": 1, + "is_called": false + }, + "caption_proposals": { + "last_value": 1, + "is_called": false + }, + "text_classification_proposals": { + "last_value": 1, + "is_called": false + }, + "generation_entries": { + "last_value": 1, + "is_called": false + } + }, + "catalog_tree_sha256": "80f243dabfc4246a87275d4b7be9f085edfd884f3eaea7a44382399141a2ac93", + "backup_tree_sha256": "e2f725e39128d2beb31dedbe373ddfbecc82d2c48e4470fc6511580fdd88bcd7", + "journal_sha256": "310a69918cc7094363ee56bc7ac8e21c129cc71652dfb1266256ebf9b2fec99c", + "retained_timing_sha256": "a5275164bd97caa1a766b826c1aa9204aadbc37917ae542014c55deee3f58431", + "recovery_request": { + "actor_id": "ac6b2d631890457c9ba8d6aa312d4613", + "completed_generation": 2, + "expected_generation": 1, + "request_key": "transfer-coordinator-recovery-0001", + "request_sha256": "8c906927e5d1b31a4248650f9f43f05dca2422fd213e669ee4cec8fef8096cc1" + }, + "stored_span_states": [ + "incomplete", + "measured" + ], + "artifact_sha256": "54ae9355f214d827afd4ec287ff813d3af22683a52c10e9ed1ea62507cfa4d10", + "artifact_bytes": 55, + "raw_transfer_equal": true, + "source_and_backup_unchanged": true, + "restore_interruptions": [ + { + "seam": "metadata_transaction_rollback", + "destination_published": false, + "table_sha256": "20ce31ed06a9ac732d86432973dfd4427d55a939c72b0ccdf1797979ae7322c5", + "sequence_sha256": "915979a8c675d1c5bb78aef6cc05730ce55215dc052fcc0a43c88459b5cbda19", + "tree_sha256": "85fc6728d0f6dfb9eb85359f6a9be63dce68eb8c230015bf75c98d696951bf6c", + "source_and_backup_unchanged": true + }, + { + "seam": "committed_metadata_partial_files", + "destination_published": false, + "table_sha256": "5901912a5fe0c2c5a4f9d7b62cb05cf04c52d450498d271dfc6ac3d281c218a2", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "c6f2a47dc730a4e669ec43a1326ab804977b8c024f120d7f7393546daf3467bc", + "source_and_backup_unchanged": true + }, + { + "seam": "published_destination_lost_reply", + "destination_published": true, + "table_sha256": "5901912a5fe0c2c5a4f9d7b62cb05cf04c52d450498d271dfc6ac3d281c218a2", + "sequence_sha256": "d1237a8b0395f7df3baca42edfabc8f33859625f77f1e4747a8f540884c25f78", + "tree_sha256": "80f243dabfc4246a87275d4b7be9f085edfd884f3eaea7a44382399141a2ac93", + "source_and_backup_unchanged": true + } + ], + "raw_permissions_equal": true, + "cold_reads_and_collection_preserve_all_tables_and_sequences": true, + "compute_replay_calls": 0, + "cleanup": { + "restored_database_removed": "retention_restore_c5e58ff142164abd9c0ccbfdb360109e", + "source_schema_removed": "catalog_f6300ed343a6493794da008ee4450194" + } +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/result.json b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/result.json new file mode 100644 index 00000000..81425c4f --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/result.json @@ -0,0 +1,35 @@ +{ + "format": "interrupted_restore_focused_qualification_v1", + "source": { + "head": "0bbc6a7c4cd58dc0451d6ba6c0983193763e1f91", + "tree": "b2e7b8196a52b283fc79c93ff7fcdfdf00201f5a", + "status": "", + "test_sha256": "acd006da64a5eb36db7196a2d03ca14c687bb863724dcff602eb57d0720d657e" + }, + "seconds": 17.614621812999758, + "tests_run": 1, + "failures": 0, + "errors": 0, + "skips": 0, + "successful": true, + "resources_before": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ] + }, + "resources_after": { + "schemas": [], + "databases": [ + "postgres", + "template0", + "template1", + "tuldok_qa" + ] + }, + "final_status": "", + "log_sha256": "51f85c0c2f20c80697cf1843ba84c3fe9aebd8d78942440ef7f6c79540212955" +} diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/runner.py b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/runner.py new file mode 100644 index 00000000..75d6eda9 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/runner.py @@ -0,0 +1,39 @@ +import contextlib, hashlib, io, json, os, pathlib, subprocess, sys, time, unittest +import psycopg +from psycopg.conninfo import make_conninfo + +repo = pathlib.Path('/workspace/Tuldok-interrupted-restore-20261021') +out = pathlib.Path(sys.argv[1]); out.mkdir(parents=True, exist_ok=False, mode=0o700) +container = json.loads(subprocess.check_output(['docker', 'inspect', 'tuldok-qa-20261020']))[0] +env = dict(item.split('=', 1) for item in container['Config']['Env'] if '=' in item) +dsn = make_conninfo(host='127.0.0.1', port=55432, user=env.get('POSTGRES_USER', 'postgres'), + dbname=env.get('POSTGRES_DB', 'postgres'), password=env['POSTGRES_PASSWORD']) +os.environ['TULDOK_DATABASE_URL'] = dsn +os.environ['TULDOK_RETENTION_ACCEPTANCE_REPORT'] = str(out/'domain.json') +os.chdir(repo); sys.path[:0] = [str(repo/'tests'), str(repo)] +def git(*args): return subprocess.check_output(['git', *args]).decode().strip() +def resources(): + with psycopg.connect(dsn, autocommit=True) as db: + return {'schemas': [r[0] for r in db.execute("SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname NOT IN ('public','information_schema') ORDER BY nspname")], + 'databases': [r[0] for r in db.execute('SELECT datname FROM pg_database ORDER BY datname')]} +before = resources(); assert not before['schemas'], before +source = {'head': git('rev-parse','HEAD'), 'tree': git('rev-parse','HEAD^{tree}'), + 'status': git('status','--porcelain=v1'), 'test_sha256': hashlib.sha256((repo/'tests/test_integrated_retention_restore.py').read_bytes()).hexdigest()} +assert not source['status'], source +with (out/'unittest.log').open('x') as log: + start = time.perf_counter() + with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log): + suite = unittest.defaultTestLoader.loadTestsFromName('test_integrated_retention_restore.IntegratedRetentionRestoreTests') + result = unittest.TextTestRunner(stream=log, verbosity=2).run(suite) + elapsed = time.perf_counter()-start +after = resources() +receipt = {'format':'interrupted_restore_focused_qualification_v1', 'source':source, + 'seconds':elapsed, 'tests_run':result.testsRun, 'failures':len(result.failures), + 'errors':len(result.errors), 'skips':len(result.skipped), 'successful':result.wasSuccessful(), + 'resources_before':before, 'resources_after':after, 'final_status':git('status','--porcelain=v1'), + 'log_sha256':hashlib.sha256((out/'unittest.log').read_bytes()).hexdigest()} +(out/'result.json').write_text(json.dumps(receipt,indent=2)+'\n') +print(json.dumps(receipt)) +assert before==after, receipt +assert not receipt['final_status'], receipt +raise SystemExit(0 if result.wasSuccessful() and result.testsRun==1 and not result.skipped else 1) diff --git a/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/unittest.log b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/unittest.log new file mode 100644 index 00000000..5f598627 --- /dev/null +++ b/docs/plans/lab-workflow-reliability/reports/integrated-interrupted-restore-20261021/root/unittest.log @@ -0,0 +1,6 @@ +test_stopped_transfer_retains_recovery_timing_and_exact_publication_consumers (test_integrated_retention_restore.IntegratedRetentionRestoreTests.test_stopped_transfer_retains_recovery_timing_and_exact_publication_consumers) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 17.252s + +OK diff --git a/tests/test_integrated_retention_restore.py b/tests/test_integrated_retention_restore.py index 07dd88ea..d0023ddb 100644 --- a/tests/test_integrated_retention_restore.py +++ b/tests/test_integrated_retention_restore.py @@ -17,9 +17,10 @@ from psycopg.conninfo import make_conninfo import agent_api +import backup_metadata as transfer from app import Dataset, make_handler from backup_metadata import (backup_catalog, read_catalog, read_sequences, - restore_catalog, summaries, tree_manifest) + restore_catalog, summaries, tree_manifest, tree_modes) from metadata import Metadata, connection_info from tuldok_client import WorkflowClient from workbench import WorkbenchError @@ -95,6 +96,7 @@ def drop_source(): source.close() original_tree = tree_manifest(source.path) + original_modes = tree_modes(source.path, original_tree) journal_path = source.path/'workspace-runs'/'.coordinator-v1.json' journal_bytes = journal_path.read_bytes() journal = json.loads(journal_bytes) @@ -102,6 +104,11 @@ def drop_source(): self.assertEqual(journal['request']['completed_generation'], 2) metadata = Metadata(source.path, dsn=original_dsn, create=False) try: + # Retain a nontrivial sequence high-water mark even with no live job + # row. This authored fixture state must survive every restore retry. + sequence = read_sequences(metadata)['generation_jobs'] + metadata.connection.execute('SELECT setval(pg_get_serial_sequence(%s,%s),%s,true)', + ('generation_jobs', 'ordinal', sequence['last_value'] + 19)) original_tables, original_sequences = read_catalog(metadata), read_sequences(metadata) original_version = metadata.schema_version finally: @@ -109,6 +116,7 @@ def drop_source(): backup = fixture.path/'stopped-integrated-backup' receipt = backup_catalog(source.path, backup, dsn=original_dsn, quiesced=True) backup_tree = tree_manifest(backup) + self.assertEqual(tree_modes(backup/'catalog', original_tree), original_modes) self.assertEqual(receipt['files'], original_tree) self.assertEqual(receipt['tables'], summaries(original_tables)) @@ -128,9 +136,98 @@ def drop_restored_database(): self.addCleanup(drop_restored_database) restored_path = fixture.path/'restored-integrated-catalog' + staging = restored_path.with_name(restored_path.name+'.restore-'+receipt['dataset_id']) + marker_name = '.restore-receipt-'+digest(receipt)+'.json' + interruptions = [] + + class AuthoredRestoreInterruption(OSError): + pass + + def unchanged_originals(): + self.assertEqual(tree_manifest(source.path), original_tree) + self.assertEqual(tree_modes(source.path, original_tree), original_modes) + self.assertEqual(tree_manifest(backup), backup_tree) + self.assertEqual(tree_modes(backup/'catalog', original_tree), original_modes) + original = Metadata(source.path, dsn=original_dsn, create=False) + try: + self.assertEqual(read_catalog(original), original_tables) + self.assertEqual(read_sequences(original), original_sequences) + finally: + original.close() + + def checkpoint(seam, expected_tables, expected_sequences, *, published=False): + self.assertEqual(restored_path.exists(), published) + self.assertEqual(staging.exists(), not published) + path = restored_path if published else staging + self.assertEqual(json.loads((path/marker_name).read_bytes()), { + 'kind': 'postgresql-restore', 'backup_sha256': digest(receipt), + 'dataset_id': receipt['dataset_id']}) + held = Metadata(path, dsn=restored_dsn, create=False) + try: + tables, sequences = read_catalog(held), read_sequences(held) + self.assertEqual(tables, expected_tables) + self.assertEqual(sequences, expected_sequences) + finally: + held.close() + unchanged_originals() + tree = tree_manifest(path, (marker_name,)) + interruptions.append({'seam': seam, 'destination_published': published, + 'table_sha256': digest(tables), 'sequence_sha256': digest(sequences), + 'tree_sha256': digest(tree), 'source_and_backup_unchanged': True}) + return tree + with patch('recipe_process.launch', side_effect=AssertionError('Transfer/read replayed compute')) as launch: + # Real PostgreSQL inserts occur before the authored exception. The + # catalog transaction rolls back; its private staging identity stays + # resumable and no final destination has been published. + empty = {} + populate = transfer.populate_catalog + + def interrupt_population(metadata, tables): + empty['tables'], empty['sequences'] = read_catalog(metadata), read_sequences(metadata) + populate(metadata, tables) + self.assertEqual(read_catalog(metadata), original_tables) + raise AuthoredRestoreInterruption('after real population, before commit') + + with patch.object(transfer, 'populate_catalog', side_effect=interrupt_population): + with self.assertRaisesRegex(AuthoredRestoreInterruption, 'before commit'): + restore_catalog(backup, restored_path, dsn=restored_dsn) + self.assertFalse(any(table['count'] for table in empty['tables'].values())) + checkpoint('metadata_transaction_rollback', empty['tables'], empty['sequences']) + + # Interrupt after one exact journal file has actually been copied + # and renamed, while later Run artifacts are still absent. Metadata + # has committed; the public destination must still be absent. + rename = transfer.os.rename + journal_target = staging/'workspace-runs'/'.coordinator-v1.json' + + def interrupt_copy(src, dst): + rename(src, dst) + if Path(dst) == journal_target: + raise AuthoredRestoreInterruption('after journal file copy') + + with patch.object(transfer.os, 'rename', side_effect=interrupt_copy): + with self.assertRaisesRegex(AuthoredRestoreInterruption, 'journal file copy'): + restore_catalog(backup, restored_path, dsn=restored_dsn) + partial = checkpoint('committed_metadata_partial_files', original_tables, original_sequences) + self.assertEqual(journal_target.read_bytes(), journal_bytes) + self.assertLess(set(partial), set(original_tree)) + self.assertTrue(all(original_tree[name] == entry for name, entry in partial.items())) + + # A lost reply after the real final rename leaves a complete result. + # Retrying must reconcile that result instead of overwriting it. + def lose_rename_reply(src, dst): + rename(src, dst) + if Path(dst) == restored_path: + raise AuthoredRestoreInterruption('after destination rename') + + with patch.object(transfer.os, 'rename', side_effect=lose_rename_reply): + with self.assertRaisesRegex(AuthoredRestoreInterruption, 'destination rename'): + restore_catalog(backup, restored_path, dsn=restored_dsn) + self.assertEqual(checkpoint('published_destination_lost_reply', original_tables, + original_sequences, published=True), original_tree) + self.assertEqual(tree_modes(restored_path, original_tree), original_modes) marker = restore_catalog(backup, restored_path, dsn=restored_dsn) - marker_name = '.restore-receipt-'+digest(receipt)+'.json' self.assertEqual(marker['dataset_id'], source.db.dataset_id) self.assertEqual(tree_manifest(restored_path, (marker_name,)), original_tree) self.assertEqual((restored_path/'workspace-runs'/'.coordinator-v1.json').read_bytes(), journal_bytes) @@ -224,18 +321,11 @@ def cli(*arguments): launch.assert_not_called() restored.close() - self.assertEqual(tree_manifest(source.path), original_tree) - self.assertEqual(tree_manifest(backup), backup_tree) - original = Metadata(source.path, dsn=original_dsn, create=False) - try: - self.assertEqual(read_catalog(original), original_tables) - self.assertEqual(read_sequences(original), original_sequences) - finally: - original.close() + unchanged_originals() drop_restored_database() drop_source() if report_path := os.environ.get('TULDOK_RETENTION_ACCEPTANCE_REPORT'): - report = {'format': 'integrated_retention_restore_acceptance_v1', + report = {'format': 'integrated_retention_restore_acceptance_v2', 'scope': 'Real neutral custody, PostgreSQL stopped transfer and current HTTP/SDK/CLI; no science or performance claim', 'schema_version': original_version, 'catalog_id': ref['catalog_id'], 'run_id': run['id'], 'result_ref': ref, 'table_summaries': summaries(original_tables), @@ -245,6 +335,9 @@ def cli(*arguments): 'stored_span_states': sorted({span['state'] for span in stored_spans}), 'artifact_sha256': hashlib.sha256(fixture.input).hexdigest(), 'artifact_bytes': len(fixture.input), 'raw_transfer_equal': True, 'source_and_backup_unchanged': True, + 'restore_interruptions': interruptions, 'raw_permissions_equal': True, + 'authored_sequence_fixture': {'table': 'generation_jobs', 'before': sequence, + 'retained': original_sequences['generation_jobs']}, 'cold_reads_and_collection_preserve_all_tables_and_sequences': True, 'compute_replay_calls': launch.call_count, 'cleanup': cleanup} with Path(report_path).open('x') as stream: