From 5e18837e1b580b5b831118adb0b0dc8b1dbba4e7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 16 Oct 2026 21:41:27 -0700 Subject: [PATCH 1/9] feat: schedule independent CPU and CUDA runs concurrently --- docs/contracts/external-recipes.md | 5 +- docs/contracts/workspace-runs.md | 33 ++++- docs/plans/concurrent-cpu-cuda/plan.md | 48 +++++++ .../reports/scheduler-tests-06.json | 117 ++++++++++++++++ .../reports/scheduler-tests-07.json | 49 +++++++ recipe_contract.py | 6 +- run_capacity.py | 78 +++++++++++ runs.py | 126 ++++++++++++------ tests/test_recipe_contract.py | 11 ++ tests/test_run_capacity.py | 44 ++++++ tests/test_run_concurrency.py | 87 ++++++++++++ 11 files changed, 560 insertions(+), 44 deletions(-) create mode 100644 docs/plans/concurrent-cpu-cuda/plan.md create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-06.json create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-07.json create mode 100644 run_capacity.py create mode 100644 tests/test_run_capacity.py create mode 100644 tests/test_run_concurrency.py diff --git a/docs/contracts/external-recipes.md b/docs/contracts/external-recipes.md index 0b96f97a..98359090 100644 --- a/docs/contracts/external-recipes.md +++ b/docs/contracts/external-recipes.md @@ -98,7 +98,10 @@ retry means a fresh preflight, key and approval, not reuse of the old grant. ## Process custody and recovery -The sole Runs queue starts one pinned single-threaded Linux subreaper helper. +The sole Runs supervisor starts one pinned single-threaded Linux subreaper helper +per dispatched attempt. Resource-aware dispatch can overlap independent helpers; +each has its own control channel, process identity, custody directory and receipt. +See [Run resource scheduling](workspace-runs.md) for CPU, CUDA and memory budgets. A private bounded socket protocol carries READY, GO, cancellation and custody receipts; recipe stdout cannot declare completion. The helper and executable bytes use sealed descriptors. Recipe code receives its own stdin/stdout/stderr, diff --git a/docs/contracts/workspace-runs.md b/docs/contracts/workspace-runs.md index 346939ed..a03e8d12 100644 --- a/docs/contracts/workspace-runs.md +++ b/docs/contracts/workspace-runs.md @@ -19,7 +19,36 @@ catalog transaction serialize competing admissions. A committed queued receipt whose acknowledgment was lost wakes the existing owned, unblocked queue when the caller explicitly reconciles the same request. It never creates a second run. Restarted catalogs do not replay old queue entries. -There are at most eight active admitted attempts and one active training child. +There are at most eight active admitted attempts. The single catalog supervisor +dispatches independent attempts concurrently through an owned executor: at most +one CUDA attempt and a configurable number of CPU attempts. `run_capacity.py` +owns resource selection; `Runs` retains admission, job authorization, execution, +publication and recovery. A queued CUDA attempt cannot block an eligible CPU +attempt behind it. Unknown custody fences all further dispatch until recovery. + +`TULDOK_CPU_RUN_SLOTS` selects 1–7 CPU attempts (default up to seven, derived from +available CPUs with two reserved for interactive work). CPU placements are disjoint +within this supervisor and inherited by child processes. Recipes can reserve +`resources.cpu_cores` (1–256); otherwise CPU attempts receive one core and CUDA +attempts up to two. Numerical thread settings remain with their adapter. Affinity +is cooperative trusted-local placement, not a security sandbox. Placement and +admission budgets are retained as `execution_resources` on each dispatched Run. + +`TULDOK_CPU_RUN_MEMORY_MIB`, `TULDOK_CUDA_RUN_MEMORY_MIB` and +`TULDOK_RUN_MEMORY_RESERVE_MIB` select positive budgets of at least64 MiB; defaults +are1024/4096/1024 MiB. A recipe's optional `resources.memory_bytes` can increase +its reservation but cannot reduce the operator floor. Dispatch requires current +host/cgroup available memory to cover all active reservations, the new attempt +and the host reserve. Missing memory observation leaves work queued. Reservations +are conservative admission accounting, not per-process OS memory limits or an +OOM guarantee; trusted recipes must separately bound their allocations. The +64 GiB lab profile selects4096/8192/8192 MiB and explicit per-recipe budgets +for larger CPU batches. Other applications and catalogs do not share these leases. + +The coordinator holds every executor future until the attempt finishes its +publication/cessation path. Shutdown requests cancellation of every active +attempt and joins the entire executor before releasing catalog metadata. Restart +reconciles each durable attempt independently and never replays old queue entries. `list()` and `get(id)` return real mode, dataset/run IDs, saved graph snapshot, graph and release SHA, graph revision, adapter/runtime/parameters, attempt and @@ -175,7 +204,7 @@ and produce reasons without comparative metric values. Current record editing continues through Workbench; retained inputs and prior evidence stay immutable. Built-in model-aware trials use the same owner-only Runs admission, finite Study -reservation transaction, eight-attempt queue and one worker. A membership/budget +reservation transaction and eight-attempt queue with resource-aware dispatch. A membership/budget or external execution grant cannot launch a built-in trainer as an agent. The supervisor captures and checks the new evidence automatically; users configure neither logging nor manual artifact imports. Runtime installation is separate diff --git a/docs/plans/concurrent-cpu-cuda/plan.md b/docs/plans/concurrent-cpu-cuda/plan.md new file mode 100644 index 00000000..c025959f --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/plan.md @@ -0,0 +1,48 @@ +# Concurrent local CPU and CUDA Runs + +Status: Active; operation start. User authorized simultaneous CPU Runs alongside +one CUDA fit and continuing the pressure-model sequence. Owner: root; branch +performance/concurrent-cpu-cuda-20261017, base8af5a6a1e288e5b2f86a75b53cab93ca71222a04. +Retain this branch for review/integration; existing PR89 and its source branch +remain unchanged. No rewrite, merge or cleanup. + +Objective: remove the serial queue bottleneck while preserving exact admissions, +per-attempt custody, cancellation, drainage, restart and immutable publication. +Canonical consumer contract: docs/contracts/workspace-runs.md. + +Composition: Runs remains the sole catalog supervisor and owner of durable Run +state, execution futures and shutdown. The existing standard-library executor +provides bounded threading; run_capacity owns only admission budgets, CPU placement +and device classification. Custodians remain one per attempt. Authoritative +recipe resource requests are captured with material; observed placement is Run +execution metadata. No second queue, process supervisor or authority registry. +Changing a capacity budget affects run_capacity and its configuration/tests; +changing custody affects recipe_process independently. Removing capacity would +move resource accounting/placement back into Runs; removing the executor would +restore serial execution. Necessary interleaving is admission before dispatch, +and confirmed cessation/publication before lease release. All copied numerical +sources, runtime assets, model weights and scientific thresholds are unchanged. + +Write set: runs.py, run_capacity.py, recipe_contract.py, affected tests, +docs/contracts/{workspace-runs,external-recipes}.md and this plan directory. +No storage schema/version change; prior rows remain retained. New resource +requests use the current recipe decoder and material hash; stale deployments +reject those fields rather than guessing them. No runtime compatibility shim. + +Acceptance: focused resource/accounting tests; actual independent overlapping CPU +custodians; disjoint inherited CPU affinity; independent and queued cancellation; +CPU bypass of a blocked CUDA slot; exclusive CUDA dispatch; shutdown joins all +attempts; existing storage-loss/restart/unknown-state lifecycle tests. Actual CUDA +computation and CPU analysis overlap must be observed in the lab before deployment +is qualified for that workload. Source/Spec review and exact staged diff required. + +Current phase: Verifying. Next slice: final source-frozen lifecycle suite and +independent review. Existing actual process tests passed; GPU-slot dispatch was +simulated in that focused test and does not prove actual CUDA execution. The old +TinyCNN runtime is unavailable locally; its test fails at runtime preflight before +the scheduler. No package install or fallback to the lab's CUDA runtime is used. + +Related scientific plan: rheon-surrogate-lab/docs/plans/pressure-throughput-feedback-v1/plan.md. +Standards route snapshot:v1:bc3877fc-df9b-41cc-9a60-ec680a5ccf55, +21 selected policies, zero unresolved routing facts. Reservations are admission +budgets, not an OS memory sandbox; other applications/catalogs do not share leases. diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-06.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-06.json new file mode 100644 index 00000000..898fb632 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-06.json @@ -0,0 +1,117 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T04:37:53.955162+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "source_sha256": { + "runs.py": "c5b186929be4544dd64c0e098b670eb21b51277f185fc7ea8421f6e1f9eb5324", + "run_capacity.py": "e9f76e12fff9655e15da7c557a33b08d9dadfd57a87aeea2b4ff92747b7d77c9" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_run_capacity.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 0.03793659400253091, + "output": "test_classification_uses_captured_recipe_device_not_job_name (test_run_capacity.CapacityTests.test_classification_uses_captured_recipe_device_not_job_name) ... ok\ntest_configuration_is_explicit_and_invalid_settings_fail (test_run_capacity.CapacityTests.test_configuration_is_explicit_and_invalid_settings_fail) ... ok\ntest_cpu_and_cuda_are_independent_and_cuda_is_exclusive (test_run_capacity.CapacityTests.test_cpu_and_cuda_are_independent_and_cuda_is_exclusive) ... ok\ntest_explicit_recipe_resources_cannot_lower_operator_memory_floor (test_run_capacity.CapacityTests.test_explicit_recipe_resources_cannot_lower_operator_memory_floor) ... ok\ntest_host_memory_observation_is_positive (test_run_capacity.CapacityTests.test_host_memory_observation_is_positive) ... ok\ntest_reserves_memory_for_all_active_attempts_and_host (test_run_capacity.CapacityTests.test_reserves_memory_for_all_active_attempts_and_host) ... ok\n\n----------------------------------------------------------------------\nRan 6 tests in 0.001s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_run_concurrency.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 22.252693189002457, + "output": "test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits (test_run_concurrency.ConcurrentRuns.test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits) ... ok\ntest_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child (test_run_concurrency.ConcurrentRuns.test_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child) ... ok\ntest_two_cpu_custodians_overlap_and_cancel_independently (test_run_concurrency.ConcurrentRuns.test_two_cpu_custodians_overlap_and_cancel_independently) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 21.950s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_external_runs.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 72.3118937399995, + "output": "test_consumption_and_run_rollback_together (test_external_runs.ExternalAdmissionTests.test_consumption_and_run_rollback_together) ... ok\ntest_lost_ack_is_not_dispatched_by_an_already_running_queue (test_external_runs.ExternalAdmissionTests.test_lost_ack_is_not_dispatched_by_an_already_running_queue) ... ok\ntest_lost_commit_ack_reconciles_without_new_approval (test_external_runs.ExternalAdmissionTests.test_lost_commit_ack_reconciles_without_new_approval) ... ok\ntest_revocation_and_source_drift_leave_no_run_or_consumption (test_external_runs.ExternalAdmissionTests.test_revocation_and_source_drift_leave_no_run_or_consumption) ... ok\ntest_same_boot_missing_drain_receipt_fences_after_go_intent (test_external_runs.ExternalAdmissionTests.test_same_boot_missing_drain_receipt_fences_after_go_intent) ... ok\ntest_same_key_actor_and_digest_reconcile_exact_consumption (test_external_runs.ExternalAdmissionTests.test_same_key_actor_and_digest_reconcile_exact_consumption) ... ok\ntest_same_key_reconciles_after_approval_expiry_and_graph_drift (test_external_runs.ExternalAdmissionTests.test_same_key_reconciles_after_approval_expiry_and_graph_drift) ... ok\ntest_real_running_cancel_retains_consumed_approval (test_external_runs.ExternalExecutionTests.test_real_running_cancel_retains_consumed_approval) ... ok\ntest_real_unicode_parameter_descendant_output_cold_download_and_restart (test_external_runs.ExternalExecutionTests.test_real_unicode_parameter_descendant_output_cold_download_and_restart) ... ok\ntest_unknown_go_commit_ack_never_sends_go_to_real_helper (test_external_runs.ExternalExecutionTests.test_unknown_go_commit_ack_never_sends_go_to_real_helper) ... ok\ntest_stopped_real_helper_deadline_is_unknown_until_actual_drain (test_external_runs.ExternalStoppedHelperTests.test_stopped_real_helper_deadline_is_unknown_until_actual_drain) ... ok\n\n----------------------------------------------------------------------\nRan 11 tests in 71.991s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_recipe_contract.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 8.316273356002057, + "output": "test_cold_runtime_sitecustomize_matches_verified_python_version (test_recipe_contract.ContractTests.test_cold_runtime_sitecustomize_matches_verified_python_version) ... ok\ntest_execution_reservations_are_bounded_and_captured (test_recipe_contract.ContractTests.test_execution_reservations_are_bounded_and_captured) ... ok\ntest_output_inventory_refuses_extra_links_and_missing (test_recipe_contract.ContractTests.test_output_inventory_refuses_extra_links_and_missing) ... ok\ntest_parameters_and_closed_tokens (test_recipe_contract.ContractTests.test_parameters_and_closed_tokens) ... ok\ntest_real_runtime_closure_without_launch (test_recipe_contract.ContractTests.test_real_runtime_closure_without_launch) ... ok\ntest_strict_native_elf_rejects_headers_paths_and_loader_extensions (test_recipe_contract.ContractTests.test_strict_native_elf_rejects_headers_paths_and_loader_extensions) ... ok\ntest_capture_and_clone_preserve_exact_material_without_execution (test_recipe_contract.RecipePipelineTests.test_capture_and_clone_preserve_exact_material_without_execution) ... ok\ntest_native_policy_is_captured_and_cold_bound_without_execution (test_recipe_contract.RecipePipelineTests.test_native_policy_is_captured_and_cold_bound_without_execution) ... ok\ntest_source_and_input_drift_refuse_capture (test_recipe_contract.RecipePipelineTests.test_source_and_input_drift_refuse_capture) ... ok\n\n----------------------------------------------------------------------\nRan 9 tests in 7.964s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_external_progress.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 22.00872263799829, + "output": "test_actual_completed_stream_receipt_and_tamper (test_external_progress.ExternalProgressTests.test_actual_completed_stream_receipt_and_tamper) ... ok\ntest_actual_completed_summary_and_checkpoint_are_hash_verified (test_external_progress.ExternalProgressTests.test_actual_completed_summary_and_checkpoint_are_hash_verified) ... ok\ntest_event_contract_refuses_partial_duplicate_nonfinite_and_bad_counts (test_external_progress.ExternalProgressTests.test_event_contract_refuses_partial_duplicate_nonfinite_and_bad_counts) ... ok\ntest_http_progress_and_input_candidates_use_real_inspect_authority (test_external_progress.ExternalProgressTests.test_http_progress_and_input_candidates_use_real_inspect_authority) ... ok\ntest_pending_real_authority_and_live_bounded_projection (test_external_progress.ExternalProgressTests.test_pending_real_authority_and_live_bounded_projection) ... ok\ntest_summary_closed_shape_and_declared_checkpoint_role (test_external_progress.ExternalProgressTests.test_summary_closed_shape_and_declared_checkpoint_role) ... ok\ntest_unsafe_files_and_missing_terminal_diagnostics_are_unavailable (test_external_progress.ExternalProgressTests.test_unsafe_files_and_missing_terminal_diagnostics_are_unavailable) ... ok\n\n----------------------------------------------------------------------\nRan 7 tests in 21.667s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_bound_torch_recipe_contract.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 4.734676849999232, + "output": "test_bound_managed_cpu_keeps_current_profile_without_snapshot_relabel (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_bound_managed_cpu_keeps_current_profile_without_snapshot_relabel) ... ok\ntest_bound_managed_device_capture_and_exact_pre_admission_revalidation (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_bound_managed_device_capture_and_exact_pre_admission_revalidation) ... ok\ntest_managed_evidence_requires_current_bound_variant_and_no_native_sources (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_managed_evidence_requires_current_bound_variant_and_no_native_sources) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 4.421s\n\nOK\n" + } + ], + "elapsed_seconds": 129.66251550499874, + "passed": true +} diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-07.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-07.json new file mode 100644 index 00000000..fc4d1583 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-07.json @@ -0,0 +1,49 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T04:39:07.703427+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "source_sha256": { + "runs.py": "c5b186929be4544dd64c0e098b670eb21b51277f185fc7ea8421f6e1f9eb5324", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_run_capacity.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 0.03962383900943678, + "output": "test_classification_uses_captured_recipe_device_not_job_name (test_run_capacity.CapacityTests.test_classification_uses_captured_recipe_device_not_job_name) ... ok\ntest_configuration_is_explicit_and_invalid_settings_fail (test_run_capacity.CapacityTests.test_configuration_is_explicit_and_invalid_settings_fail) ... ok\ntest_cpu_and_cuda_are_independent_and_cuda_is_exclusive (test_run_capacity.CapacityTests.test_cpu_and_cuda_are_independent_and_cuda_is_exclusive) ... ok\ntest_explicit_recipe_resources_cannot_lower_operator_memory_floor (test_run_capacity.CapacityTests.test_explicit_recipe_resources_cannot_lower_operator_memory_floor) ... ok\ntest_host_memory_observation_is_positive (test_run_capacity.CapacityTests.test_host_memory_observation_is_positive) ... ok\ntest_reserves_memory_for_all_active_attempts_and_host (test_run_capacity.CapacityTests.test_reserves_memory_for_all_active_attempts_and_host) ... ok\n\n----------------------------------------------------------------------\nRan 6 tests in 0.001s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_run_concurrency.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 22.845117789009237, + "output": "test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits (test_run_concurrency.ConcurrentRuns.test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits) ... ok\ntest_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child (test_run_concurrency.ConcurrentRuns.test_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child) ... ok\ntest_two_cpu_custodians_overlap_and_cancel_independently (test_run_concurrency.ConcurrentRuns.test_two_cpu_custodians_overlap_and_cancel_independently) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 22.520s\n\nOK\n" + } + ], + "elapsed_seconds": 22.884824883993133, + "passed": true +} diff --git a/recipe_contract.py b/recipe_contract.py index 251d2df4..c0802c10 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -188,7 +188,11 @@ def decode_definition(raw): if not evidence or arg is not True:raise WorkbenchError('Model context requires its explicit recipe version.') elif kind not in ('output_dir', 'input_evidence') or arg is not True: raise WorkbenchError('Unsupported argument token.') - closed(value['resources'], ('wall_seconds', 'max_output_bytes'), ('profile',)) + closed(value['resources'], ('wall_seconds', 'max_output_bytes'), ('profile', 'cpu_cores', 'memory_bytes')) + if 'cpu_cores' in value['resources']: + bounded_int(value['resources']['cpu_cores'], 1, 256, 'CPU core reservation') + if 'memory_bytes' in value['resources']: + bounded_int(value['resources']['memory_bytes'], 64 * 1024 * 1024, 1024 ** 4, 'Memory reservation bytes') large = 'profile' in value['resources'] if large and (value['resources']['profile'] != LARGE_OUTPUT_PROFILE or value['runtime'] != CPU_PROFILE): raise WorkbenchError('Large outputs require the declared captured CPU resource profile.') diff --git a/run_capacity.py b/run_capacity.py new file mode 100644 index 00000000..9436cf49 --- /dev/null +++ b/run_capacity.py @@ -0,0 +1,78 @@ +"""Dispatch admission only; Runs retains authorization, execution and drainage. + +Memory reservations are conservative operator budgets, not OS memory limits. +Queued jobs never consume a reservation; unknown custody fences further dispatch. +""" +from dataclasses import dataclass, field +import os +from pathlib import Path + +MIB = 1024 * 1024 +MAX_ATTEMPTS = 8 + + +def usable_cores(): + available = tuple(sorted(os.sched_getaffinity(0))) + return available[2:] if len(available) > 3 else available + + +def available_memory(): + """Respect host and unified cgroup headroom; missing host data fails closed.""" + values = dict(line.split(':', 1) for line in Path('/proc/meminfo').read_text().splitlines()) + available = int(values['MemAvailable'].split()[0]) * 1024 + # The process's cgroup may be below the mount root (e.g. a desktop user slice). + group = next(line[3:] for line in Path('/proc/self/cgroup').read_text().splitlines() if line.startswith('0::')) + root = Path('/sys/fs/cgroup') / group.lstrip('/') + if '..' in root.parts: + raise ValueError('Cannot resolve the execution memory cgroup.') + while root.is_relative_to('/sys/fs/cgroup'): + ceiling = root / 'memory.max' + if ceiling.exists(): + limit = ceiling.read_text().strip() + if limit != 'max': + available = min(available, max(0, int(limit) - int((root / 'memory.current').read_text()))) + if root == Path('/sys/fs/cgroup'): + break + root = root.parent + return available + + +def _integer(environment, name, default, minimum, maximum): + raw = environment.get(name, str(default)) + if not raw.isdecimal() or not minimum <= int(raw) <= maximum: + raise ValueError(f'{name} must be an integer in {minimum}..{maximum}.') + return int(raw) + + +@dataclass(frozen=True) +class Capacity: + cpu_slots: int + cpu_memory_bytes: int + cuda_memory_bytes: int + reserve_bytes: int + cores: tuple = field(default_factory=usable_cores) + + @classmethod + def configured(cls, environment=None): + environment = os.environ if environment is None else environment + cores = len(usable_cores()) + return cls( + _integer(environment, 'TULDOK_CPU_RUN_SLOTS', min(7, cores), 1, 7), + _integer(environment, 'TULDOK_CPU_RUN_MEMORY_MIB', 1024, 64, 1048576) * MIB, + _integer(environment, 'TULDOK_CUDA_RUN_MEMORY_MIB', 4096, 64, 1048576) * MIB, + _integer(environment, 'TULDOK_RUN_MEMORY_RESERVE_MIB', 1024, 64, 1048576) * MIB) + + def claim(self, run): + recipe = run.get('material', {}).get('recipe', {}) + device = 'cuda' if recipe.get('runtime_device') == 'cuda' else 'cpu' + resources = recipe.get('resources', {}) + memory = self.cuda_memory_bytes if device == 'cuda' else self.cpu_memory_bytes + return device, max(memory, resources.get('memory_bytes', 0)), resources.get('cpu_cores', min(2, len(self.cores)) if device == 'cuda' else 1) + + def admits(self, claim, active, available_bytes): + device, memory_bytes, cores = claim + maximum = 1 if device == 'cuda' else self.cpu_slots + return (len(active) < MAX_ATTEMPTS + and sum(other[0] == device for other in active) < maximum + and sum(other[2] for other in active) + cores <= len(self.cores) + and sum(other[1] for other in active) + memory_bytes + self.reserve_bytes <= available_bytes) diff --git a/runs.py b/runs.py index af53ad75..6877ffd6 100644 --- a/runs.py +++ b/runs.py @@ -91,6 +91,8 @@ def __init__(self, dataset, pipelines): self._external_blocked = False self._confirmed_external = set() self._closed = False + from run_capacity import Capacity + self._capacity = Capacity.configured() with self.lock: self.runtime_owner = self.db.claim_runtime('workspace_runs') if self.runtime_owner: @@ -313,6 +315,8 @@ def _public(self, run): 'quality_claim', 'execution_kind', 'result_kind', 'material', 'material_sha256', 'approval_id', 'actor_id', 'custody_phase') result = {key: copy.deepcopy(run[key]) for key in keys} + if 'execution_resources' in run: + result['execution_resources'] = copy.deepcopy(run['execution_resources']) if 'approval_material_sha256' in run:result['approval_material_sha256']=run['approval_material_sha256'] association=self.db.execute('SELECT study_id,trial_id FROM model_attempt_reservations WHERE run_id=%s',(run['id'],)).fetchone() if association is not None:result['study_membership']=dict(association) @@ -520,6 +524,9 @@ def _admit_external_prepared(self, body, captured, run_id, staging, *, captured['material']['graph_execution_sha256'] != execution_digest(graph)): raise WorkbenchError('Prepared execution identity differs.', 'stale', 409) self.db.on_rollback(lambda: shutil.rmtree(staging, ignore_errors=True)) + claim = self._capacity.claim({'material': captured['material']}) + if claim[2] > len(self._capacity.cores): + raise WorkbenchError('Recipe CPU reservation exceeds this supervisor CPU pool.', 'unsupported', 409) extension_context=None if recipe_contract.is_evidence(captured['material']['recipe']): if approval_material_sha256 is None: @@ -1279,49 +1286,86 @@ def cancel(self, run_id): def _start(self): with self._thread_lock: + if self._closed or self._stop.is_set(): + return if self._thread is None: self._thread = threading.Thread(target=self._loop, name='workspace-runs', daemon=True) self._thread.start() self._wake.set() def _loop(self): - while not self._stop.is_set(): - if self._blocked or not self.runtime_owner: - self._wake.wait(.25) - self._wake.clear() - continue + from concurrent.futures import ThreadPoolExecutor + from run_capacity import available_memory, MAX_ATTEMPTS + # Only this coordinator owns futures/reservations. Each attempt retains + # its existing independent process custodian and exact job context. + with ThreadPoolExecutor(max_workers=MAX_ATTEMPTS, thread_name_prefix='workspace-attempt') as executor: + active = {} try: - with self.lock: - queued = [self._load(row[0]) for row in self.db.execute('SELECT id FROM workspace_runs')] - queued = sorted((run for run in queued if run['state'] == 'queued' and - (not self._external(run) or (not self._external_blocked and run['id'] in self._confirmed_external))), - key=lambda run: (run['created_at'], run['id'])) - except psycopg.Error: + while not self._stop.is_set(): + for run_id, (future, _, _) in list(active.items()): + if future.done(): + future.result() + del active[run_id] + with self.lock: + blocked = self._blocked or self._external_blocked or not self.runtime_owner + if blocked: + self._wake.wait(.25) + self._wake.clear() + continue + with self.lock: + queued = [self._load(row[0]) for row in self.db.execute('SELECT id FROM workspace_runs')] + queued = sorted((run for run in queued if run['state'] == 'queued' and run['id'] not in active + and (not self._external(run) or run['id'] in self._confirmed_external)), + key=lambda run: (run['created_at'], run['id'])) + try: + headroom = available_memory() + except (OSError, ValueError, KeyError, StopIteration): + headroom = 0 + for run in queued: + claim = self._capacity.claim(run) + with self.lock: + blocked = self._blocked or self._external_blocked + if self._stop.is_set() or blocked: + break + if self._capacity.admits(claim, [item[1] for item in active.values()], headroom): + used = {cpu for _, _, cpus in active.values() for cpu in cpus} + cpus = tuple(cpu for cpu in self._capacity.cores if cpu not in used)[:claim[2]] + active[run['id']] = (executor.submit(self._run_attempt, run, cpus, claim), claim, cpus) + self._wake.wait(.05 if active else .25) + self._wake.clear() + finally: + # On coordinator/storage failure, active attempts observe stop + # and drain before executor shutdown releases ownership. self._stop.set() - return - if not queued: - self._wake.wait(.25) - self._wake.clear() - continue + + def _run_attempt(self, admitted, cpus, claim): + affinity = os.sched_getaffinity(0) + try: + # Linux affinity belongs to this executor thread. Children inherit it; + # other attempts and the HTTP/coordinator threads remain independent. + os.sched_setaffinity(0, cpus) + with self.access._job('runs', admitted['id'], admitted.get('admission_id'), admitted['request_sha256']): + with self.lock, self.db.transaction(): + current = self._load(admitted['id']) + current['execution_resources'] = {'device': claim[0], 'memory_reservation_bytes': claim[1], + 'cpu_affinity': list(cpus), 'host_memory_reserve_bytes': self._capacity.reserve_bytes} + self._save(current) + self._execute(admitted['id']) + except Exception: try: - admitted = queued[0] - with self.access._job('runs', admitted['id'], admitted.get('admission_id'), admitted['request_sha256']): - self._execute(admitted['id']) - except Exception as error: - try: - with self.access._reconcile(), self.lock, self.db.transaction(): - run = self._load(queued[0]['id']) - if run['state'] in ACTIVE: - run['error'] = {'code': 'failed', 'message': 'Run supervisor could not complete this attempt.'} - state = self._external_recovery_state(run) if self._external(run) else 'failed' - self._external_blocked |= state == 'unknown' and self._external(run) - self._event(run, state, run['error']['message']) - self._save(run) - except psycopg.Error: - # _execute's finally owns child drainage even when persistence - # disappears. The retained active row reconciles on reopening. - self._stop.set() - return + with self.access._reconcile(), self.lock, self.db.transaction(): + run = self._load(admitted['id']) + if run['state'] in ACTIVE: + run['error'] = {'code': 'failed', 'message': 'Run supervisor could not complete this attempt.'} + state = self._external_recovery_state(run) if self._external(run) else 'failed' + self._external_blocked |= state == 'unknown' and self._external(run) + self._event(run, state, run['error']['message']) + self._save(run) + except psycopg.Error: + self._stop.set() + finally: + os.sched_setaffinity(0, affinity) + self._wake.set() def _execute(self, run_id): with self.lock: @@ -2132,12 +2176,14 @@ def compare_model_evidence(self, run_id, other_run_id, body=None): def close(self): - self._closed = True - self._stop.set() - self._wake.set() - if self._thread is not None: - self._thread.join(timeout=60) - if self._thread.is_alive(): + with self._thread_lock: + self._closed = True + self._stop.set() + self._wake.set() + thread = self._thread + if thread is not None: + thread.join(timeout=60) + if thread.is_alive(): raise RuntimeError('Training supervisor did not drain; metadata must remain open.') if self.runtime_owner: try: diff --git a/tests/test_recipe_contract.py b/tests/test_recipe_contract.py index 5b3b4c98..235c4140 100644 --- a/tests/test_recipe_contract.py +++ b/tests/test_recipe_contract.py @@ -26,6 +26,17 @@ def definition(source=b'raise RuntimeError("preflight must never execute")\n'): class ContractTests(unittest.TestCase): + def test_execution_reservations_are_bounded_and_captured(self): + value = definition() + value['resources'].update(cpu_cores=12, memory_bytes=24 * 1024 ** 3) + self.assertEqual(contract.decode_definition(json.dumps(value).encode()), value) + for name, bad in [('cpu_cores', 0), ('cpu_cores', True), ('cpu_cores', 257), + ('memory_bytes', -1), ('memory_bytes', True), ('memory_bytes', 1024 ** 4 + 1)]: + changed = copy.deepcopy(value) + changed['resources'][name] = bad + with self.subTest(name=name, value=bad), self.assertRaises(WorkbenchError): + contract.decode_definition(json.dumps(changed).encode()) + def test_parameters_and_closed_tokens(self): value=definition() self.assertEqual(contract.decode_definition(json.dumps(value).encode()),value) diff --git a/tests/test_run_capacity.py b/tests/test_run_capacity.py new file mode 100644 index 00000000..1721f363 --- /dev/null +++ b/tests/test_run_capacity.py @@ -0,0 +1,44 @@ +"""Admission budgets cannot oversubscribe devices, memory or attempt capacity.""" +import unittest + +from run_capacity import Capacity, MIB, available_memory + + +class CapacityTests(unittest.TestCase): + def setUp(self): + self.capacity = Capacity(3, 1024, 4096, 1024, cores=tuple(range(8))) + + def test_cpu_and_cuda_are_independent_and_cuda_is_exclusive(self): + cpu, cuda = ('cpu', 1024, 1), ('cuda', 4096, 2) + self.assertTrue(self.capacity.admits(cpu, [cuda, cpu, cpu], 20000)) + self.assertFalse(self.capacity.admits(cpu, [cpu] * 3, 20000)) + self.assertTrue(self.capacity.admits(cuda, [cpu] * 3, 20000)) + self.assertFalse(self.capacity.admits(cuda, [cuda], 20000)) + + def test_reserves_memory_for_all_active_attempts_and_host(self): + cpu, cuda = ('cpu', 1024, 1), ('cuda', 4096, 2) + self.assertTrue(self.capacity.admits(cpu, [cuda], 6144)) + self.assertFalse(self.capacity.admits(cpu, [cuda], 6143)) + self.assertFalse(self.capacity.admits(cpu, [], 0)) + + def test_classification_uses_captured_recipe_device_not_job_name(self): + self.assertEqual(self.capacity.claim({'name': 'cuda'}), ('cpu', 1024, 1)) + self.assertEqual(self.capacity.claim({'material': {'recipe': {'runtime_device': 'cuda'}}}), ('cuda', 4096, 2)) + + def test_explicit_recipe_resources_cannot_lower_operator_memory_floor(self): + value = {'material': {'recipe': {'resources': {'cpu_cores': 4, 'memory_bytes': 512}}}} + self.assertEqual(self.capacity.claim(value), ('cpu', 1024, 4)) + value['material']['recipe']['resources']['memory_bytes'] = 8192 + self.assertEqual(self.capacity.claim(value), ('cpu', 8192, 4)) + + def test_configuration_is_explicit_and_invalid_settings_fail(self): + configured = Capacity.configured({'TULDOK_CPU_RUN_SLOTS': '6', 'TULDOK_CPU_RUN_MEMORY_MIB': '4096', + 'TULDOK_CUDA_RUN_MEMORY_MIB': '8192', 'TULDOK_RUN_MEMORY_RESERVE_MIB': '8192'}) + self.assertEqual(configured, Capacity(6, 4096 * MIB, 8192 * MIB, 8192 * MIB)) + for name, value in [('TULDOK_CPU_RUN_SLOTS', '8'), ('TULDOK_CPU_RUN_SLOTS', '0'), + ('TULDOK_CPU_RUN_MEMORY_MIB', 'nan'), ('TULDOK_RUN_MEMORY_RESERVE_MIB', '-1')]: + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + Capacity.configured({name: value}) + + def test_host_memory_observation_is_positive(self): + self.assertGreater(available_memory(), 0) diff --git a/tests/test_run_concurrency.py b/tests/test_run_concurrency.py new file mode 100644 index 00000000..9f6f7f71 --- /dev/null +++ b/tests/test_run_concurrency.py @@ -0,0 +1,87 @@ +"""Real catalog and custodians: independent attempts overlap and drain separately.""" +import copy +import os +import time +from unittest.mock import patch + +from test_external_runs import ExternalCase +from runs import process_status +from run_capacity import Capacity + + +class ConcurrentRuns(ExternalCase): + def sleeping(self): + graph = copy.deepcopy(self.graph) + graph['nodes'][1]['parameters']['mode'] = 'sleep' + self.graph = self.dataset.pipelines.save(graph['id'], + {'revision': graph['revision'], 'document': graph}) + self.preflight = self.dataset.pipelines.preflight_external(self.graph['id'], self.graph['revision']) + + def test_two_cpu_custodians_overlap_and_cancel_independently(self): + with self.owner.bound(self.dataset): + self.sleeping() + runs = [self.queued(self.body(approval=self.approve())) for _ in range(2)] + owners = [] + self.dataset.runs._start() + try: + until = time.monotonic() + 5 + while time.monotonic() < until: + with self.owner.bound(self.dataset): + current = [self.dataset.runs.get(run['id']) for run in runs] + if all(run.get('custody_phase') == 'go_intent' for run in current): + break + time.sleep(.03) + self.assertTrue(all(run['state'] == 'running' and run['custody_phase'] == 'go_intent' + for run in current), [(r['state'], r.get('custody_phase')) for r in current]) + with self.dataset.lock: + owners = [self.dataset.runs._load(r['id'])['custody_identity']['helper'] for r in current] + self.assertNotEqual(owners[0], owners[1]) + assigned = [set(r['execution_resources']['cpu_affinity']) for r in current] + self.assertFalse(assigned[0] & assigned[1]) + for owner, cpus in zip(owners, assigned): + self.assertEqual(set(os.sched_getaffinity(owner['pid'])), cpus) + with self.owner.bound(self.dataset): + self.dataset.runs.cancel(runs[0]['id']) + self.assertEqual(self.await_state(runs[0]['id'], ('cancelled', 'unknown'))['state'], 'cancelled') + with self.owner.bound(self.dataset): + self.assertEqual(self.dataset.runs.get(runs[1]['id'])['state'], 'running') + finally: + self.dataset.close() + for owner in owners: + self.assertEqual(process_status(owner), 'ceased') + + def test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits(self): + with self.owner.bound(self.dataset): + self.sleeping() + runs = [self.queued(self.body(approval=self.approve())) for _ in range(3)] + gpu = {run['id'] for run in runs[:2]} + capacity = Capacity(1, 1024, 1024, 1024) + # Real custodians exercise dispatch; device classification is explicitly + # simulated here. Actual CUDA computation is qualified by the lab fit. + with patch.object(self.dataset.runs, '_capacity', capacity), patch.object(Capacity, 'claim', + side_effect=lambda run: ('cuda' if run['id'] in gpu else 'cpu', 1024, 1)): + self.dataset.runs._start() + try: + for index in (0, 2): + self.await_state(runs[index]['id'], ('running',)) + with self.owner.bound(self.dataset): + self.assertEqual(self.dataset.runs.get(runs[1]['id'])['state'], 'queued') + self.dataset.runs.cancel(runs[0]['id']) + self.assertEqual(self.await_state(runs[0]['id'], ('cancelled', 'unknown'))['state'], 'cancelled') + self.await_state(runs[1]['id'], ('running',)) + with self.owner.bound(self.dataset): + self.assertEqual(self.dataset.runs.get(runs[2]['id'])['state'], 'running') + finally: + self.dataset.close() + self.assertFalse(self.dataset.runs._thread.is_alive()) + + def test_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child(self): + run = self.queued(self.body(approval=self.approve())) + with patch('run_capacity.available_memory', return_value=0), patch('recipe_process.launch') as launch: + self.dataset.runs._start() + time.sleep(.3) + with self.owner.bound(self.dataset): + self.assertEqual(self.dataset.runs.get(run['id'])['state'], 'queued') + self.assertEqual(self.dataset.runs.cancel(run['id'])['state'], 'cancelled') + self.dataset.close() + launch.assert_not_called() From 7fbc82d9abe80182cfb3186a6691649298e98eb7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 16 Oct 2026 21:47:32 -0700 Subject: [PATCH 2/9] fix: fence attempt startup when sibling custody becomes unknown --- .../reports/scheduler-fence-red.json | 27 ++++++++++ .../reports/scheduler-tests-08.json | 34 +++++++++++++ .../reports/scheduler-tests-09.json | 49 +++++++++++++++++++ runs.py | 14 ++++-- tests/test_runs.py | 34 +++++++++++++ 5 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-fence-red.json create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-08.json create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-09.json diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-fence-red.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-fence-red.json new file mode 100644 index 00000000..bbd32823 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-fence-red.json @@ -0,0 +1,27 @@ +{ + "schema": "scheduler-fence-regression-v1", + "source_commit": "5e18837e1b580b5b831118adb0b0dc8b1dbba4e7", + "source_runs_sha256": "c5b186929be4544dd64c0e098b670eb21b51277f185fc7ea8421f6e1f9eb5324", + "test_source_sha256": "9aa59e7bf39fcf942be63ad915f3d5231e39fea63a9ebfc1e41edae774b2f7b9", + "command": [ + "../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_runs.py", + "-k", + "sibling_unknown", + "-v" + ], + "environment": "dedicated isolated UTF8 PostgreSQL fixture; DSN omitted", + "observed_failure": "AssertionError: True is not false : a worker started after unknown sibling custody", + "tests_run": 1, + "elapsed_test_seconds": 0.838, + "exit_code": 1, + "cleanup": "owner.close drained the actual sleeping worker in finally", + "fix": "common fence rechecked under catalog transaction at attempt entry and both worker paths; queued work stays queued if sibling custody is unknown" +} diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-08.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-08.json new file mode 100644 index 00000000..3b4ae6f3 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-08.json @@ -0,0 +1,34 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T04:44:52.273897+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "source_sha256": { + "runs.py": "6dc3b9dafd15a9de43cc8a8686384cc13bec7041a3c88c73f9b1925f64479d92", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_runs.py", + "-v", + "-k", + "test_runs.RunTests.*" + ], + "exit_code": 0, + "elapsed_seconds": 12.13288097800978, + "output": "test_ambiguous_spawn_reconciles_only_after_matching_process_ceases (test_runs.RunTests.test_ambiguous_spawn_reconciles_only_after_matching_process_ceases) ... ok\ntest_current_exposure_refreshes_without_mutating_admission_evidence (test_runs.RunTests.test_current_exposure_refreshes_without_mutating_admission_evidence) ... ok\ntest_database_disconnect_still_drains_child_before_shutdown_returns (test_runs.RunTests.test_database_disconnect_still_drains_child_before_shutdown_returns) ... ok\ntest_lost_ack_repeated_key_returns_receipt_before_new_graph_validation (test_runs.RunTests.test_lost_ack_repeated_key_returns_receipt_before_new_graph_validation) ... ok\ntest_lost_commit_ack_reconciliation_wakes_exact_committed_queue (test_runs.RunTests.test_lost_commit_ack_reconciliation_wakes_exact_committed_queue) ... ok\ntest_queued_cancel_and_peer_supervisor_are_honest (test_runs.RunTests.test_queued_cancel_and_peer_supervisor_are_honest) ... ok\ntest_recovery_keeps_observed_live_prior_process_unknown (test_runs.RunTests.test_recovery_keeps_observed_live_prior_process_unknown) ... ok\ntest_restart_does_not_replay_queue_or_signal_reused_pid (test_runs.RunTests.test_restart_does_not_replay_queue_or_signal_reused_pid) ... ok\ntest_restart_during_proven_pre_spawn_validation_is_interrupted (test_runs.RunTests.test_restart_during_proven_pre_spawn_validation_is_interrupted) ... ok\ntest_running_cancel_confirms_real_process_group_drain (test_runs.RunTests.test_running_cancel_confirms_real_process_group_drain) ... ok\ntest_sibling_unknown_between_capacity_check_and_entry_prevents_worker_start (test_runs.RunTests.test_sibling_unknown_between_capacity_check_and_entry_prevents_worker_start) ... ok\ntest_unknown_attempt_fences_already_queued_work (test_runs.RunTests.test_unknown_attempt_fences_already_queued_work) ... ok\ntest_worker_failure_retry_preserves_frozen_parent_and_reuses_no_checkpoint (test_runs.RunTests.test_worker_failure_retry_preserves_frozen_parent_and_reuses_no_checkpoint) ... ok\n\n----------------------------------------------------------------------\nRan 13 tests in 11.857s\n\nOK\n" + } + ], + "elapsed_seconds": 12.132910197993624, + "passed": true +} diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-09.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-09.json new file mode 100644 index 00000000..3178cdcc --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-09.json @@ -0,0 +1,49 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T04:45:03.547241+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "source_sha256": { + "runs.py": "6dc3b9dafd15a9de43cc8a8686384cc13bec7041a3c88c73f9b1925f64479d92", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_run_concurrency.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 22.022980377005297, + "output": "test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits (test_run_concurrency.ConcurrentRuns.test_busy_cuda_slot_does_not_block_cpu_and_next_cuda_waits) ... ok\ntest_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child (test_run_concurrency.ConcurrentRuns.test_memory_shortage_preserves_queue_and_queued_cancel_creates_no_child) ... ok\ntest_two_cpu_custodians_overlap_and_cancel_independently (test_run_concurrency.ConcurrentRuns.test_two_cpu_custodians_overlap_and_cancel_independently) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 21.719s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_external_runs.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 72.5197094000032, + "output": "test_consumption_and_run_rollback_together (test_external_runs.ExternalAdmissionTests.test_consumption_and_run_rollback_together) ... ok\ntest_lost_ack_is_not_dispatched_by_an_already_running_queue (test_external_runs.ExternalAdmissionTests.test_lost_ack_is_not_dispatched_by_an_already_running_queue) ... ok\ntest_lost_commit_ack_reconciles_without_new_approval (test_external_runs.ExternalAdmissionTests.test_lost_commit_ack_reconciles_without_new_approval) ... ok\ntest_revocation_and_source_drift_leave_no_run_or_consumption (test_external_runs.ExternalAdmissionTests.test_revocation_and_source_drift_leave_no_run_or_consumption) ... ok\ntest_same_boot_missing_drain_receipt_fences_after_go_intent (test_external_runs.ExternalAdmissionTests.test_same_boot_missing_drain_receipt_fences_after_go_intent) ... ok\ntest_same_key_actor_and_digest_reconcile_exact_consumption (test_external_runs.ExternalAdmissionTests.test_same_key_actor_and_digest_reconcile_exact_consumption) ... ok\ntest_same_key_reconciles_after_approval_expiry_and_graph_drift (test_external_runs.ExternalAdmissionTests.test_same_key_reconciles_after_approval_expiry_and_graph_drift) ... ok\ntest_real_running_cancel_retains_consumed_approval (test_external_runs.ExternalExecutionTests.test_real_running_cancel_retains_consumed_approval) ... ok\ntest_real_unicode_parameter_descendant_output_cold_download_and_restart (test_external_runs.ExternalExecutionTests.test_real_unicode_parameter_descendant_output_cold_download_and_restart) ... ok\ntest_unknown_go_commit_ack_never_sends_go_to_real_helper (test_external_runs.ExternalExecutionTests.test_unknown_go_commit_ack_never_sends_go_to_real_helper) ... ok\ntest_stopped_real_helper_deadline_is_unknown_until_actual_drain (test_external_runs.ExternalStoppedHelperTests.test_stopped_real_helper_deadline_is_unknown_until_actual_drain) ... ok\n\n----------------------------------------------------------------------\nRan 11 tests in 72.175s\n\nOK\n" + } + ], + "elapsed_seconds": 94.54278237899416, + "passed": true +} diff --git a/runs.py b/runs.py index 6877ffd6..280a4698 100644 --- a/runs.py +++ b/runs.py @@ -863,7 +863,7 @@ def _execute_external(self, run_id): with self.lock, self.db.transaction(): run = self._load(run_id) self.access.require_job('runs', run_id, run['admission_id'], run['request_sha256']) - if run['state'] != 'queued' or self._external_blocked: + if run['state'] != 'queued' or self._blocked or self._external_blocked or self._stop.is_set(): return if run['spec']['cwd'] != str(self._external_directory(run)): raise WorkbenchError('Restored execution paths cannot be dispatched. Approve a new attempt.', 'stale', 409) @@ -889,7 +889,7 @@ def _execute_external(self, run_id): self._save(run) with self.lock, self.db.transaction(): run = self._load(run_id) - if run['state'] == 'cancelling' or self._stop.is_set(): + if run['state'] == 'cancelling' or self._stop.is_set() or self._blocked or self._external_blocked: handle.cancel('cancelled') continue run['custody_phase'] = 'go_intent' @@ -1347,6 +1347,10 @@ def _run_attempt(self, admitted, cpus, claim): with self.access._job('runs', admitted['id'], admitted.get('admission_id'), admitted['request_sha256']): with self.lock, self.db.transaction(): current = self._load(admitted['id']) + # Executor submission is provisional. This transaction is + # the dispatch decision shared with sibling unknown fences. + if current['state'] != 'queued' or self._blocked or self._external_blocked or self._stop.is_set(): + return current['execution_resources'] = {'device': claim[0], 'memory_reservation_bytes': claim[1], 'cpu_affinity': list(cpus), 'host_memory_reserve_bytes': self._capacity.reserve_bytes} self._save(current) @@ -1376,7 +1380,7 @@ def _execute(self, run_id): with self.lock, self.db.transaction(): run = self._load(run_id) self.access.require_job('runs', run_id, run.get('admission_id'), run['request_sha256']) - if run['state'] != 'queued': + if run['state'] != 'queued' or self._blocked or self._external_blocked or self._stop.is_set(): return self._event(run, 'running', 'Verifying frozen release and adapter before starting the CPU worker.') run['started_at'] = timestamp() @@ -1421,6 +1425,10 @@ def _execute(self, run_id): current = self._load(run_id) current['execution_test_exposure'] = self.pipelines.exposures(current['source_records']) self._save(current) + if self._blocked or self._external_blocked: + self._event(current, 'interrupted', 'Sibling custody became unknown before worker creation.') + self._save(current) + return if current['state'] == 'cancelling' or self._stop.is_set(): self._event(current, 'cancelled', 'Cancelled before worker process creation.') self._save(current) diff --git a/tests/test_runs.py b/tests/test_runs.py index e8f53d0b..8d3165b8 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -9,6 +9,7 @@ import subprocess import sys import time +import threading import unittest import uuid from unittest.mock import patch @@ -21,6 +22,39 @@ class RunTests(WorkspaceCase): + def test_sibling_unknown_between_capacity_check_and_entry_prevents_worker_start(self): + run = self.submit_queued() + owner = self.dataset.runs + from run_capacity import Capacity + admit = Capacity.admits + fence, spawned = threading.Event(), threading.Event() + popen = subprocess.Popen + + def late_fence(capacity, claim, active, available): + allowed = admit(capacity, claim, active, available) + with self.dataset.lock: + owner._external_blocked = True + fence.set() + return allowed + + def sleeping_worker(*_args, **kwargs): + child = popen([sys.executable, '-c', 'import time; time.sleep(30)'], **kwargs) + spawned.set() + return child + + # The sibling's unknown outcome is injected precisely after the last + # coordinator observation; the target uses the real builtin worker path. + with patch.object(self.pipelines, 'probe', return_value=self.ready()), \ + patch.object(Capacity, 'admits', autospec=True, side_effect=late_fence), \ + patch('runs.subprocess.Popen', side_effect=sleeping_worker): + try: + owner._start() + self.assertTrue(fence.wait(5)) + self.assertFalse(spawned.wait(1), 'a worker started after unknown sibling custody') + self.assertEqual(owner.get(run['id'])['state'], 'queued') + finally: + owner.close() + def body(self, key='contract-request-0001'): return {'request_key': key, 'graph_id': self.graph['id'], 'graph_revision': self.graph['revision'], 'validation_token': self.validate()['validation_token']} From a9e99d28fb88320323dbcbfb63ae8224b89043ff Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 16 Oct 2026 21:55:28 -0700 Subject: [PATCH 3/9] fix: coordinate drain fences with concurrent dispatch --- docs/plans/concurrent-cpu-cuda/plan.md | 17 +- .../reports/actual-device-overlap-03.json | 2026 +++++++++++++++++ .../reports/independent-review.json | 27 + .../reports/scheduler-tests-10.json | 34 + runs.py | 19 +- 5 files changed, 2113 insertions(+), 10 deletions(-) create mode 100644 docs/plans/concurrent-cpu-cuda/reports/actual-device-overlap-03.json create mode 100644 docs/plans/concurrent-cpu-cuda/reports/independent-review.json create mode 100644 docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-10.json diff --git a/docs/plans/concurrent-cpu-cuda/plan.md b/docs/plans/concurrent-cpu-cuda/plan.md index c025959f..674c1760 100644 --- a/docs/plans/concurrent-cpu-cuda/plan.md +++ b/docs/plans/concurrent-cpu-cuda/plan.md @@ -1,6 +1,6 @@ # Concurrent local CPU and CUDA Runs -Status: Active; operation start. User authorized simultaneous CPU Runs alongside +Status: Accepted; operation start. User authorized simultaneous CPU Runs alongside one CUDA fit and continuing the pressure-model sequence. Owner: root; branch performance/concurrent-cpu-cuda-20261017, base8af5a6a1e288e5b2f86a75b53cab93ca71222a04. Retain this branch for review/integration; existing PR89 and its source branch @@ -36,12 +36,21 @@ attempts; existing storage-loss/restart/unknown-state lifecycle tests. Actual CU computation and CPU analysis overlap must be observed in the lab before deployment is qualified for that workload. Source/Spec review and exact staged diff required. -Current phase: Verifying. Next slice: final source-frozen lifecycle suite and -independent review. Existing actual process tests passed; GPU-slot dispatch was -simulated in that focused test and does not prove actual CUDA execution. The old +Current phase: terminal for scheduler implementation and hardware qualification. +Next slice: none in this plan; deployment and model experiments continue in the +linked lab plan. All required process checks and both independent review axes +pass. Real CUDA pressure training overlapped managed CPU computation for1.2556s; +the one-update checkpoint restored on CPU with zero updates and CUDA uninitialized. +Exact restored weights/normalization/history and joined executor workers are +recorded in actual-device-overlap-03.json (270.023s total). The old TinyCNN runtime is unavailable locally; its test fails at runtime preflight before the scheduler. No package install or fallback to the lab's CUDA runtime is used. +Independent review found a late dispatch/fence race and one uncoordinated drain +fence writer. The real red startup regression, passing13-case built-in lifecycle +receipt10,14-case external/concurrency receipt09 and the final review retain both +findings and their closure. Commit correction preserves the original history. + Related scientific plan: rheon-surrogate-lab/docs/plans/pressure-throughput-feedback-v1/plan.md. Standards route snapshot:v1:bc3877fc-df9b-41cc-9a60-ec680a5ccf55, 21 selected policies, zero unresolved routing facts. Reservations are admission diff --git a/docs/plans/concurrent-cpu-cuda/reports/actual-device-overlap-03.json b/docs/plans/concurrent-cpu-cuda/reports/actual-device-overlap-03.json new file mode 100644 index 00000000..32b19cf8 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/actual-device-overlap-03.json @@ -0,0 +1,2026 @@ +{ + "schema": "pressure-actual-device-overlap-v1", + "status": "passed", + "source_sha256": "69656a866cdd9dbd9a8a7e809a23e7ad0978ec3b3a9dce366b8ebef0b72ddb44", + "tuldok_source_sha256": { + "runs.py": "fc7a83397404ac77a18a653d5eaaff63773f0818328e7df44e5f00ad64784b5b", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa", + "recipe_contract.py": "4a8c7226a212e8d7ce3639fb160c2d004d3c7d9ffdb0b64c03bc6c60d8769dc7", + "recipe_process.py": "91d392b1c3c88dd43062457e01176551757b835cc5f8cad61e5e9b77795e421d", + "torch_runtime.py": "84a66b6a195c30db01ffc574f94e0139f4df7d7c7872452a3f50ce82e9db3ad0" + }, + "runtime_manifest_sha256": "b4329b27c27622d1fdf2b6c5712d6420c2d3e5fedf01c1450fc1a236cdbc9682", + "configuration": { + "TULDOK_CPU_RUN_SLOTS": "6", + "TULDOK_CPU_RUN_MEMORY_MIB": "4096", + "TULDOK_CUDA_RUN_MEMORY_MIB": "8192", + "TULDOK_RUN_MEMORY_RESERVE_MIB": "8192" + }, + "optimizer_updates_authorized": 1, + "original_catalog_used": false, + "scientific_promotion": false, + "runs": [ + { + "mode": "cpu", + "id": "0e343ac0c3cd45119bc923c380799950", + "material_sha256": "7d5caeeea34e734e5715b7aa1f9b484efb1de50a91e33c00394413b8ce3cc8d4" + }, + { + "mode": "train", + "id": "946ca3ad70bc4ee99b20761c71ae674b", + "material_sha256": "b31a28b127a09381f04803d5de1c301457284e084f130906746f3a27d6e292f4" + }, + { + "mode": "restore", + "id": "b597f3df2ba34485b8640903b6c22760", + "material_sha256": "676ebcef3c7c7962a02ccc42f24588b52b22f922cc23c29c93364e5939c7f051" + } + ], + "observations": [ + { + "elapsed_seconds": 63.97717623699282, + "states": [ + "running", + "queued" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 64.7760341349931, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 65.87825041399628, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 67.89721791699412, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 69.6973073400004, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 70.74097990700102, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 71.62368744300329, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 72.54712175999884, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 73.42114777299867, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 74.17038935799792, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "prepared" + ] + }, + { + "elapsed_seconds": 74.9516968389944, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 75.74590968199482, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 76.5791725970048, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 77.49620787799358, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 78.35172935099399, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 79.15699048399983, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 79.95642378699267, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 81.09157731699815, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 82.01723279200087, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 82.9044735780044, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 83.92632571999275, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 85.02963550999993, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 85.88088133599376, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 86.88312314299401, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 87.8514188919944, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 88.80720891999954, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 89.50032417099283, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 90.37617291099741, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 91.24058034799236, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 92.03479393900489, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 92.78773931300384, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 93.54799776700384, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 94.29176448599901, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 95.03032754399464, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 95.80659674899653, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 96.57393932099512, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 97.39135340800567, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 98.26875224200194, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 99.02362156299932, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 99.77661619600258, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 100.55031662200054, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 101.27573316100461, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 102.11178407099214, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 102.84586223200313, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 103.6141288469953, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 104.45215352799278, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 105.24315347800439, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 105.94795861300372, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 106.67277150700102, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 107.38643771100033, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 108.11772757599829, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 108.82192510999448, + "states": [ + "running", + "running" + ], + "custody_phases": [ + "go_intent", + "go_intent" + ] + }, + { + "elapsed_seconds": 109.60376907099271, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 110.36415758699877, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 111.1921955239959, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 111.88675420200161, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 112.42073652500403, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 113.09650207399682, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 113.80253428500146, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 114.53669400699437, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 115.21423110899923, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 115.92185114200402, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 116.68491113999335, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 117.49726836899936, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 118.03898955100158, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 118.79163881599379, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 119.51489541299816, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 120.1828501370037, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 120.90993517500465, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 121.58410699499655, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 122.28377894600271, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 123.00573268599692, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 123.68764049500169, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 124.37580469700333, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 125.07867302000523, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 125.58461384900147, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 126.10704997299763, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 126.62466634399607, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 127.33168724800635, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 128.03342172599514, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 128.73577259900048, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 129.4586848750041, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 130.1819280320051, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 130.92005432500446, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 131.5389576479938, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 132.26299099999596, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 132.9849683640059, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 133.49589218999608, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 134.20594651599822, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 134.72296893199382, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 135.46662395600288, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 136.21229960399796, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 136.93374409800163, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 137.70609565499763, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 138.3719666609977, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 139.02331433199288, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 139.6983900280029, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 140.41355060100614, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 141.11341490100313, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 141.84101270799874, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 142.5846065539954, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 143.32524755499617, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 144.12495772300463, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 144.8755815589975, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 145.49245310400147, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 146.2315708929964, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 146.9073272849928, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 147.41721754499304, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 148.13913099000638, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 148.86929314000008, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 149.5975320389989, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 150.37610375699296, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 151.01026161399204, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 151.72289900199394, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 152.4225738170062, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 153.12310415800312, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 153.82389338000212, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 154.52534451500105, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 155.24387739799567, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 155.99188248999417, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 156.78984230999777, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 157.375138702002, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 158.08442994000507, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 158.79720178700518, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 159.5011383670062, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 160.026091214997, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 160.74891988599848, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 161.45951058699575, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 162.25045396199857, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 163.03112596199207, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 163.67280727499747, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 164.445690381006, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 165.15821604100347, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 165.87351539899828, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 166.5811962430016, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 167.297052074995, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 167.96430664800573, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 168.72635746799642, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 169.60954202299763, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 170.3060898620024, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 171.04079182799615, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 171.6761492539954, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 172.37078178400407, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 173.08998861799773, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 173.79755738200038, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 174.51602950399683, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 175.28306983299262, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 176.10520416600048, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 176.85896844799572, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 177.53536381700542, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 178.2601677670027, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 178.96718964900356, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 179.73644690999936, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 180.45274560099642, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 181.1706119360024, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 181.84704917400086, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 182.66715162299806, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 183.40813634700316, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 184.09445474100357, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 184.7971834659984, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 185.5125546039926, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 186.2462001459935, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 187.06514530499408, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 187.61892845100374, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 188.45162933900428, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 189.26119259400002, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 190.0185633969959, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 190.70812704699347, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 191.40003457400599, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 192.06382517999737, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 192.6609588059946, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 193.45013870300318, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 194.20389112200064, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 194.8889178840036, + "states": [ + "running", + "completed" + ], + "custody_phases": [ + "go_intent", + "drained" + ] + }, + { + "elapsed_seconds": 195.6596973489941, + "states": [ + "completed", + "completed" + ], + "custody_phases": [ + "drained", + "drained" + ] + } + ], + "execution_resources": [ + { + "cpu_affinity": [ + 2 + ], + "device": "cpu", + "host_memory_reserve_bytes": 8589934592, + "memory_reservation_bytes": 4294967296 + }, + { + "cpu_affinity": [ + 3, + 4 + ], + "device": "cuda", + "host_memory_reserve_bytes": 8589934592, + "memory_reservation_bytes": 8589934592 + } + ], + "cpu_compute": { + "device": "cpu", + "cuda_initialized": false, + "compute_started_unix": 1792212623.3215737, + "cpu_affinity": [ + 2 + ], + "optimizer_updates": 0, + "compute_finished_unix": 1792212743.324527, + "iterations": 464422, + "checksum": 23212.560546875, + "torch_threads": 1, + "scope": "real CPU arithmetic and overlap qualification; no scientific/model accuracy claim" + }, + "cuda_execution": { + "amp": false, + "batches_cuda": true, + "cuda_cpu_pressure_max_abs_difference": 8.01061787569779e-06, + "deterministic_algorithms": true, + "device": "cuda:0", + "gpu": "NVIDIA GeForce RTX 5090 Laptop GPU", + "model_dtype": "float32", + "parameters_gradients_adam_moments_cuda": true, + "peak_allocated_bytes": 767402496, + "peak_reserved_bytes": 790626304, + "tf32": false + }, + "cuda_training_interval": { + "training_started_unix": 1792212655.512819, + "training_finished_unix": 1792212656.7684212 + }, + "actual_optimizer_cpu_overlap_seconds": 1.2556021213531494, + "cpu_restore": { + "cuda_initialized": false, + "device": "cpu", + "optimizer_updates": 0 + }, + "restored_artifacts_exact": true, + "all_executor_workers_joined": true, + "elapsed_seconds": 270.0232263039943, + "wall_monotonic_offset_change_seconds": -2.384185791015625e-07 +} diff --git a/docs/plans/concurrent-cpu-cuda/reports/independent-review.json b/docs/plans/concurrent-cpu-cuda/reports/independent-review.json new file mode 100644 index 00000000..9334a7cd --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/independent-review.json @@ -0,0 +1,27 @@ +{ + "schema": "concurrent-runs-two-axis-review-v1", + "fixed_base": "8af5a6a1e288e5b2f86a75b53cab93ca71222a04", + "first_reviewed_head": "5e18837e1b580b5b831118adb0b0dc8b1dbba4e7", + "correction_head": "7fbc82d9abe80182cfb3186a6691649298e98eb7", + "final_source_evidence": "scheduler-tests-10.json and actual-device-overlap-03.json source hashes", + "standards": { + "reviewer": "scheduler_standards_review", + "findings": [ + "Dispatch could cross an unknown-custody fence after capacity check.", + "One builtin final-drain fence writer was outside the catalog lock." + ], + "resolution": "Provisional executor submission; dispatch and every fence writer coordinate at entry/transition under catalog lock. Lock/persistence failure sets stop.", + "remaining_findings": 0 + }, + "spec": { + "reviewer": "scheduler_spec_review", + "findings": [ + "Same dispatch/fence race.", + "Retained06/07 lacked explicit built-in lifecycle-class receipt." + ], + "resolution": "Real red startup regression and13 passing builtin lifecycle cases retained; sibling fence admission fixed.", + "remaining_findings": 0 + }, + "scope": "read-only independent source/spec reviews; execution evidence owned by root; legacy TinyCNN numerical preflight remains unavailable, not claimed passing", + "actual_hardware": "real CUDA pressure update1, managed CPU computation0updates, fresh CPUrestore0updates; exact artifacts; all completed/drained" +} diff --git a/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-10.json b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-10.json new file mode 100644 index 00000000..5752bb34 --- /dev/null +++ b/docs/plans/concurrent-cpu-cuda/reports/scheduler-tests-10.json @@ -0,0 +1,34 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T04:49:17.573094+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "source_sha256": { + "runs.py": "fc7a83397404ac77a18a653d5eaaff63773f0818328e7df44e5f00ad64784b5b", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok/../app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_runs.py", + "-v", + "-k", + "test_runs.RunTests.*" + ], + "exit_code": 0, + "elapsed_seconds": 13.363920347008388, + "output": "test_ambiguous_spawn_reconciles_only_after_matching_process_ceases (test_runs.RunTests.test_ambiguous_spawn_reconciles_only_after_matching_process_ceases) ... ok\ntest_current_exposure_refreshes_without_mutating_admission_evidence (test_runs.RunTests.test_current_exposure_refreshes_without_mutating_admission_evidence) ... ok\ntest_database_disconnect_still_drains_child_before_shutdown_returns (test_runs.RunTests.test_database_disconnect_still_drains_child_before_shutdown_returns) ... ok\ntest_lost_ack_repeated_key_returns_receipt_before_new_graph_validation (test_runs.RunTests.test_lost_ack_repeated_key_returns_receipt_before_new_graph_validation) ... ok\ntest_lost_commit_ack_reconciliation_wakes_exact_committed_queue (test_runs.RunTests.test_lost_commit_ack_reconciliation_wakes_exact_committed_queue) ... ok\ntest_queued_cancel_and_peer_supervisor_are_honest (test_runs.RunTests.test_queued_cancel_and_peer_supervisor_are_honest) ... ok\ntest_recovery_keeps_observed_live_prior_process_unknown (test_runs.RunTests.test_recovery_keeps_observed_live_prior_process_unknown) ... ok\ntest_restart_does_not_replay_queue_or_signal_reused_pid (test_runs.RunTests.test_restart_does_not_replay_queue_or_signal_reused_pid) ... ok\ntest_restart_during_proven_pre_spawn_validation_is_interrupted (test_runs.RunTests.test_restart_during_proven_pre_spawn_validation_is_interrupted) ... ok\ntest_running_cancel_confirms_real_process_group_drain (test_runs.RunTests.test_running_cancel_confirms_real_process_group_drain) ... ok\ntest_sibling_unknown_between_capacity_check_and_entry_prevents_worker_start (test_runs.RunTests.test_sibling_unknown_between_capacity_check_and_entry_prevents_worker_start) ... ok\ntest_unknown_attempt_fences_already_queued_work (test_runs.RunTests.test_unknown_attempt_fences_already_queued_work) ... ok\ntest_worker_failure_retry_preserves_frozen_parent_and_reuses_no_checkpoint (test_runs.RunTests.test_worker_failure_retry_preserves_frozen_parent_and_reuses_no_checkpoint) ... ok\n\n----------------------------------------------------------------------\nRan 13 tests in 13.020s\n\nOK\n" + } + ], + "elapsed_seconds": 13.363949083010084, + "passed": true +} diff --git a/runs.py b/runs.py index 280a4698..a54ebc74 100644 --- a/runs.py +++ b/runs.py @@ -1543,12 +1543,19 @@ def _execute(self, run_id): try: self._drain(process, owner) except (WorkbenchError, OSError, subprocess.SubprocessError): - self._blocked = True - with self.lock, self.db.transaction(): - current = self._load(run_id) - current['error'] = {'code': 'unknown', 'message': 'Worker cessation could not be confirmed.'} - self._event(current, 'unknown', current['error']['message']) - self._save(current) + try: + with self.lock: + self._blocked = True + with self.db.transaction(): + current = self._load(run_id) + current['error'] = {'code': 'unknown', 'message': 'Worker cessation could not be confirmed.'} + self._event(current, 'unknown', current['error']['message']) + self._save(current) + except BaseException: + # CatalogLock acquisition can itself fail after storage + # loss. Stop dispatch even when no durable fence is possible. + self._stop.set() + raise if process is not None and process.stdout is not None: process.stdout.close() From 1688f7d9fc6f941d8a23d8b077df6af4213f683a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 16 Oct 2026 22:28:56 -0700 Subject: [PATCH 4/9] fix: allow bounded managed CUDA training deadlines up to thirty minutes --- docs/contracts/managed-torch-runtime.md | 7 ++ docs/plans/cuda-training-deadline/plan.md | 18 +++++ .../reports/tests-01.json | 66 +++++++++++++++++++ .../reports/verification.json | 18 +++++ recipe_contract.py | 4 +- recipe_process.py | 3 + tests/test_bound_torch_recipe_contract.py | 9 +++ tests/test_torch_runtime_lifecycle.py | 8 ++- 8 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 docs/plans/cuda-training-deadline/plan.md create mode 100644 docs/plans/cuda-training-deadline/reports/tests-01.json create mode 100644 docs/plans/cuda-training-deadline/reports/verification.json diff --git a/docs/contracts/managed-torch-runtime.md b/docs/contracts/managed-torch-runtime.md index 73a98058..e063af0b 100644 --- a/docs/contracts/managed-torch-runtime.md +++ b/docs/contracts/managed-torch-runtime.md @@ -99,3 +99,10 @@ snapshot as a bound trainer. This keeps fresh CPU restore and its typed Run artifact inputs in the current Project contract. Bound evidence still requires Project v2; unbound field-evidence recipes and other ordinary runtime profiles are not admitted through this Project v2 path. + +Explicit managed CUDA recipes may reserve1–1800seconds of execution. Ordinary +CPU recipes retain their600second bound. The neutral custody owner and recipe +admission share the CUDA maximum; operators still choose a smaller exact +per-attempt deadline. Exceeding the requested deadline stops/drains the attempt +and is not a numerical accuracy result. Installed-runtime verification before +leader creation retains its separate finite preparation allowance. diff --git a/docs/plans/cuda-training-deadline/plan.md b/docs/plans/cuda-training-deadline/plan.md new file mode 100644 index 00000000..f47f5498 --- /dev/null +++ b/docs/plans/cuda-training-deadline/plan.md @@ -0,0 +1,18 @@ +# Finite CUDA training deadline + +Status: Active. User authorized Tuldok improvements needed for the ongoing +CUDA-training/CPU-inference sequence. One bounded slice extends only explicitly +managed CUDA execution from600 to1800seconds. Ordinary CPU remains600seconds; +captured CPU large-output profile remains2400seconds. One CUDA slot, memory +reservations, stopping/physical gates and optimizer budgets do not change. + +Evidence: the5000-update control Run f8e6bd15f5ba4e6bbd5571071ecb2798 drained +after the600second execution deadline, last reported update4500. Retain that +failed attempt and close its Study before a fresh from-initialization run. + +Write set: recipe admission, custody validation, affected contract tests/docs. +The custody owner supplies the shared CUDA bound to admission. No new receipt or +spec dialect, runtime installation or model arithmetic. Verify rejection above +the CUDA limit and unchanged CPU restriction, plus lifecycle cancellation/drain. +Deploy only after isolated verification and no live active Runs. Do not replay +any unknown mutation or overwrite completed migration gates. diff --git a/docs/plans/cuda-training-deadline/reports/tests-01.json b/docs/plans/cuda-training-deadline/reports/tests-01.json new file mode 100644 index 00000000..ccf9ec03 --- /dev/null +++ b/docs/plans/cuda-training-deadline/reports/tests-01.json @@ -0,0 +1,66 @@ +{ + "schema": "pressure-throughput-test-receipt-v1", + "started_at_utc": "2026-10-17T05:27:47.005083+00:00", + "source": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "python": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python", + "source_sha256": { + "runs.py": "fc7a83397404ac77a18a653d5eaaff63773f0818328e7df44e5f00ad64784b5b", + "run_capacity.py": "6811c5b2255b2a57c92147bab448a82c89876072d260ef217eb7284c511e8cfa" + }, + "database": "explicit isolated fixture; DSN omitted", + "tests": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_recipe_contract.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 8.30771823800751, + "output": "test_cold_runtime_sitecustomize_matches_verified_python_version (test_recipe_contract.ContractTests.test_cold_runtime_sitecustomize_matches_verified_python_version) ... ok\ntest_execution_reservations_are_bounded_and_captured (test_recipe_contract.ContractTests.test_execution_reservations_are_bounded_and_captured) ... ok\ntest_output_inventory_refuses_extra_links_and_missing (test_recipe_contract.ContractTests.test_output_inventory_refuses_extra_links_and_missing) ... ok\ntest_parameters_and_closed_tokens (test_recipe_contract.ContractTests.test_parameters_and_closed_tokens) ... ok\ntest_real_runtime_closure_without_launch (test_recipe_contract.ContractTests.test_real_runtime_closure_without_launch) ... ok\ntest_strict_native_elf_rejects_headers_paths_and_loader_extensions (test_recipe_contract.ContractTests.test_strict_native_elf_rejects_headers_paths_and_loader_extensions) ... ok\ntest_capture_and_clone_preserve_exact_material_without_execution (test_recipe_contract.RecipePipelineTests.test_capture_and_clone_preserve_exact_material_without_execution) ... ok\ntest_native_policy_is_captured_and_cold_bound_without_execution (test_recipe_contract.RecipePipelineTests.test_native_policy_is_captured_and_cold_bound_without_execution) ... ok\ntest_source_and_input_drift_refuse_capture (test_recipe_contract.RecipePipelineTests.test_source_and_input_drift_refuse_capture) ... ok\n\n----------------------------------------------------------------------\nRan 9 tests in 7.964s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_bound_torch_recipe_contract.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 5.536225854011718, + "output": "test_bound_managed_cpu_keeps_current_profile_without_snapshot_relabel (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_bound_managed_cpu_keeps_current_profile_without_snapshot_relabel) ... ok\ntest_bound_managed_device_capture_and_exact_pre_admission_revalidation (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_bound_managed_device_capture_and_exact_pre_admission_revalidation) ... ok\ntest_cuda_training_deadline_is_finite_and_cpu_remains_bounded (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_cuda_training_deadline_is_finite_and_cpu_remains_bounded) ... ok\ntest_managed_evidence_requires_current_bound_variant_and_no_native_sources (test_bound_torch_recipe_contract.BoundTorchRecipeContractTests.test_managed_evidence_requires_current_bound_variant_and_no_native_sources) ... ok\n\n----------------------------------------------------------------------\nRan 4 tests in 5.207s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_torch_runtime_lifecycle.py", + "-v" + ], + "exit_code": 0, + "elapsed_seconds": 0.9686342519999016, + "output": "test_cancel_during_inventory_drains_without_fork_and_retains_cold_identity (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_cancel_during_inventory_drains_without_fork_and_retains_cold_identity) ... ok\ntest_captured_bootstrap_drift_after_ready_refuses_recipe_entry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_captured_bootstrap_drift_after_ready_refuses_recipe_entry) ... ok\ntest_controller_loss_during_inventory_drains_without_fork (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_controller_loss_during_inventory_drains_without_fork) ... ok\ntest_exact_managed_spec_requires_one_profile_device_and_captured_identity (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_exact_managed_spec_requires_one_profile_device_and_captured_identity) ... ok\ntest_finite_cuda_deadline_keeps_cpu_deadline_unchanged (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_finite_cuda_deadline_keeps_cpu_deadline_unchanged) ... ok\ntest_inventory_deadline_fails_and_drains_before_child_wall_clock (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_inventory_deadline_fails_and_drains_before_child_wall_clock) ... ok\ntest_managed_context_drift_after_ready_drains_without_recipe_entry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_drift_after_ready_drains_without_recipe_entry) ... ok\ntest_managed_context_is_reverified_after_child_exit_before_completion (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_is_reverified_after_child_exit_before_completion) ... ok\ntest_managed_context_keeps_one_exact_bounded_captured_argument (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_keeps_one_exact_bounded_captured_argument) ... ok\ntest_missing_or_changed_inventory_refused_after_go_without_child_or_cpu_retry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_missing_or_changed_inventory_refused_after_go_without_child_or_cpu_retry) ... ok\n\n----------------------------------------------------------------------\nRan 10 tests in 0.894s\n\nOK\n" + } + ], + "elapsed_seconds": 14.812711909005884, + "passed": true +} diff --git a/docs/plans/cuda-training-deadline/reports/verification.json b/docs/plans/cuda-training-deadline/reports/verification.json new file mode 100644 index 00000000..0ca17d9d --- /dev/null +++ b/docs/plans/cuda-training-deadline/reports/verification.json @@ -0,0 +1,18 @@ +{ + "format": "tuldok_cuda_deadline_verification_v1", + "status": "passed", + "tests_receipt_sha256": "6f93018099e46569f961fc0f201f12c555b674ee1a5caa649a47f43c3627cdca", + "source_sha256": { + "recipe_process.py": "c73b96a1eb006f75f98fa66119f4ccb6eb5421123d48bcde48c08138372ebc57", + "recipe_contract.py": "5c3072c7f528f042ceae4375ec9cf8ce404a6f60e71234054ba4aab32d2738bc", + "tests/test_torch_runtime_lifecycle.py": "d370a56576fef912d5c013d8b2e4b3476d2eb4a324f139592323d89da058cafa", + "tests/test_bound_torch_recipe_contract.py": "b817e31f9b0a05cf8ea32a0159a1cc1468e3c4f558294183ec3db1210357a7a2" + }, + "limits": { + "managed_cuda": 1800, + "ordinary_cpu": 600, + "captured_cpu_large": 2400 + }, + "runtime_imported": false, + "optimizer_updates": 0 +} diff --git a/recipe_contract.py b/recipe_contract.py index c0802c10..c8914398 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -8,6 +8,7 @@ import struct from projects import closed, decode, relative_path +from recipe_process import CUDA_WALL_SECONDS_LIMIT from workbench import WorkbenchError, encode NAME = re.compile(r'[A-Za-z][A-Za-z0-9_-]{0,63}') @@ -210,7 +211,8 @@ def decode_definition(raw): bounded_int(output['max_bytes'], 1, 512 * 1024 * 1024 if large else MAX_CAPTURE_BYTES, 'Output bytes') if value['environment'] != {}: raise WorkbenchError('Initial runtime admits no project environment overrides.', 'unsupported', 400) - bounded_int(value['resources']['wall_seconds'], 1, 2400 if large else 600, 'Wall seconds') + wall_limit=2400 if large else CUDA_WALL_SECONDS_LIMIT if value['runtime']==TORCH_PROFILE and value['runtime_device']=='cuda' else 600 + bounded_int(value['resources']['wall_seconds'], 1, wall_limit, 'Wall seconds') bounded_int(value['resources']['max_output_bytes'], 1, 4 * 1024**3 if large else MAX_CAPTURE_BYTES, 'Output bytes') if evidence: if value['arguments'].count({'model_evidence_context':True}) != 1: diff --git a/recipe_process.py b/recipe_process.py index d364f383..8f082cf5 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -23,6 +23,7 @@ GRACE_SECONDS = 3 CLEANUP_SECONDS = 10 STREAM_LIMIT = 1024 * 1024 +CUDA_WALL_SECONDS_LIMIT = 1800 OUTPUT_LIMIT = 64 * 1024 * 1024 CAPTURE_FILE_LIMIT = 64 * 1024 * 1024 LARGE_OUTPUT_PROFILE = 'captured_cpu_large_outputs_v1' @@ -428,6 +429,8 @@ def validate_spec(value): if sum(item['bytes'] for item in files) > capture_limit or sum(item['bytes'] for item in runtime) > 512 * 1024 * 1024: raise ProcessError('Runtime/source closure exceeds its byte bound.') wall_limit, output_limit = 600, OUTPUT_LIMIT + if 'torch_runtime' in value and argv[7]=='cuda': + wall_limit=CUDA_WALL_SECONDS_LIMIT if 'resource_profile' in value: if value['resource_profile'] != LARGE_OUTPUT_PROFILE or 'cpu_snapshot' not in value: raise ProcessError('Large outputs require the explicit qualified CPU resource profile.') diff --git a/tests/test_bound_torch_recipe_contract.py b/tests/test_bound_torch_recipe_contract.py index 4abb1a22..26b883b7 100644 --- a/tests/test_bound_torch_recipe_contract.py +++ b/tests/test_bound_torch_recipe_contract.py @@ -68,3 +68,12 @@ def test_managed_evidence_requires_current_bound_variant_and_no_native_sources(s if changed['schema_version']==2:changed.pop('extension_selection') with self.assertRaises(WorkbenchError): recipe_contract.decode_definition(json.dumps(changed).encode()) + + def test_cuda_training_deadline_is_finite_and_cpu_remains_bounded(self): + self.managed() + value=copy.deepcopy(self.recipe);value['resources']['wall_seconds']=1800 + self.assertEqual(recipe_contract.decode_definition(json.dumps(value).encode())['resources']['wall_seconds'],1800) + for device,seconds in (('cuda',1801),('cpu',601)): + value['runtime_device']=device;value['resources']['wall_seconds']=seconds + with self.subTest(device=device),self.assertRaises(WorkbenchError): + recipe_contract.decode_definition(json.dumps(value).encode()) diff --git a/tests/test_torch_runtime_lifecycle.py b/tests/test_torch_runtime_lifecycle.py index 55a68111..22456b54 100644 --- a/tests/test_torch_runtime_lifecycle.py +++ b/tests/test_torch_runtime_lifecycle.py @@ -136,7 +136,7 @@ def test_exact_managed_spec_requires_one_profile_device_and_captured_identity(se self.assertEqual(custody.validate_spec(spec), spec) for change in ({'env': {'CUDA_VISIBLE_DEVICES': ''}}, {'cpu_snapshot': {}}, {'resource_profile': custody.LARGE_OUTPUT_PROFILE}, - {'wall_seconds': 601}, {'torch_runtime': dict(self.descriptor, extra=True)}): + {'wall_seconds': custody.CUDA_WALL_SECONDS_LIMIT+1}, {'torch_runtime': dict(self.descriptor, extra=True)}): with self.subTest(change=change), self.assertRaises(custody.ProcessError): custody.validate_spec(dict(spec, **change)) for index, value in ((7, 'auto'), (7, 'CPU'), (4, '/tmp/torch_runtime.py'), @@ -150,6 +150,12 @@ def test_exact_managed_spec_requires_one_profile_device_and_captured_identity(se with self.assertRaises(custody.ProcessError): custody.validate_spec(changed) + def test_finite_cuda_deadline_keeps_cpu_deadline_unchanged(self): + spec=self.spec('cuda');spec['wall_seconds']=1800 + self.assertEqual(custody.validate_spec(spec)['wall_seconds'],1800) + spec=self.spec('cpu');spec['wall_seconds']=601 + with self.assertRaises(custody.ProcessError):custody.validate_spec(spec) + def test_missing_or_changed_inventory_refused_after_go_without_child_or_cpu_retry(self): for mutation in ('missing', 'changed'): with self.subTest(mutation=mutation): From 92ef4393c327b9bdb06dc044267a4ce682eca1f9 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 16 Oct 2026 22:31:35 -0700 Subject: [PATCH 5/9] docs: retain qualified CUDA deadline deployment --- docs/plans/cuda-training-deadline/plan.md | 7 +++++- .../reports/live-deployment.json | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 docs/plans/cuda-training-deadline/reports/live-deployment.json diff --git a/docs/plans/cuda-training-deadline/plan.md b/docs/plans/cuda-training-deadline/plan.md index f47f5498..bf58dd23 100644 --- a/docs/plans/cuda-training-deadline/plan.md +++ b/docs/plans/cuda-training-deadline/plan.md @@ -1,6 +1,6 @@ # Finite CUDA training deadline -Status: Active. User authorized Tuldok improvements needed for the ongoing +Status: Accepted. Terminal; no further implementation slice. User authorized Tuldok improvements needed for the ongoing CUDA-training/CPU-inference sequence. One bounded slice extends only explicitly managed CUDA execution from600 to1800seconds. Ordinary CPU remains600seconds; captured CPU large-output profile remains2400seconds. One CUDA slot, memory @@ -16,3 +16,8 @@ spec dialect, runtime installation or model arithmetic. Verify rejection above the CUDA limit and unchanged CPU restriction, plus lifecycle cancellation/drain. Deploy only after isolated verification and no live active Runs. Do not replay any unknown mutation or overwrite completed migration gates. + +Qualified23 isolated tests and both independent reviews. Deployed exact1688f7d +as current-05 after confirming zero active Runs;15.765s switch and all nine +existing Run states preserved. Original catalog and owner session retained. +Scientific fits continue in the lab plan, not this completed infrastructure plan. diff --git a/docs/plans/cuda-training-deadline/reports/live-deployment.json b/docs/plans/cuda-training-deadline/reports/live-deployment.json new file mode 100644 index 00000000..d11b7553 --- /dev/null +++ b/docs/plans/cuda-training-deadline/reports/live-deployment.json @@ -0,0 +1,23 @@ +{ + "active_runs": 0, + "automatic_retry": false, + "capacity_environment": { + "TULDOK_CPU_RUN_MEMORY_MIB": "4096", + "TULDOK_CPU_RUN_SLOTS": "6", + "TULDOK_CUDA_RUN_MEMORY_MIB": "8192", + "TULDOK_RUN_MEMORY_RESERVE_MIB": "8192" + }, + "catalog_id": "a7da4a8de53541a2a0e2aad08005bb71", + "deployment": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok-deployed/current-05", + "elapsed_s": 15.764875220003887, + "existing_run_states_preserved": true, + "existing_runs": 9, + "format": "pressure-scheduler-live-deployment-v1", + "manifest_sha256": "3fb00e03d284ee043978f6ef5c36923dc13dedf3e4c21a82709ab09dfc46e033", + "old_pid": 604211, + "old_start_ticks": "6862841", + "pid": 619646, + "source_revision": "1688f7d9fc6f941d8a23d8b077df6af4213f683a", + "start_ticks": "7043224", + "status": "passed" +} From 709d0ddf49f8af207116d3b7abb8486f26d28fe3 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 17 Oct 2026 07:12:51 -0700 Subject: [PATCH 6/9] fix: tolerate output renames during live custody scans --- docs/plans/output-inventory-rename/plan.md | 26 ++++++++++++ .../reports/tests.json | 42 +++++++++++++++++++ .../reports/verification.json | 28 +++++++++++++ recipe_process.py | 9 +++- tests/test_recipe_process.py | 23 ++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 docs/plans/output-inventory-rename/plan.md create mode 100644 docs/plans/output-inventory-rename/reports/tests.json create mode 100644 docs/plans/output-inventory-rename/reports/verification.json diff --git a/docs/plans/output-inventory-rename/plan.md b/docs/plans/output-inventory-rename/plan.md new file mode 100644 index 00000000..37a83c89 --- /dev/null +++ b/docs/plans/output-inventory-rename/plan.md @@ -0,0 +1,26 @@ +# Output inventory rename race + +Status: Active. Owner: root. User authority: Tuldok improvements needed by the +ongoing CUDA/CPU model sequence. Separate infrastructure branch, no model edits. + +Run786bdd62edb845fe8950c4a952bbcc78 reached5000updates then drained with reason +output_limit. Retained output files total under32MiB. The pressure evidence writer +renames profile.json to training-profile.json. A deterministic real rename between +live directory listing and lstat reproduces FileNotFoundError, which the helper +classifies as output_limit. Stderr empty, leader SIGTERM; no successful published +checkpoint is reused. This identifies a reproducible cause compatible with the +failure; the old receipt did not retain the exact observation exception. + +Fix only the live monitor: ignore FileNotFoundError for a listed child that has +vanished, allowing next scan and existing final publication validation to check +surviving files. Keep other errors, regular-file/count/size/total bounds, wall +limits and final custody validation unchanged. This is observational sampling, +not an instantaneous inventory or OS memory sandbox. No silent retry/replay. + +Verify deterministic rename, subsequent oversize rejection, other-error rejection, +existing exact output bounds and Linux custody lifecycle tests. Deploy only with +no active catalog Runs, preserving all prior states and exact owner/catalog. +Close the failed Study and admit one fresh late-arm fit from initialization; +retain original successful early fit and fresh CPUrestore, read-only recovery. +Standards MCP snapshot:v1:f2c192e8-4ff4-41a9-b019-8e6b050c27a7 applies; independent +Standards and Spec reviews required before completion. No push or merge. diff --git a/docs/plans/output-inventory-rename/reports/tests.json b/docs/plans/output-inventory-rename/reports/tests.json new file mode 100644 index 00000000..748d0706 --- /dev/null +++ b/docs/plans/output-inventory-rename/reports/tests.json @@ -0,0 +1,42 @@ +{ + "umask": "077", + "suites": [ + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python3.12", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_recipe_process.py", + "-v" + ], + "cwd": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "elapsed_s": 15.626875494999695, + "returncode": 0, + "output": "test_cancel_during_delayed_go_verification_drains_before_fork (test_recipe_process.CpuSnapshotContractTests.test_cancel_during_delayed_go_verification_drains_before_fork) ... ok\ntest_canonical_roundtrip_without_live_packages (test_recipe_process.CpuSnapshotContractTests.test_canonical_roundtrip_without_live_packages) ... ok\ntest_cpu_capture_and_input_bounds_are_profile_specific (test_recipe_process.CpuSnapshotContractTests.test_cpu_capture_and_input_bounds_are_profile_specific) ... ok\ntest_cpu_exact_aggregate_boundaries_and_per_input_limit (test_recipe_process.CpuSnapshotContractTests.test_cpu_exact_aggregate_boundaries_and_per_input_limit) ... ok\ntest_cpu_launch_environment_is_fresh_unicode_capable_and_locale_closed (test_recipe_process.CpuSnapshotContractTests.test_cpu_launch_environment_is_fresh_unicode_capable_and_locale_closed) ... ok\ntest_cpu_launch_vector_requires_bound_descriptor (test_recipe_process.CpuSnapshotContractTests.test_cpu_launch_vector_requires_bound_descriptor) ... ok\ntest_directory_and_preparation_budget_are_bounded (test_recipe_process.CpuSnapshotContractTests.test_directory_and_preparation_budget_are_bounded) ... ok\ntest_large_output_scan_has_exact_file_total_and_count_bounds_without_large_writes (test_recipe_process.CpuSnapshotContractTests.test_large_output_scan_has_exact_file_total_and_count_bounds_without_large_writes) ... ok\ntest_large_resource_profile_requires_full_cpu_identity_and_keeps_input_bounds (test_recipe_process.CpuSnapshotContractTests.test_large_resource_profile_requires_full_cpu_identity_and_keeps_input_bounds) ... ok\ntest_manifest_change_and_kind_mismatch_refused (test_recipe_process.CpuSnapshotContractTests.test_manifest_change_and_kind_mismatch_refused) ... ok\ntest_mutated_bytes_mode_missing_and_unlisted_refused (test_recipe_process.CpuSnapshotContractTests.test_mutated_bytes_mode_missing_and_unlisted_refused) ... ok\ntest_output_scan_does_not_ignore_other_observation_errors (test_recipe_process.CpuSnapshotContractTests.test_output_scan_does_not_ignore_other_observation_errors) ... ok\ntest_output_scan_tolerates_a_regular_file_renamed_after_listing (test_recipe_process.CpuSnapshotContractTests.test_output_scan_tolerates_a_regular_file_renamed_after_listing) ... ok\ntest_pure_bounds_paths_modes_and_descriptor_binding (test_recipe_process.CpuSnapshotContractTests.test_pure_bounds_paths_modes_and_descriptor_binding) ... ok\ntest_symlink_and_hardlink_file_custody_refused (test_recipe_process.CpuSnapshotContractTests.test_symlink_and_hardlink_file_custody_refused) ... ok\ntest_virtual_torch_origins_are_exact_and_captured (test_recipe_process.CpuSnapshotContractTests.test_virtual_torch_origins_are_exact_and_captured) ... ok\ntest_artifact_source_cold_identity_and_completed_custody (test_recipe_process.CustodyTests.test_artifact_source_cold_identity_and_completed_custody) ... ok\ntest_barrier_complete_exact_output_and_cold_receipt (test_recipe_process.CustodyTests.test_barrier_complete_exact_output_and_cold_receipt) ... ok\ntest_cancel_before_go_and_controller_loss (test_recipe_process.CustodyTests.test_cancel_before_go_and_controller_loss) ... ok\ntest_closed_controller_channel_reports_lost_once_and_cancel_is_idempotent (test_recipe_process.CustodyTests.test_closed_controller_channel_reports_lost_once_and_cancel_is_idempotent) ... ok\ntest_controller_eof_after_go_drains_escaped_child (test_recipe_process.CustodyTests.test_controller_eof_after_go_drains_escaped_child) ... ok\ntest_helper_death_preserves_unknown_even_when_leader_is_dead (test_recipe_process.CustodyTests.test_helper_death_preserves_unknown_even_when_leader_is_dead) ... ok\ntest_malformed_control_and_receipt_loss_never_claim_completion (test_recipe_process.CustodyTests.test_malformed_control_and_receipt_loss_never_claim_completion) ... ok\ntest_native_policy_absence_is_closed_and_receipt_is_pure (test_recipe_process.CustodyTests.test_native_policy_absence_is_closed_and_receipt_is_pure) ... ok\ntest_output_and_runtime_drift_bounds (test_recipe_process.CustodyTests.test_output_and_runtime_drift_bounds) ... ok\ntest_setsid_double_fork_and_term_resistance_are_drained (test_recipe_process.CustodyTests.test_setsid_double_fork_and_term_resistance_are_drained) ... ok\ntest_source_drift_after_ready_does_not_execute (test_recipe_process.CustodyTests.test_source_drift_after_ready_does_not_execute) ... ok\ntest_spec_rejects_invalid_bounds_and_executable_hash_before_spawn (test_recipe_process.CustodyTests.test_spec_rejects_invalid_bounds_and_executable_hash_before_spawn) ... ok\ntest_wall_and_stream_limits_and_nonzero_exit (test_recipe_process.CustodyTests.test_wall_and_stream_limits_and_nonzero_exit) ... ok\n\n----------------------------------------------------------------------\nRan 29 tests in 15.356s\n\nOK\n" + }, + { + "command": [ + "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/app-env/bin/python3.12", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_torch_runtime_lifecycle.py", + "-v" + ], + "cwd": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok", + "elapsed_s": 2.4478574179956922, + "returncode": 0, + "output": "test_cancel_during_inventory_drains_without_fork_and_retains_cold_identity (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_cancel_during_inventory_drains_without_fork_and_retains_cold_identity) ... ok\ntest_captured_bootstrap_drift_after_ready_refuses_recipe_entry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_captured_bootstrap_drift_after_ready_refuses_recipe_entry) ... ok\ntest_controller_loss_during_inventory_drains_without_fork (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_controller_loss_during_inventory_drains_without_fork) ... ok\ntest_exact_managed_spec_requires_one_profile_device_and_captured_identity (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_exact_managed_spec_requires_one_profile_device_and_captured_identity) ... ok\ntest_finite_cuda_deadline_keeps_cpu_deadline_unchanged (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_finite_cuda_deadline_keeps_cpu_deadline_unchanged) ... ok\ntest_inventory_deadline_fails_and_drains_before_child_wall_clock (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_inventory_deadline_fails_and_drains_before_child_wall_clock) ... ok\ntest_managed_context_drift_after_ready_drains_without_recipe_entry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_drift_after_ready_drains_without_recipe_entry) ... ok\ntest_managed_context_is_reverified_after_child_exit_before_completion (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_is_reverified_after_child_exit_before_completion) ... ok\ntest_managed_context_keeps_one_exact_bounded_captured_argument (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_managed_context_keeps_one_exact_bounded_captured_argument) ... ok\ntest_missing_or_changed_inventory_refused_after_go_without_child_or_cpu_retry (test_torch_runtime_lifecycle.ManagedInventoryCustodyTests.test_missing_or_changed_inventory_refused_after_go_without_child_or_cpu_retry) ... ok\n\n----------------------------------------------------------------------\nRan 10 tests in 2.237s\n\nOK\n" + } + ], + "passed": true +} diff --git a/docs/plans/output-inventory-rename/reports/verification.json b/docs/plans/output-inventory-rename/reports/verification.json new file mode 100644 index 00000000..6bd80468 --- /dev/null +++ b/docs/plans/output-inventory-rename/reports/verification.json @@ -0,0 +1,28 @@ +{ + "format": "output-inventory-rename-verification-v1", + "status": "passed", + "failed_run_id": "786bdd62edb845fe8950c4a952bbcc78", + "failed_custody_outcome": "failed", + "failed_custody_reason": "output_limit", + "leader_returncode": -15, + "stderr_bytes": 0, + "historical_exact_observation_exception_retained": false, + "deterministic_real_rename_repro": "FileNotFoundError at listed child lstat before fix; passes after fix", + "unchanged_bounds_and_other_errors_tested": true, + "test_count": 39, + "secure_fixture_umask": "077", + "initial_full_suite_attempt": { + "tests": 29, + "errors": 3, + "cause": "default writable group directory permissions rejected by existing exact CPU snapshot contract; repeated under secure077", + "model_or_deployment_failure": false + }, + "reviews": { + "spec": "passed; no material findings", + "standards": "passed; no material findings" + }, + "source_sha256": "247c573d6a9bfdc4e189b3a940650c91102cb40273751225f0fef6dd79999e6f", + "test_sha256": "f89c0e7985d4a9a44f7dca26d9a747cc1ee272719b6cf5577a14adac816f7e3c", + "live_deployed": false, + "unpublished_weights_reused": false +} diff --git a/recipe_process.py b/recipe_process.py index 8f082cf5..87a652c6 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -904,7 +904,14 @@ def _outputs(spec): if root.is_symlink() or not root.is_dir(): raise ProcessError('Output directory changed.') for path in root.iterdir(): - info=path.lstat();count+=1;total+=info.st_size + try: + info=path.lstat() + except FileNotFoundError: + # A recipe may atomically publish or rename a regular output while + # this live scan runs. The next scan and final publication validate + # the surviving inventory; other observation errors remain fatal. + continue + count+=1;total+=info.st_size if count>count_limit or not stat.S_ISREG(info.st_mode) or info.st_size>file_limit or total>spec['max_output_bytes']: raise ProcessError('Output inventory exceeded its admitted bound.') diff --git a/tests/test_recipe_process.py b/tests/test_recipe_process.py index ebfff082..81135555 100644 --- a/tests/test_recipe_process.py +++ b/tests/test_recipe_process.py @@ -376,6 +376,29 @@ def test_large_resource_profile_requires_full_cpu_identity_and_keeps_input_bound large['captured_files'].append(dict(path=str(self.base/'source/oversize'),bytes=64*1024*1024+1,sha256='e'*64)) with self.assertRaisesRegex(custody.ProcessError,'captured file size'):custody.validate_spec(large) + def test_output_scan_tolerates_a_regular_file_renamed_after_listing(self): + output=self.base/'rename-output';output.mkdir() + profile=output/'profile.json';profile.write_bytes(b'original profile') + spec={'output_dir':str(output),'max_output_bytes':32} + original=Path.lstat + def rename_then_stat(path): + if path==profile:profile.rename(output/'training-profile.json') + return original(path) + with patch.object(Path,'lstat',rename_then_stat):custody._outputs(spec) + custody._outputs(spec) + (output/'training-profile.json').write_bytes(b'x'*33) + with self.assertRaises(custody.ProcessError):custody._outputs(spec) + + def test_output_scan_does_not_ignore_other_observation_errors(self): + output=self.base/'unreadable-output';output.mkdir() + item=output/'result';item.write_bytes(b'x') + original=Path.lstat + def deny_file(path): + if path==item:raise PermissionError('fixture observation denied') + return original(path) + with patch.object(Path,'lstat',deny_file),self.assertRaises(PermissionError): + custody._outputs({'output_dir':str(output),'max_output_bytes':32}) + def test_large_output_scan_has_exact_file_total_and_count_bounds_without_large_writes(self): output=self.base/'large-output';output.mkdir() spec=dict(self.cpu_spec(),resource_profile=custody.LARGE_OUTPUT_PROFILE, From 225f8e862f6ee7a6f778522c6e15a5fbc97fe5ca Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 17 Oct 2026 07:14:52 -0700 Subject: [PATCH 7/9] docs: retain qualified output monitor deployment --- docs/plans/output-inventory-rename/plan.md | 15 ++++++++++-- .../reports/live-deployment.json | 24 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 docs/plans/output-inventory-rename/reports/live-deployment.json diff --git a/docs/plans/output-inventory-rename/plan.md b/docs/plans/output-inventory-rename/plan.md index 37a83c89..b605f112 100644 --- a/docs/plans/output-inventory-rename/plan.md +++ b/docs/plans/output-inventory-rename/plan.md @@ -1,6 +1,6 @@ # Output inventory rename race -Status: Active. Owner: root. User authority: Tuldok improvements needed by the +Status: Accepted. Terminal; no further implementation slice. Owner: root. User authority: Tuldok improvements needed by the ongoing CUDA/CPU model sequence. Separate infrastructure branch, no model edits. Run786bdd62edb845fe8950c4a952bbcc78 reached5000updates then drained with reason @@ -21,6 +21,17 @@ Verify deterministic rename, subsequent oversize rejection, other-error rejectio existing exact output bounds and Linux custody lifecycle tests. Deploy only with no active catalog Runs, preserving all prior states and exact owner/catalog. Close the failed Study and admit one fresh late-arm fit from initialization; -retain original successful early fit and fresh CPUrestore, read-only recovery. +retain original successful early fit and fresh CPUrestore, ordinary hash-verified collection. Standards MCP snapshot:v1:f2c192e8-4ff4-41a9-b019-8e6b050c27a7 applies; independent Standards and Spec reviews required before completion. No push or merge. + + +Completed39custody/runtime tests under secure077fixture permissions; both +independent reviews passed. Deployed exact709d0ddf49f8af207116d3b7abb8486f26d28fe3 +as current-06 with zero active Runs,27.721982285s switch, all20prior Run states +and owner/catalog/capacity settings preserved. EarlyCUDA+CPUrestore collected +normally before switch; no readback recovery or retraining was needed. Failed +lateStudy02 stop acknowledged; a client postcondition used the wrong response +level, then verified nested study.open=false from its durable acknowledgment +without mutation replay. Fresh lateStudy03 belongs to the lab experiment, not +this completed infrastructure change. No weights/datasets/credentials committed. diff --git a/docs/plans/output-inventory-rename/reports/live-deployment.json b/docs/plans/output-inventory-rename/reports/live-deployment.json new file mode 100644 index 00000000..921e7471 --- /dev/null +++ b/docs/plans/output-inventory-rename/reports/live-deployment.json @@ -0,0 +1,24 @@ +{ + "format": "tuldok-output-monitor-deployment-v1", + "status": "passed", + "source_revision": "709d0ddf49f8af207116d3b7abb8486f26d28fe3", + "old_pid": 619646, + "old_start_ticks": "7043224", + "catalog_id": "a7da4a8de53541a2a0e2aad08005bb71", + "existing_runs": 20, + "active_runs": 0, + "deployment": "/media/jeremy/OrangeCream/Linux Software/repos/owned/graphics-3d/rheon-surrogate-lab/runs/dependency-migration-v1/Tuldok-deployed/current-06", + "manifest_sha256": "3c9aa3daa616649e986457adf9ef7902d68c890ffb0e3833d33b0183b093d1f4", + "automatic_retry": false, + "pid": 855603, + "start_ticks": "10183125", + "elapsed_s": 27.72198228498746, + "existing_run_states_preserved": true, + "original_owner_session_preserved": true, + "capacity_environment": { + "TULDOK_CPU_RUN_SLOTS": "6", + "TULDOK_CPU_RUN_MEMORY_MIB": "4096", + "TULDOK_CUDA_RUN_MEMORY_MIB": "8192", + "TULDOK_RUN_MEMORY_RESERVE_MIB": "8192" + } +} From eed2b84d21050943d211defffc3133609422950a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 18 Oct 2026 05:14:46 -0700 Subject: [PATCH 8/9] perf(runs): avoid terminal history scans during coordination Select scheduling candidates by saved state, fully validate every selected Run, and wait for committed admission when idle. Preserve dispatch, startup, artifact integrity and resource retry behavior. --- .../request-latency-v1/execution-ledger.md | 70 ++++++++ docs/plans/request-latency-v1/issues.md | 27 ++++ docs/plans/request-latency-v1/plan.md | 117 ++++++++++++++ .../reports/focused-tests.json | 14 ++ .../reports/http-baseline-01.json | 30 ++++ .../reports/http-profile.json | 12 ++ .../reports/preflight-before.json | 9 ++ .../reports/queue-scan-after.json | 7 + .../reports/queue-scan-before.json | 7 + .../reads-before-coordinator-exited.json | 62 ++++++++ .../reports/regression-suite.json | 19 +++ .../request-latency-v1/reports/waits-01.json | 60 +++++++ .../reports/waits-coordinator-exited.json | 60 +++++++ runs.py | 25 ++- tests/test_run_scheduling_reads.py | 150 ++++++++++++++++++ 15 files changed, 663 insertions(+), 6 deletions(-) create mode 100644 docs/plans/request-latency-v1/execution-ledger.md create mode 100644 docs/plans/request-latency-v1/issues.md create mode 100644 docs/plans/request-latency-v1/plan.md create mode 100644 docs/plans/request-latency-v1/reports/focused-tests.json create mode 100644 docs/plans/request-latency-v1/reports/http-baseline-01.json create mode 100644 docs/plans/request-latency-v1/reports/http-profile.json create mode 100644 docs/plans/request-latency-v1/reports/preflight-before.json create mode 100644 docs/plans/request-latency-v1/reports/queue-scan-after.json create mode 100644 docs/plans/request-latency-v1/reports/queue-scan-before.json create mode 100644 docs/plans/request-latency-v1/reports/reads-before-coordinator-exited.json create mode 100644 docs/plans/request-latency-v1/reports/regression-suite.json create mode 100644 docs/plans/request-latency-v1/reports/waits-01.json create mode 100644 docs/plans/request-latency-v1/reports/waits-coordinator-exited.json create mode 100644 tests/test_run_scheduling_reads.py diff --git a/docs/plans/request-latency-v1/execution-ledger.md b/docs/plans/request-latency-v1/execution-ledger.md new file mode 100644 index 00000000..2c043a1c --- /dev/null +++ b/docs/plans/request-latency-v1/execution-ledger.md @@ -0,0 +1,70 @@ +# Execution ledger + +2026-10-18: user paused model improvement to address Tuldok latency. Fetched GitHub +origin; main matches 300f12d and is already an ancestor of deployed integration. +Isolated a branch from the documentation successor of the actual deployed code. +Preserved other branches/worktrees and unrelated surrogate-lab changes. + +The old saved owner session had expired. Renewed the existing local owner through +Access._local_owner_token(require_existing=True) and exchange, using the established +operational procedure. No new actor, Project or Run; two audited credentials only. +Session remains in a private file and is excluded from receipts/commits. + +Read-only HTTP baseline command: app-env Python runs/tuldok-performance-v1/http_probe.py +--output runs/tuldok-performance-v1/http-baseline-01.json. It reproduces 7.35–8.27 s +Run reads and 7.67–7.89 s reads of a 189-byte artifact. Command exits 1 with +latency_reproduced, not an incidental authentication error. The first unauthenticated +attempt failed before any workload and is not counted as performance evidence. +A direct current read-owner profile is running without a second supervisor. + +Cause selection: the coordinator's while-loop revalidates every Run at every +empty-queue poll, under the catalog lock. The queue-capacity checks do the same. +A profiled unchanged scan loaded 55 historical Runs (zero queued), making 174M +calls and using 22.717 CPU / 30.277 wall seconds. Raw versus typed HTTP showed +client protocol overhead <0.03 s; both waited ~7.5 s. PostgreSQL was idle waiting +for Python while the app consumed nearly one full CPU core. Sampling attachment +was unavailable under local ptrace permissions and passworded sudo; direct +current-owner profiling is retained instead. + +Regression command against the unchanged product: python3 +runs/tuldok-performance-v1/run_tests.py test_run_scheduling_reads. Both tests fail +at the intended assertions: empty coordination polls at .25 seconds rather than +waiting for an event, and idle coordination calls _load on terminal history. +Tests use a separate private PostgreSQL service under /tmp/tuldok-perf-pg-20261018, +not the live user database. Implementation bound to Runs scheduling/capacity and +its targeted tests; full dispatch/read/integrity/startup checks remain unchanged. + + +Repair: Runs._in_states selects saved scheduling states in PostgreSQL, then calls +unchanged _load on every selected Run under the existing catalog lock. Queue +coordination/close select queued; built-in and external capacity select ACTIVE. +Idle coordination waits for the existing on-commit/stop event. Queued headroom +shortages continue timed retries; active attempt polling remains unchanged. +Startup recovery and every artifact/explicit read retain their full checks. +No persistent-state schema, API contract, new cache or capacity policy changed. + +After repair, the same empty-catalog lookup has 756 profiled calls / 0.00157 CPU +seconds rather than 174M / 22.717 CPU seconds. This is a mechanism observation; +end-to-end HTTP must still be measured after an actual completed attempt warms +the coordinator. Final focused 7/7 pass (8.070 s), including exact corrupt-record +rejection and actual admission wake. The broader suite has 39 passing checks and +one legacy pinned CPU runtime unavailable (40 checks, 122.024 s). That same old +image-trainer test fails identically on the unmodified 225f8e8 base; it is not a +new scheduling failure and is not counted as a pass. Existing Pumas pressure CPU +restore will qualify the actual workflow separately. + +Fresh preflight of the existing C67 restore graph completed in 15.104 s with no +admission/training. Historical 102.654 s remains retained but is not used as the +fresh paired baseline. Existing session/catalog and graph material remain exact. + + +Baseline validity correction: the live old coordinator subsequently exited with +psycopg.errors.LockNotAvailable at its catalog-lock acquisition. The direct full +history diagnostic held that same lock for longer than its 15-second timeout; +this diagnostic may have triggered the exit. A later read benchmark measured +0.46–0.55 s and zero idle app CPU because the coordinator had exited, not because +old source improved. Keep that receipt explicitly classified as coordinator-exited. +The earlier live 7–8 s / one-core measurements remain valid observations of the +running coordinator. Do not pair the later fast receipt as a healthy baseline. +After activation, require an actual fresh completed attempt and post-completion +read measurements. No claim of an exact paired speedup or full-pipeline speedup. diff --git a/docs/plans/request-latency-v1/issues.md b/docs/plans/request-latency-v1/issues.md new file mode 100644 index 00000000..8e4da554 --- /dev/null +++ b/docs/plans/request-latency-v1/issues.md @@ -0,0 +1,27 @@ +# Findings + +PERF-1 (high, owner Runs, fix-now): constant full-history validation under the +catalog lock in _loop and queue-capacity checks. Real minimal HTTP read plus +unchanged queue-scan profile and targeted regressions identify it. Selected-state +routing and idle event waiting are implemented; live qualification remains. + +ENV-1 (qualification limitation, owner legacy image adapter): the pinned CPU +training runtime is absent from this application environment. One existing +RealCPURunTests check fails identically on the unchanged base. Record unavailable, +not passed; use actual Pumas CPU restore plus existing neutral real-custodian tests +for this pressure-workflow qualification. Revisit if supporting that old adapter's +numerical training environment becomes part of the requested work. + +Public list/input-candidate and startup recovery still intentionally perform full +selected disclosure/recovery integrity. No performance change is claimed there. +Revisit only if representative after-fix measurements still identify them as a +limiting user workflow. No skipped artifact, authorization or dispatch checks. + + +RES-1 (existing resilience limitation, owner Runs, deferred separate repair): a +catalog advisory-lock timeout can terminate _loop; _start does not recreate an +already assigned dead thread. Observed after the long read-only queue profile. +The latency repair removes that repeated long history scan but does not claim to +repair arbitrary coordinator exceptions. Do not treat an exited coordinator as +healthy idle performance. Scope a separate bounded recovery/liveness change with +failure tests rather than hiding arbitrary errors or dispatch retries here. diff --git a/docs/plans/request-latency-v1/plan.md b/docs/plans/request-latency-v1/plan.md new file mode 100644 index 00000000..bff0b2ec --- /dev/null +++ b/docs/plans/request-latency-v1/plan.md @@ -0,0 +1,117 @@ +# Tuldok request latency + +Status: Active — implementation. Canonical operation: start this performance plan. +Jeremy paused the next model experiment and authorized improving Tuldok's speed. +Objective: remove the measured idle-looking waits from ordinary local Run reads, +artifact readback and the affected preparation/admission path without weakening +current authorization, immutable material, source integrity or custody checks. +No model/training changes. The surrogate lab retains workload receipts separately. + +Live baseline: deployed source 709d0ddf49f8af207116d3b7abb8486f26d28fe3, +known completed C67 Run and 189-byte published training-profile.json. Two typed +Run reads take 7.351/8.271 s; two artifact reads take 7.889/7.674 s. The same +workflow previously spent 72.386 s training but 1,223.625 s in managed execution. +The benchmark's two-second trigger reproduces visible pauses, not a product SLA. +Performance acceptance is a representative before/after comparison on the same +catalog/workload, with variability and CPU/resource costs reported; do not infer +end-to-end training speed from a helper microbenchmark. + +Discovery hypotheses: repeated request initialization; catalog/DB lock waiting; +repeated unrelated or transitive material validation; client transport delay. +Measure before selecting a change. A direct authenticated domain read can be +profiled while asserting it did not acquire catalog runtime supervision. Never +start a second supervisor on the live catalog. No admission retries or new +training merely to obtain profiling data. Credential renewal uses the established +existing-owner administration path and private 0600 storage, never SQL inserts +or an actor bootstrap. + +Discovery write set: this plan directory, read-only benchmark/profiling tooling +under surrogate-lab runs/tuldok-performance-v1, and its retained lab reports. +Implementation write set: runs.py, tests/test_run_scheduling_reads.py and this +plan directory. The measured owner is Runs coordination/capacity selection. +Keep full validation in _load, all artifact accessors, dispatch and startup recovery. +Select only queued Runs for coordination/close and ACTIVE Runs for queue-capacity +checks; fully load every selected record under the existing catalog lock. Wait +for the existing on-commit wake when there are neither queued nor active attempts; +retain timed resource retries for blocked queued work and active-attempt polling. +Replan if the measured cause changes supported state, persistence, API, security +or runtime supervision semantics. No blanket caching, skipped checks or concurrency +expansion is authorized by a latency observation alone. + +Verification: preserve typed output and all existing integrity/authorization/ +revocation/changed-file checks; add a regression at the actual repeated-work seam; +qualify representative latency before/after, and exercise affected concurrent and +failure paths. Distinguish passive Run inspection, artifact whole/segment reads, +material preparation and execution admission. Each gets only its measured claim. +Before activating a candidate, finish isolated checks, verify no active local +work, and preserve the live catalog, configured roots and runtime selection. +Record the concrete deployment procedure and evidence before service transition. + +Branches: performance/catalog-request-latency-20261018 in Tuldok, private proposal +based on 225f8e862f6ee7a6f778522c6e15a5fbc97fe5ca (numerical/product code matches +live 709d0dd; successor changes are documentation only). GitHub origin/main was +fetched and is 300f12d, an ancestor of this deployed integration; do not drop the +existing CUDA/concurrency work to patch main. Lab receipts branch: +performance/tuldok-request-latency-20261018, based on 3ae583618605c36c26b5bf710cbb40172335b67f. +Jeremy owns integration. No automatic push, merge, history rewrite or schema +migration. Retain branches for review; retire after integration or explicit +disposition. Preserve all unrelated dirty work and other worktrees. + +One task-created Tuldok worktree lives at surrogate-lab/runs/tuldok-performance-v1/Tuldok. +Its purpose is isolated implementation/qualification. Keep its exact head reachable +from the named branch; when its purpose ends, remove it only if clean and after +recording the protected head and branch disposition. Never prune other worktrees. +Coding-standards snapshot: snapshot:v1:ffcfc3f8-b52a-434e-b57f-d259a0355711. + +Current acceptance: baseline/cause and repair regressions satisfied; live activation +and representative after-update workload qualification pending. +Exactly one next slice: activate the qualified source with no active Runs, qualify +one fresh zero-update CPU restore to warm the coordinator, and measure identical +known-Run reads and preflight before/after. Ledger: execution-ledger.md; +findings: issues.md; evidence: reports/. + + +Bounded caller inventory: _loop, _admit_external_prepared, built-in _submit and +close share the queue/capacity selection invariant and change together. _recover +keeps validating the complete retained history on startup; list/input_candidates +retain their existing disclosure/integrity behavior. These are distinct surfaces, +not caches of scheduling state. Their separate latency is deferred to measurement +if still significant after removing the coordinator contention. No projection is +used to authorize artifact disclosure or execution. JSON state selection is only +candidate routing; _load and the final dispatch transaction still revalidate it. +No API/schema/persisted state or concurrency-capacity policy changes. + +Profile evidence: one unchanged scan of 55 terminal Runs, zero queued, took 30.277 s +wall / 22.717 s process CPU under cProfile and made 174 million calls. A live +three-second sample showed 2.95 s app CPU while PostgreSQL waited for its client. +Raw and typed HTTP both wait ~7.5–7.8 s; client CPU is <0.03 s, excluding transport +as the main cause. Direct selected artifact ownership is 1.475 s under profiling. +Live stack attachment was unavailable without passworded sudo; no process injection +or service restart was used as a profiling workaround. The real unchanged queue +expression supplied the causal mechanism profile. No claim uses profiler-inflated +durations as an unprofiled before/after latency comparison. + + +Activation boundary: prepare a new immutable current-07 source copy from the +qualified commit and the existing 193-file deployment inventory; verify old +manifest bytes and only intended product changes. Confirm selected PID/start +identity and no queued/running/cancelling/unknown catalog Runs; retain every old +Run document hash/state. Preserve the exact catalog, project roots, environment, +capacity settings, interpreter and owner session. Gracefully SIGINT the selected +old app and prove cessation before launching one candidate; no admission retry. +Verify HTTP readiness, owner/catalog identity and unchanged old Run documents. +No schema migration or new actor. If activation cannot prove cessation or health, +retain explicit failure and stop that transition; never launch a competing owner. +The existing original source/env remain available for a separately verified +restoration if needed. + +After startup, one bounded fresh C67 CPU restore (zero optimizer updates) through +the existing graph provides the real admission/custody/artifact proof and starts +the coordinator. Journal each mutation once. Compare completed bytes to retained +C67 restore weights/normalization/predictions, verify completed/drained custody, +then remeasure reads after completion. Startup-only idle behavior is insufficient: +the fixed coordinator must be quiet after an actual attempt has finished. + +Composed-design applicability: existing Runs remains the single coordination, +capacity and custody owner. The new private selector serves four existing callers; +no new production architectural owner or duplicate durable state is introduced. diff --git a/docs/plans/request-latency-v1/reports/focused-tests.json b/docs/plans/request-latency-v1/reports/focused-tests.json new file mode 100644 index 00000000..dc65c484 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/focused-tests.json @@ -0,0 +1,14 @@ +{ + "status": "passed", + "tests": 7, + "elapsed_s": 8.07, + "claims": [ + "terminal history is not loaded for idle coordination", + "empty coordinator waits for an event and joins on shutdown", + "fresh committed admission wakes idle coordination", + "queued resource shortage retries without another admission", + "builtin and external capacity checks skip terminal history", + "selected corrupt queued envelopes remain unavailable" + ], + "source_sha256": "db18e4c6bae6d2e596868bc953acbb9c45c9c0f3db1dd4492d7a511bae23611a" +} diff --git a/docs/plans/request-latency-v1/reports/http-baseline-01.json b/docs/plans/request-latency-v1/reports/http-baseline-01.json new file mode 100644 index 00000000..e5734217 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/http-baseline-01.json @@ -0,0 +1,30 @@ +{ + "status": "latency_reproduced", + "scope": "Two current typed reads of a known completed Run and its 189-byte artifact; no admission or training.", + "request_target_s": 2, + "artifact_bytes": 189, + "run_id": "e17624f982b54e858698207227010410", + "rows": [ + { + "operation": "owner_initialization", + "elapsed_s": 3.1915215759945568 + }, + { + "operation": "runs.get", + "elapsed_s": 7.351186351996148 + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 7.889473076007562 + }, + { + "operation": "runs.get", + "elapsed_s": 8.270514289004495 + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 7.674140114017064 + } + ], + "source_sha256": "8c6b1cedca245fc073b2ad806836632a4f01073b54f48869605e7d5d39f405c0" +} diff --git a/docs/plans/request-latency-v1/reports/http-profile.json b/docs/plans/request-latency-v1/reports/http-profile.json new file mode 100644 index 00000000..1969e686 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/http-profile.json @@ -0,0 +1,12 @@ +[ + { + "mode": "raw", + "wall_s": 7.514717735000886, + "client_cpu_s": 0.0014333380000000062 + }, + { + "mode": "typed", + "wall_s": 7.785370839003008, + "client_cpu_s": 0.021335930999999864 + } +] diff --git a/docs/plans/request-latency-v1/reports/preflight-before.json b/docs/plans/request-latency-v1/reports/preflight-before.json new file mode 100644 index 00000000..e6f0b62a --- /dev/null +++ b/docs/plans/request-latency-v1/reports/preflight-before.json @@ -0,0 +1,9 @@ +{ + "status": "completed", + "operation": "graphs.preflight_external", + "elapsed_s": 15.104385499987984, + "graph_id": "209124c1cc424fbbb41afab9314f1416", + "revision": 1, + "material_sha256": "d06e6ee426c0e0e661010a8107b3069de01ee4522b3e875cfd909236dcb07a68", + "training_started": false +} diff --git a/docs/plans/request-latency-v1/reports/queue-scan-after.json b/docs/plans/request-latency-v1/reports/queue-scan-after.json new file mode 100644 index 00000000..ad0f59e9 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/queue-scan-after.json @@ -0,0 +1,7 @@ +{ + "run_count": 0, + "queued": 0, + "wall_s": 0.3547653589921538, + "cpu_s": 0.0015716580000000313, + "scope": "Candidate selected-state queue lookup, once; no executor or dispatch, no active Runs." +} diff --git a/docs/plans/request-latency-v1/reports/queue-scan-before.json b/docs/plans/request-latency-v1/reports/queue-scan-before.json new file mode 100644 index 00000000..328bafda --- /dev/null +++ b/docs/plans/request-latency-v1/reports/queue-scan-before.json @@ -0,0 +1,7 @@ +{ + "run_count": 55, + "queued": 0, + "wall_s": 30.27675883800839, + "cpu_s": 22.717225617, + "scope": "The unchanged coordinator queue-selection expression, once; no executor or dispatch, no active Runs." +} diff --git a/docs/plans/request-latency-v1/reports/reads-before-coordinator-exited.json b/docs/plans/request-latency-v1/reports/reads-before-coordinator-exited.json new file mode 100644 index 00000000..f98adbd1 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/reads-before-coordinator-exited.json @@ -0,0 +1,62 @@ +{ + "status": "completed", + "kind": "reads", + "owner_initialization_s": 1.0151079000206664, + "rows": [ + { + "operation": "runs.get", + "elapsed_s": 0.5049598229816183, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.4643416860199068, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + }, + { + "operation": "runs.get", + "elapsed_s": 0.5495082029956393, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.5029060239903629, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + }, + { + "operation": "runs.get", + "elapsed_s": 0.5545839669939596, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.5126400589942932, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + } + ], + "summary": { + "runs.artifact_chunk": { + "samples": 3, + "minimum_s": 0.4643416860199068, + "median_s": 0.5029060239903629, + "maximum_s": 0.5126400589942932 + }, + "runs.get": { + "samples": 3, + "minimum_s": 0.5049598229816183, + "median_s": 0.5495082029956393, + "maximum_s": 0.5545839669939596 + } + }, + "run_id": "e17624f982b54e858698207227010410", + "training_started": false, + "scope": "Same client, catalog and immutable workload. Read-only current-authority checks; timings are observations, not a universal product SLA.", + "config_sha256": "a2e54720b2f5e2c1457926b24cf94253fdedd913eb7ffd74b5ccef92f01eae2e", + "source_sha256": "a49b274cd87215512c72b88de4d97e4fd9af391a496974275021dcc53d026124" +} diff --git a/docs/plans/request-latency-v1/reports/regression-suite.json b/docs/plans/request-latency-v1/reports/regression-suite.json new file mode 100644 index 00000000..8adbf4d0 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/regression-suite.json @@ -0,0 +1,19 @@ +{ + "status": "39_passed_1_environment_unavailable", + "tests_run": 40, + "elapsed_s": 122.024, + "modules": [ + "test_run_scheduling_reads", + "test_run_concurrency", + "test_runs", + "test_external_runs", + "test_study_admission" + ], + "passing_tests": 39, + "unavailable_test": "test_runs.RealCPURunTests.test_real_cpu_results_holdout_exposure_correction_restart_and_tamper", + "reason": "Pinned legacy CPU trainer is not installed in the application environment; validation returns code unavailable.", + "unmodified_base_same_failure": true, + "baseline_log_sha256": "d169fc48669e7cb0f96f29ff8f3d5e4f52a2fe6f6f242fa686471de7112e68ee", + "database": "Isolated PostgreSQL 16 service; live catalog not used for tests.", + "command": "python3 runs/tuldok-performance-v1/run_tests.py test_run_scheduling_reads test_run_concurrency test_runs test_external_runs test_study_admission" +} diff --git a/docs/plans/request-latency-v1/reports/waits-01.json b/docs/plans/request-latency-v1/reports/waits-01.json new file mode 100644 index 00000000..54d3d088 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/waits-01.json @@ -0,0 +1,60 @@ +{ + "wall_s": 3.0123237500083633, + "app_cpu_s": 2.9500000000116415, + "samples": [ + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM workspace_recipe_materials WHERE sha256=$1" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ] + ] +} diff --git a/docs/plans/request-latency-v1/reports/waits-coordinator-exited.json b/docs/plans/request-latency-v1/reports/waits-coordinator-exited.json new file mode 100644 index 00000000..84ba919b --- /dev/null +++ b/docs/plans/request-latency-v1/reports/waits-coordinator-exited.json @@ -0,0 +1,60 @@ +{ + "wall_s": 3.011039430013625, + "app_cpu_s": 0.0, + "samples": [ + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ], + [ + { + "pid": 855604, + "state": "idle", + "wait_type": "Client", + "wait_event": "ClientRead", + "sql": "SELECT document FROM model_materials WHERE id=$1 AND revision=$2 AND sha256=$3" + } + ] + ] +} diff --git a/runs.py b/runs.py index a54ebc74..9fbf8595 100644 --- a/runs.py +++ b/runs.py @@ -228,6 +228,18 @@ def _load(self, run_id): except (ValueError, KeyError, TypeError): raise WorkbenchError('Stored run envelope is invalid; refusing to guess execution state.', 'unavailable', 409) from None + def _in_states(self, states): + """Route queue candidates by saved state; fully validate every selection. + + Terminal history is still validated by explicit reads and startup recovery. + Coordination must not reopen its immutable source closures on every poll. + """ + with self.lock: + rows = self.db.execute( + "SELECT id FROM workspace_runs WHERE document::jsonb->>'state' = ANY(%s)", + (list(states),)).fetchall() + return [self._load(row[0]) for row in rows] + def _save(self, run): self.db.execute('UPDATE workspace_runs SET document=%s WHERE id=%s', (encode(run), run['id'])) action = 'runs.reconcile' if self.access.identity().get('mode') == 'system' else 'runs.update' @@ -556,7 +568,7 @@ def _admit_external_prepared(self, body, captured, run_id, staging, *, raise WorkbenchError('Insufficient storage for declared outputs, publication copy and reserve.', 'unavailable', 409) if check_current and self.pipelines._get(graph['id']) != graph: raise WorkbenchError('Graph changed during capture.', 'stale', 409) - if sum(self._load(row[0])['state'] in ACTIVE for row in self.db.execute('SELECT id FROM workspace_runs')) >= 8: + if len(self._in_states(ACTIVE)) >= 8: raise WorkbenchError('The local queue is full (8 attempts).', 'unavailable', 409) grant_id = body.get('approval_id') if grant_id is None: @@ -1191,7 +1203,7 @@ def _reconcile_receipt(self, receipt): return receipt def _admit(self, key, request_hash, graph, validation, release, parent=None, *, run_id=None, composition=None): - queued = sum(self._load(row[0])['state'] in ACTIVE for row in self.db.execute('SELECT id FROM workspace_runs')) + queued = len(self._in_states(ACTIVE)) if queued >= 8: raise WorkbenchError('The local queue is full (8 attempts). Wait or cancel a queued attempt.', 'unavailable', 409) run_id = run_id or uuid.uuid4().hex @@ -1313,7 +1325,7 @@ def _loop(self): self._wake.clear() continue with self.lock: - queued = [self._load(row[0]) for row in self.db.execute('SELECT id FROM workspace_runs')] + queued = self._in_states(('queued',)) queued = sorted((run for run in queued if run['state'] == 'queued' and run['id'] not in active and (not self._external(run) or run['id'] in self._confirmed_external)), key=lambda run: (run['created_at'], run['id'])) @@ -1331,7 +1343,9 @@ def _loop(self): used = {cpu for _, _, cpus in active.values() for cpu in cpus} cpus = tuple(cpu for cpu in self._capacity.cores if cpu not in used)[:claim[2]] active[run['id']] = (executor.submit(self._run_attempt, run, cpus, claim), claim, cpus) - self._wake.wait(.05 if active else .25) + # With no pending work only admission/stop can change the queue. + # Queued resource waits still retry when headroom changes. + self._wake.wait(.05 if active else .25 if queued else None) self._wake.clear() finally: # On coordinator/storage failure, active attempts observe stop @@ -2203,8 +2217,7 @@ def close(self): if self.runtime_owner: try: with self.lock, self.db.transaction(): - for row in self.db.execute('SELECT id FROM workspace_runs'): - run = self._load(row[0]) + for run in self._in_states(('queued',)): if run['state'] == 'queued': self._event(run, 'cancelled', 'Application closed before this queued attempt started.') self._save(run) diff --git a/tests/test_run_scheduling_reads.py b/tests/test_run_scheduling_reads.py new file mode 100644 index 00000000..934adf2a --- /dev/null +++ b/tests/test_run_scheduling_reads.py @@ -0,0 +1,150 @@ +"""Queue coordination must not revalidate terminal history while holding its lock.""" +import json +import threading + +from workbench import WorkbenchError, encode +from unittest.mock import patch + +from test_pipeline import WorkspaceCase +from test_external_runs import ExternalCase + + +class RecordedWake: + def __init__(self): + self.event = threading.Event() + self.waiting = threading.Event() + self.timeouts = [] + + def set(self): + self.event.set() + + def clear(self): + self.event.clear() + + def wait(self, timeout=None): + self.timeouts.append(timeout) + self.waiting.set() + return self.event.wait(timeout) + + +class SchedulingReadTests(WorkspaceCase): + def queued(self, key): + body = {'request_key': key, 'graph_id': self.graph['id'], + 'graph_revision': self.graph['revision'], + 'validation_token': self.validate()['validation_token']} + with patch.object(self.pipelines, 'probe', return_value=self.ready()), \ + patch.object(self.dataset.runs, '_start'): + return self.dataset.runs.submit(body) + + def test_idle_coordinator_never_loads_terminal_runs(self): + owner = self.dataset.runs + history = self.queued('terminal-history-0001') + self.assertEqual(owner.cancel(history['id'])['state'], 'cancelled') + wake = RecordedWake() + owner._wake = wake + with patch.object(owner, '_load', wraps=owner._load) as load: + try: + owner._start() + self.assertTrue(wake.waiting.wait(5), 'coordinator never reached its wait') + self.assertNotIn(history['id'], [call.args[0] for call in load.call_args_list], + 'idle coordination fully validated terminal history') + finally: + owner.close() + self.assertFalse(owner._thread.is_alive()) + + def test_empty_coordinator_waits_for_an_event_and_shutdown_wakes_it(self): + owner = self.dataset.runs + wake = RecordedWake() + owner._wake = wake + try: + owner._start() + self.assertTrue(wake.waiting.wait(5), 'coordinator never reached its wait') + self.assertIsNone(wake.timeouts[0], 'an empty queue must not poll the catalog') + finally: + owner.close() + self.assertFalse(owner._thread.is_alive()) + + def test_new_admission_wakes_an_empty_coordinator(self): + owner = self.dataset.runs + wake, executed = RecordedWake(), threading.Event() + owner._wake = wake + + def finish(run_id): + owner._finish(run_id, 'cancelled', 'Fixture finishes without spawning a worker.') + executed.set() + + with patch.object(owner, '_execute', side_effect=finish): + try: + owner._start() + self.assertTrue(wake.waiting.wait(5)) + body = {'request_key': 'wake-idle-coordinator-0001', 'graph_id': self.graph['id'], + 'graph_revision': self.graph['revision'], + 'validation_token': self.validate()['validation_token']} + with patch.object(self.pipelines, 'probe', return_value=self.ready()): + run = owner.submit(body) + self.assertTrue(executed.wait(5), 'committed admission did not wake the idle coordinator') + self.assertEqual(owner.get(run['id'])['state'], 'cancelled') + finally: + owner.close() + + def test_queued_resource_shortage_retries_without_a_new_admission(self): + owner = self.dataset.runs + run = self.queued('retry-headroom-0001') + wake, executed = RecordedWake(), threading.Event() + owner._wake = wake + available = [0] + + def finish(run_id): + owner._finish(run_id, 'cancelled', 'Fixture finishes after resource availability changes.') + executed.set() + + with patch('run_capacity.available_memory', side_effect=lambda: available[0]), \ + patch.object(owner, '_execute', side_effect=finish): + try: + owner._start() + self.assertTrue(wake.waiting.wait(5)) + self.assertEqual(wake.timeouts[0], .25) + self.assertFalse(executed.is_set()) + available[0] = 64 * 1024**3 + self.assertTrue(executed.wait(5), 'queued work did not retry after headroom changed') + self.assertEqual(owner.get(run['id'])['state'], 'cancelled') + finally: + owner.close() + + def test_builtin_admission_does_not_load_terminal_history_for_capacity(self): + owner = self.dataset.runs + history = self.queued('builtin-history-0001') + owner.cancel(history['id']) + with patch.object(owner, '_load', wraps=owner._load) as load: + self.queued('builtin-after-history-0002') + self.assertNotIn(history['id'], [call.args[0] for call in load.call_args_list]) + + def test_selected_queued_run_still_rejects_a_corrupt_envelope(self): + owner = self.dataset.runs + run = self.queued('queued-integrity-0001') + self.assertEqual([row['id'] for row in owner._in_states(('queued',))], [run['id']]) + with self.dataset.lock, self.dataset.db.transaction(): + row = self.dataset.db.execute('SELECT document FROM workspace_runs WHERE id=%s', (run['id'],)).fetchone() + stored = json.loads(row[0]) + stored['graph_sha256'] = '0' * 64 + self.dataset.db.execute('UPDATE workspace_runs SET document=%s WHERE id=%s', (encode(stored), run['id'])) + try: + with self.assertRaises(WorkbenchError) as failure: + owner._in_states(('queued',)) + self.assertEqual(failure.exception.code, 'unavailable') + self.assertEqual(str(failure.exception), 'Stored run envelope is invalid; refusing to guess execution state.') + finally: + with self.dataset.lock, self.dataset.db.transaction(): + self.dataset.db.execute('UPDATE workspace_runs SET document=%s WHERE id=%s', (row[0], run['id'])) + + + +class ExternalSchedulingReads(ExternalCase): + def test_external_admission_does_not_load_terminal_history_for_capacity(self): + owner = self.dataset.runs + history = self.queued(self.body(approval=self.approve())) + with self.owner.bound(self.dataset): + owner.cancel(history['id']) + with patch.object(owner, '_load', wraps=owner._load) as load: + self.queued(self.body(approval=self.approve())) + self.assertNotIn(history['id'], [call.args[0] for call in load.call_args_list]) From de0f1b833f1922b4a86c12036405fe8cd795b4a0 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 18 Oct 2026 05:20:51 -0700 Subject: [PATCH 9/9] docs(perf): retain live request latency and CPU restore qualification --- .../request-latency-v1/execution-ledger.md | 44 ++++ docs/plans/request-latency-v1/issues.md | 3 +- docs/plans/request-latency-v1/plan.md | 28 +- .../reports/cpu-restore.json | 116 ++++++++ .../reports/deployment.json | 249 ++++++++++++++++++ .../reports/http-after-01.json | 30 +++ .../reports/idle-after.json | 9 + .../reports/preflight-after.json | 27 ++ .../reports/reads-after.json | 62 +++++ .../reports/restore-receipt-comparison.json | 10 + 10 files changed, 570 insertions(+), 8 deletions(-) create mode 100644 docs/plans/request-latency-v1/reports/cpu-restore.json create mode 100644 docs/plans/request-latency-v1/reports/deployment.json create mode 100644 docs/plans/request-latency-v1/reports/http-after-01.json create mode 100644 docs/plans/request-latency-v1/reports/idle-after.json create mode 100644 docs/plans/request-latency-v1/reports/preflight-after.json create mode 100644 docs/plans/request-latency-v1/reports/reads-after.json create mode 100644 docs/plans/request-latency-v1/reports/restore-receipt-comparison.json diff --git a/docs/plans/request-latency-v1/execution-ledger.md b/docs/plans/request-latency-v1/execution-ledger.md index 2c043a1c..364b317d 100644 --- a/docs/plans/request-latency-v1/execution-ledger.md +++ b/docs/plans/request-latency-v1/execution-ledger.md @@ -68,3 +68,47 @@ The earlier live 7–8 s / one-core measurements remain valid observations of th running coordinator. Do not pair the later fast receipt as a healthy baseline. After activation, require an actual fresh completed attempt and post-completion read measurements. No claim of an exact paired speedup or full-pipeline speedup. + + +Activation: qualified source eed2b84d21050943d211defffc3133609422950a copied +into current-07 using the unchanged 193-file inventory. Only runs.py product bytes +differ. Confirmed no active/unknown Runs, gracefully ceased the exact previous +PID, then started one owner with unchanged arguments/environment/interpreter. +HTTP/session/catalog checks passed; all 55 existing Run document hashes preserved. +Transition took 18.003 s. Procedure source and receipts are retained by the lab. + +Actual Pumas CPU qualification: existing C67 graph, identical material digest, +one journaled approval and submission, zero optimizer updates. New Run +29472d8e679a48b1933d4683e401ad02 completed with drained custody. Approval 13.937 s, +admission 27.849 s, completion from controller start 78.097 s; all readback done +in 100.273 s. Ten artifacts whole-hash verified; nine expected immutable outputs +match original C67 restore exactly, including weights and predictions. The tenth, +receipt.json, differs only in elapsed_s, per_case_inference_s and peak_rss_bytes. +No CUDA training, numerical change, promotion or scientific-history replacement. + +Original HTTP reproduction now passes its two-second diagnostic trigger: selected +old Run reads 0.443/0.446 s and old 189-byte artifact reads 0.393/0.394 s, versus +earlier 7.351/8.271 s and 7.889/7.674 s with the old coordinator actually running. +Same client script hash; same owner/catalog/Run/artifact. Additional three-sample +readback medians: Run .527 s, artifact .476 s (see ranges). Its first measurements +were near the end of restore-artifact collection; the original two-sample rerun +was fully quiescent. Idle app uses 0 CPU seconds in 3.000 s after both benchmarks, +three threads remain and service log has no coordinator exception. This combines +real attempt completion, live process continuity and targeted event-wakeup tests; +an idle server alone is not the evidence. Measurements are observations, not an +exact paired speedup, universal latency SLA or whole-training throughput claim. + +Remaining source preparation: paired current graph preflight 15.104 -> 14.263 s, +no substantive change. New restore artifact reads take ~2.1–2.3 s each because +that workload has a different captured input closure; do not substitute selected +old-Run .4 s reads for all artifacts. Historical C67 admission was 210.781 s; +retain as context, not a controlled before/after timing. Further preparation +optimization must preserve current material, authorization and custody checks. + +Environment: Linux x86_64, Core Ultra 9 275HX, 24 logical CPUs, ~62 GiB RAM; +application CPython 3.12, PostgreSQL 16; actual managed pressure restore uses the +existing Pumas-qualified runtime and exact existing one-CPU resource claim. +Capacity settings remain six CPU slots, 4 GiB per CPU Run, 8 GiB per CUDA Run, +8 GiB reserve. The task-created private PostgreSQL test service was stopped after +qualification. No debug instrumentation, profiler package or credential added to +product/deployment. Existing RES-1 and ENV-1 are explicitly separate limitations. diff --git a/docs/plans/request-latency-v1/issues.md b/docs/plans/request-latency-v1/issues.md index 8e4da554..2242eb56 100644 --- a/docs/plans/request-latency-v1/issues.md +++ b/docs/plans/request-latency-v1/issues.md @@ -3,7 +3,8 @@ PERF-1 (high, owner Runs, fix-now): constant full-history validation under the catalog lock in _loop and queue-capacity checks. Real minimal HTTP read plus unchanged queue-scan profile and targeted regressions identify it. Selected-state -routing and idle event waiting are implemented; live qualification remains. +routing and idle event waiting are implemented and live qualification passed. +Resolved in eed2b84; no claim that all preparation latency is removed. ENV-1 (qualification limitation, owner legacy image adapter): the pinned CPU training runtime is absent from this application environment. One existing diff --git a/docs/plans/request-latency-v1/plan.md b/docs/plans/request-latency-v1/plan.md index bff0b2ec..21939c70 100644 --- a/docs/plans/request-latency-v1/plan.md +++ b/docs/plans/request-latency-v1/plan.md @@ -1,6 +1,6 @@ # Tuldok request latency -Status: Active — implementation. Canonical operation: start this performance plan. +Status: Complete — scoped repair activated and qualified locally. Jeremy paused the next model experiment and authorized improving Tuldok's speed. Objective: remove the measured idle-looking waits from ordinary local Run reads, artifact readback and the affected preparation/admission path without weakening @@ -63,12 +63,13 @@ from the named branch; when its purpose ends, remove it only if clean and after recording the protected head and branch disposition. Never prune other worktrees. Coding-standards snapshot: snapshot:v1:ffcfc3f8-b52a-434e-b57f-d259a0355711. -Current acceptance: baseline/cause and repair regressions satisfied; live activation -and representative after-update workload qualification pending. -Exactly one next slice: activate the qualified source with no active Runs, qualify -one fresh zero-update CPU restore to warm the coordinator, and measure identical -known-Run reads and preflight before/after. Ledger: execution-ledger.md; -findings: issues.md; evidence: reports/. +Current acceptance: baseline/cause, red-before-green scheduling regressions, +local activation, real CPU restore and healthy post-attempt HTTP measurements are +satisfied. See execution-ledger.md and reports/. No model training was started. +A new performance slice would first profile remaining graph preparation and +approval/admission source-validation costs; it is not included in this repair. +Existing coordinator lock-timeout resilience is a separate retained finding. + Bounded caller inventory: _loop, _admit_external_prepared, built-in _submit and @@ -115,3 +116,16 @@ the fixed coordinator must be quiet after an actual attempt has finished. Composed-design applicability: existing Runs remains the single coordination, capacity and custody owner. The new private selector serves four existing callers; no new production architectural owner or duplicate durable state is introduced. + + +Terminal resource disposition: after this final documentation commit, remove only +the clean task-created worktree, protecting its exact head on +performance/catalog-request-latency-20261018. The surrogate lab acceptance report +records the exact protected OID and confirms removed-reachable. The deployed +current-07 is an independent immutable copy of the qualified eed2b84 source. +Retained proposal contract: Jeremy owns review/integration into the existing +fix/output-inventory-rename-20261017 integration line (base 225f8e8). Keep the +proposal unchanged for review; no automatic synchronization/rebase/push. Consumers +are the local current-07 deployment and this lab qualification. Retire the proposal +after accepted integration or explicit disposition. Upstream main integration +must also account for the existing CUDA/concurrency changes inherited at base. diff --git a/docs/plans/request-latency-v1/reports/cpu-restore.json b/docs/plans/request-latency-v1/reports/cpu-restore.json new file mode 100644 index 00000000..553f6df0 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/cpu-restore.json @@ -0,0 +1,116 @@ +{ + "status": "passed", + "run_id": "29472d8e679a48b1933d4683e401ad02", + "material_sha256": "d06e6ee426c0e0e661010a8107b3069de01ee4522b3e875cfd909236dcb07a68", + "completed_drained": true, + "optimizer_updates": 0, + "training_started": false, + "completed_elapsed_s": 78.09680777398171, + "total_elapsed_s": 100.27325369298342, + "stages": [ + { + "operation": "/api/workspace/runs/approve-external", + "elapsed_s": 13.93690972100012 + }, + { + "operation": "runs.submit_external", + "elapsed_s": 27.849457297998015 + } + ], + "artifacts": [ + { + "name": "checkpoint-000.npz", + "bytes": 1485452, + "sha256": "3ac3c9a3a64c436f653947085aa34968b7699c2e0fd6264390ac257e460de954", + "elapsed_s": 2.2767949480039533, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "profile.json", + "bytes": 189, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5", + "elapsed_s": 2.193891868024366, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "predictions.npz", + "bytes": 129230, + "sha256": "5f15255f57398ea3c7ef6f7bf8946f771060a0640b3aaf5977f4603fef8a62d3", + "elapsed_s": 2.2210796549916267, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "weights.npz", + "bytes": 1485452, + "sha256": "0e32d48fc84fa555459559198bc782cb6d1680c2953dc1e7883a3b71e93885b8", + "elapsed_s": 2.28736215698882, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "history.json", + "bytes": 10196, + "sha256": "07d1eb568a034b3bdc4aacd7566a298a9e2a73896196037d3cb471a1505a8e6f", + "elapsed_s": 2.208552375988802, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "splits.json", + "bytes": 256361, + "sha256": "bdcfb5f5d8e4dbd54f797d827e2dd0715fcc2bfb6510697fc51996c3d5a5779e", + "elapsed_s": 2.207806910009822, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "checkpoint-1600.npz", + "bytes": 1485452, + "sha256": "0e32d48fc84fa555459559198bc782cb6d1680c2953dc1e7883a3b71e93885b8", + "elapsed_s": 2.2771396540047135, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "normalization.json", + "bytes": 6752, + "sha256": "f73b6cf32108f63cbd8d5edef8b08afce2e3a05d47234886458ddfbc99e1d1c1", + "elapsed_s": 2.237189501989633, + "whole_sha256_verified": true, + "matches_original": true + }, + { + "name": "receipt.json", + "bytes": 12291, + "sha256": "e6933aee80c016c24785e1ed419ccba99b6ca0cf58356ec2e2c0f2b59c368132", + "elapsed_s": 2.147997332009254, + "whole_sha256_verified": true, + "matches_original": false + }, + { + "name": "summary.json", + "bytes": 367, + "sha256": "219bf46c11d3d952ad027d7dad84d8652e11b395faf40358c684a8500ec4a2e2", + "elapsed_s": 2.118357458995888, + "whole_sha256_verified": true, + "matches_original": true + } + ], + "unchanged_artifacts": [ + "checkpoint-000.npz", + "checkpoint-1600.npz", + "history.json", + "normalization.json", + "predictions.npz", + "profile.json", + "splits.json", + "summary.json", + "weights.npz" + ], + "source_sha256": "92092ac2e080012a009b91043fc81d619062d87a665e6fc9f4be824703105317", + "config_sha256": "b2661eb6e1432a931fd09bf2d65a167dbcdc295026969791a0b01d18e96bb8fa", + "automatic_retry": false +} diff --git a/docs/plans/request-latency-v1/reports/deployment.json b/docs/plans/request-latency-v1/reports/deployment.json new file mode 100644 index 00000000..5a5ec230 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/deployment.json @@ -0,0 +1,249 @@ +{ + "format": "tuldok-request-latency-deployment-v1", + "status": "passed", + "source_revision": "eed2b84d21050943d211defffc3133609422950a", + "old_pid": 855603, + "old_start_ticks": "10183125", + "catalog_id": "a7da4a8de53541a2a0e2aad08005bb71", + "existing_runs": 55, + "active_runs": 0, + "old_run_snapshot": { + "598ef9774bd547fba9eddb72f0b6114d": { + "state": "completed", + "document_sha256": "e93b6760e209b665a9c3e200599dac5373ef00330e4af2fb73c482e2604851f6" + }, + "f94e2a345cb14cdf8672365dbe8d615f": { + "state": "completed", + "document_sha256": "7a67cbfe9e6534fdaf352edb1d756e6f8155572d0e9cd0476e28e071e77f8578" + }, + "5c5b54957b1544ba87240263aba9a8e6": { + "state": "completed", + "document_sha256": "ae72f801c7fb83487981930a64388abe32c3cd7ee11d44373a2ec66b6142e1f9" + }, + "4f5bfea59ebd474d934ca967ff01927c": { + "state": "completed", + "document_sha256": "2b53de1c0cf66fdcf859401869f2c5be770e2a3de572fd3e64173b07928e5c8e" + }, + "80e35ae9571045898d4344b5c4a42f2c": { + "state": "completed", + "document_sha256": "72e117588fdf1c27290fa96ecbdf3eda20ca17cad318efb2f4ba8e47eebb0833" + }, + "6b75f3ce89e64c8fb45ac514d0f386ae": { + "state": "completed", + "document_sha256": "a1c563b9b13623081711e885b225f7a10e3f70665f63b9dfb8424b0beb81fe31" + }, + "f8e6bd15f5ba4e6bbd5571071ecb2798": { + "state": "failed", + "document_sha256": "44f684dee0afbe6ffa9d763d7091899a626a854f34490db4d3e84587a2eebc1f" + }, + "89e5e42c4563405d9afb3942907aea14": { + "state": "completed", + "document_sha256": "1c2a3c06df35634c9ea18b2a283bb21f37d4a3e494e45b7345601fea58be4bf1" + }, + "cfe01464cb8d4444b546cc06b4fc304f": { + "state": "completed", + "document_sha256": "b75681b90b615f931e30181b5c33809b44de7bec889ce010d8e33a38f71a2c2f" + }, + "b88b4eb0c465471d93dc3f8f5cb103e7": { + "state": "completed", + "document_sha256": "c3d4f018504f13a24a6fc75e2306d6a3218a0f5ad3ac0795575511055a48b7a3" + }, + "6f0d80863535407e86f10f625a0975a6": { + "state": "completed", + "document_sha256": "59bf6d38c9b1717e5cc2be4cdb10f2a51adac640952b12974ba645f3a8f94081" + }, + "01b94f66a77f4415b622617148123dca": { + "state": "completed", + "document_sha256": "f67478e1bcc0bac65e3be8e6cf4eb4718ee0320cb363b4c57d2396352a16ea72" + }, + "786bdd62edb845fe8950c4a952bbcc78": { + "state": "failed", + "document_sha256": "35d7884166d03165a331052bc269de4521361a3dd0786fd10bb5dfec817888db" + }, + "3987f38145474a2d83a025de8a28d1c4": { + "state": "completed", + "document_sha256": "99a174926d8ef1f249d8b97e902fd7be7eb7a30fbe3aba525acc6ebc157bf589" + }, + "9c9777fcb4b541e98dc03e66c1df4f73": { + "state": "completed", + "document_sha256": "c3f97ce7adf5eb6bc6c708852299a22be94fa3c375ebe22e606f6cfb17e4c6c5" + }, + "210a52fd86524ebf9411fed3aae2dd4b": { + "state": "completed", + "document_sha256": "ffdad06c7a315aedb9443e032a99a5dfbeb287ba7e58c1b4ff7070147d38180c" + }, + "20c3de2d9be14873a5fa0fc9ae391323": { + "state": "completed", + "document_sha256": "8d43d881978264a51cfd7eb183a8fd25282701b495fce1bf62f6266795b388dd" + }, + "db713a51ecec4c689ffa0ff3cff391cc": { + "state": "completed", + "document_sha256": "dabee42c2343cbe48de41a588ee454fbd27c25ac4a379c5819424f48dba9aefa" + }, + "ae3dc266d76147aea4fbd120b28a7502": { + "state": "completed", + "document_sha256": "22a8406be8eaac39def800b8f369b0434af65b5e559c952c30f6f18ea2aa945d" + }, + "42b18c5da68c4382b7e5ff511c28998d": { + "state": "completed", + "document_sha256": "3e6bfcc1c3b80c0068852ab4d7f062d4415c408c91363dc78010dd08212df717" + }, + "2d79680874d343c7978c6249667b2df9": { + "state": "completed", + "document_sha256": "fb57aff7ff58612677ce3c3f07e4fe76a25b750ffed488cb2044f63340b544f3" + }, + "8bc0a9c839aa42a0a01064c20d70af86": { + "state": "completed", + "document_sha256": "d717bc82d05aed2fce6be9d5ecdf13ab8b6ab5f257434d071a869a90f29680f8" + }, + "a7f26e3d6f2d42d297ebf80d23e09e4c": { + "state": "completed", + "document_sha256": "52ded73e2ac8fe06989189e19d8723a04dfaeb0e494bff21d317182fd9947a54" + }, + "21b9fee4399f46748cf7c5a6af051b34": { + "state": "completed", + "document_sha256": "de59eff2a467e021e853575fafd5b782abd2fd016767d327684b90d8b85b9902" + }, + "bc971e81be5f40b7bfa8e585ef47bd2f": { + "state": "completed", + "document_sha256": "19314e2a9573a4b7ecf0042ced8b45c1b45bffc62a55a7f95e945bed52bd07bc" + }, + "e8f26974fa8a4705ba4d63cf7179a3ab": { + "state": "failed", + "document_sha256": "b3d1d9232e65f8f3c355f69659da1fd44b58ca73f9a51974efad88f6db335555" + }, + "5770116825f647ddabd2ba7e5413d885": { + "state": "failed", + "document_sha256": "0edc47b64c283efef01d7791e9e7b33fe01135be4a160bf00f1ded7f0948d027" + }, + "ab4a6325b4e94631be46567530bb563c": { + "state": "completed", + "document_sha256": "5c15be69e2b577a3960f2947656ec3cb6ce6fea6a9ece4d9752fa86b6c25183f" + }, + "d0b98276805b40a2942e2b123fe35330": { + "state": "failed", + "document_sha256": "7c64e9cb8436125f3b67dca36ba05fffab0f1ab159a01d7b6f370f20ec565c11" + }, + "49d70e61e134410595852c1ff2b36472": { + "state": "completed", + "document_sha256": "caccaf74724a24c6db9fb69975d0786dfa7836a9f915f0f7658cd16c9f2dae7e" + }, + "707c16c97bd54738b038e6c7b7828689": { + "state": "completed", + "document_sha256": "a3192d351e16cb490e9216fc306a6ef933747c565f31b83800356b72b1b0919a" + }, + "54a0a37abbbf47aa8e5bd5a4aedc3e7f": { + "state": "completed", + "document_sha256": "2cc9917f30143d664ed69f3ecbb934c705fbfa10fdc6ba2b4941d4153fe89311" + }, + "c01e1741f48a474d9a2a3252b3971827": { + "state": "completed", + "document_sha256": "1f4bbf2d9aa1fb5df5b2afafbb6b04ff104dc0b9da8e075f55d3d596642662e8" + }, + "1f751fab85a2472f8879493ff017022f": { + "state": "completed", + "document_sha256": "946e4fb0207aa4078b961bb3a8221a829adc2ea88a3decb65ec4caea86275726" + }, + "d7fe5ebc0fdf47b6b53eeb8442c4902e": { + "state": "completed", + "document_sha256": "21345c214dc3de94c9601699feaf06e698c116a896aacd7d76e593caac3d93b3" + }, + "474749c5b43c481f941b7d022d0e7a55": { + "state": "completed", + "document_sha256": "9d644c1f54c16bd9af2fe621f0d6ff6a995d878c14e1ec363fa0b383b6cf1a54" + }, + "9849b84200b340679c84a9d937bdb492": { + "state": "completed", + "document_sha256": "091531677e29246d75f9370f925fa97430ac7d4841744fab5a193dec1d48b0ed" + }, + "1d6afc137a5b46909688484521d29b5f": { + "state": "completed", + "document_sha256": "eaeeaec07b4cb449e1caa8f0578165dbef78507a48b7134370779fe3bd3492d2" + }, + "06df7e568a8d4cf7bd2190e82c7e531a": { + "state": "completed", + "document_sha256": "80a0bb58041e39baac0e12cc1b965a0601a1d8e46fe5d303230d44e6742f55c3" + }, + "b870e3b264b3446d9bc9c90480050991": { + "state": "completed", + "document_sha256": "9a34badd567b1bb2c235f41a6e1a94789cec4d4c6e6ccaa8e17c9367f10ac0fe" + }, + "00b8fcac2b364acfa5b370cef3c8d3ad": { + "state": "completed", + "document_sha256": "29ded3fe5af5322cd2aee6922dcd66ca9e7c843c414c39e0835efd626738666a" + }, + "27a85aea6bea49ffa166bcabce8e0aa3": { + "state": "completed", + "document_sha256": "389ecdbc403fe399c01c2d9fcfa4f4fab78f83bd00e74a5cf8b6d5e4cc73b359" + }, + "f4011890960040359b6ab83db245dd6e": { + "state": "completed", + "document_sha256": "2202e590351420c58764116236f6aae3fde42e441b51d54e31f650f759eaf1e7" + }, + "71e22ac78d16420bb89ebb8b0dc695e0": { + "state": "completed", + "document_sha256": "dc3b35e142e30eb3ae5094338c5f11c576933eb25300324500ffe9c8bb348ccd" + }, + "c42bb8e6d3144cd38d140663cbda90c7": { + "state": "completed", + "document_sha256": "293cd3a3b27f8535dc95e1f90638fcaa5adab91855c86396a599f1d4f1584756" + }, + "2097763eed3249c186119e920eeee788": { + "state": "completed", + "document_sha256": "9412568e697f59c6b23b068fd43ff7703a9d060dae0f7e0d151c69609504a4e0" + }, + "b6ab6203750543a18ad61cd4de588cd4": { + "state": "completed", + "document_sha256": "5e2fde141fcaf83a6d95622038378d03c64220b4238305b4c54e62965978f8bb" + }, + "456cdcccb54c4e99b2bdebc19393f991": { + "state": "completed", + "document_sha256": "a4d9f3507841e77032e6ecd3c9e9c4f36ad7b4639e82cdb0011ac0dc398ef4a9" + }, + "e17624f982b54e858698207227010410": { + "state": "completed", + "document_sha256": "abca924084186422c3a34decd04fccf1f9dd392a053594d4a043c0a3f3ea1bce" + }, + "324fbebc9fdb4c65b11dc6a43b4a55c8": { + "state": "completed", + "document_sha256": "7c30550c4e39c3ba54fcc55b6d798f14907467c40d0e215ffac696b74b4cfb70" + }, + "622caf928da24d3c8b0f2d83433b953d": { + "state": "completed", + "document_sha256": "fba35a100adaf4fffc23cd12fd68f415c5fee3b64949adc00eeda0c5879487f3" + }, + "ef4298f2e657491080732e7aaf089895": { + "state": "completed", + "document_sha256": "3341e7df901ddc02fc44e4a4b993262c11873730b04ce8cea6e26a233d4c2cd3" + }, + "019534bd13974678ab4c2eb11eda5de2": { + "state": "completed", + "document_sha256": "3be976b81b81ef3e213e94017b28c4635b0b85515ca7ff1893d566e4f9f2e6ab" + }, + "849bdddaeff3476a88b37813e311da4a": { + "state": "completed", + "document_sha256": "083344804e5eda3b1d770a9a5818821878edc2117564f62410bdc4edff8988bf" + }, + "0b8f06c8e62d4dd481c6b5c2183c2b5b": { + "state": "completed", + "document_sha256": "41ab9b868cbf7a4796ba86943a2027f29ea4d38d797b83bc25035c68b9ab6a33" + } + }, + "changed_product_files": [ + "runs.py" + ], + "deployment": "runs/dependency-migration-v1/Tuldok-deployed/current-07", + "manifest_sha256": "8d6fa3f3924ee1f4c5e8bb836e5ef9094dbae3165173681699f1819ccecaa951", + "automatic_retry": false, + "pid": 869892, + "start_ticks": "18114060", + "elapsed_s": 18.00320268899668, + "existing_run_documents_preserved": true, + "original_owner_session_preserved": true, + "capacity_environment": { + "TULDOK_CPU_RUN_SLOTS": "6", + "TULDOK_CPU_RUN_MEMORY_MIB": "4096", + "TULDOK_CUDA_RUN_MEMORY_MIB": "8192", + "TULDOK_RUN_MEMORY_RESERVE_MIB": "8192" + } +} diff --git a/docs/plans/request-latency-v1/reports/http-after-01.json b/docs/plans/request-latency-v1/reports/http-after-01.json new file mode 100644 index 00000000..88d32f0e --- /dev/null +++ b/docs/plans/request-latency-v1/reports/http-after-01.json @@ -0,0 +1,30 @@ +{ + "status": "passed", + "scope": "Two current typed reads of a known completed Run and its 189-byte artifact; no admission or training.", + "request_target_s": 2, + "artifact_bytes": 189, + "run_id": "e17624f982b54e858698207227010410", + "rows": [ + { + "operation": "owner_initialization", + "elapsed_s": 1.1586178519937675 + }, + { + "operation": "runs.get", + "elapsed_s": 0.44344867701875046 + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.39251917801448144 + }, + { + "operation": "runs.get", + "elapsed_s": 0.44550499800243415 + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.3940472009999212 + } + ], + "source_sha256": "8c6b1cedca245fc073b2ad806836632a4f01073b54f48869605e7d5d39f405c0" +} diff --git a/docs/plans/request-latency-v1/reports/idle-after.json b/docs/plans/request-latency-v1/reports/idle-after.json new file mode 100644 index 00000000..1a6c1e7a --- /dev/null +++ b/docs/plans/request-latency-v1/reports/idle-after.json @@ -0,0 +1,9 @@ +{ + "wall_s": 3.0002375210169703, + "app_cpu_s": 0.0, + "pid": 869892, + "thread_count": 3, + "start_ticks": "18114060", + "no_thread_exception_in_service_log": true, + "phase": "Quiescent, after completed CPU restore, all artifact readback and both HTTP benchmarks finished" +} diff --git a/docs/plans/request-latency-v1/reports/preflight-after.json b/docs/plans/request-latency-v1/reports/preflight-after.json new file mode 100644 index 00000000..9498f81e --- /dev/null +++ b/docs/plans/request-latency-v1/reports/preflight-after.json @@ -0,0 +1,27 @@ +{ + "status": "completed", + "kind": "preflight", + "owner_initialization_s": 1.0212907360109966, + "rows": [ + { + "operation": "graphs.preflight_external", + "elapsed_s": 14.263433745974908, + "material_sha256": "d06e6ee426c0e0e661010a8107b3069de01ee4522b3e875cfd909236dcb07a68", + "graph_id": "209124c1cc424fbbb41afab9314f1416", + "revision": 1 + } + ], + "summary": { + "graphs.preflight_external": { + "samples": 1, + "minimum_s": 14.263433745974908, + "median_s": 14.263433745974908, + "maximum_s": 14.263433745974908 + } + }, + "run_id": "e17624f982b54e858698207227010410", + "training_started": false, + "scope": "Same client, catalog and immutable workload. Read-only current-authority checks; timings are observations, not a universal product SLA.", + "config_sha256": "a2e54720b2f5e2c1457926b24cf94253fdedd913eb7ffd74b5ccef92f01eae2e", + "source_sha256": "a49b274cd87215512c72b88de4d97e4fd9af391a496974275021dcc53d026124" +} diff --git a/docs/plans/request-latency-v1/reports/reads-after.json b/docs/plans/request-latency-v1/reports/reads-after.json new file mode 100644 index 00000000..7d1d4005 --- /dev/null +++ b/docs/plans/request-latency-v1/reports/reads-after.json @@ -0,0 +1,62 @@ +{ + "status": "completed", + "kind": "reads", + "owner_initialization_s": 1.0310473950230516, + "rows": [ + { + "operation": "runs.get", + "elapsed_s": 0.5212976390030235, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.45595599099760875, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + }, + { + "operation": "runs.get", + "elapsed_s": 0.5269138979783747, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.49457995701231994, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + }, + { + "operation": "runs.get", + "elapsed_s": 0.5266742210078519, + "completed_drained": true + }, + { + "operation": "runs.artifact_chunk", + "elapsed_s": 0.47564829999464564, + "bytes": 189, + "whole_sha256_verified": true, + "sha256": "bc399290a58d4042c0034fce0726719aa83d6cbb0569c47714b95b7225ee26e5" + } + ], + "summary": { + "runs.artifact_chunk": { + "samples": 3, + "minimum_s": 0.45595599099760875, + "median_s": 0.47564829999464564, + "maximum_s": 0.49457995701231994 + }, + "runs.get": { + "samples": 3, + "minimum_s": 0.5212976390030235, + "median_s": 0.5266742210078519, + "maximum_s": 0.5269138979783747 + } + }, + "run_id": "e17624f982b54e858698207227010410", + "training_started": false, + "scope": "Same client, catalog and immutable workload. Read-only current-authority checks; timings are observations, not a universal product SLA.", + "config_sha256": "b2661eb6e1432a931fd09bf2d65a167dbcdc295026969791a0b01d18e96bb8fa", + "source_sha256": "a49b274cd87215512c72b88de4d97e4fd9af391a496974275021dcc53d026124" +} diff --git a/docs/plans/request-latency-v1/reports/restore-receipt-comparison.json b/docs/plans/request-latency-v1/reports/restore-receipt-comparison.json new file mode 100644 index 00000000..8336a1fa --- /dev/null +++ b/docs/plans/request-latency-v1/reports/restore-receipt-comparison.json @@ -0,0 +1,10 @@ +{ + "status": "passed", + "different_paths": [ + "/elapsed_s", + "/per_case_inference_s", + "/runtime/peak_rss_bytes" + ], + "all_other_fields_identical": true, + "scope": "Only wall timing, per-case inference timing and peak RSS differ; numerical outputs verified separately." +}