From 551b3418d4ea6dab6400e40efe6ccf69a26b7a6e Mon Sep 17 00:00:00 2001 From: ejc3 Date: Tue, 27 Oct 2026 06:56:52 +0000 Subject: [PATCH] fleet: viewers read a fixed sentence when the word check cannot answer A viewer who wrote a word on staging read "the text check failed: no text check is configured": apply.rs prefixed the checker's own error, and that error also went into the receipt's check and the public failures list, where a Bedrock error would name the model, region or AWS message. - text_check.rs: an error verdict's reason is "words are turned off right now" for --text-check none (NoCheck answers with a Disabled error) and "couldn't check that word, try again" for any other error or the 5 s timeout. The error itself is logged at warn. - apply.rs: an error verdict refuses with its reason as is. - 07_words.json vector and the web fixture regenerated from it, PROTOCOL.md. - Tests: viewers_read_a_fixed_reason_not_the_checker_error, without_a_text_check_viewers_read_that_words_are_off, and a_failing_text_check_degrades_the_coordinator now requires the fixed reason in the receipt, its check and the failures list. Each fails on the old behavior. --- fleet/PROTOCOL.md | 5 +- fleet/core/src/apply.rs | 6 +- fleet/core/tests/vectors/07_words.json | 6 +- fleet/web/core/test/fixtures/coordinator.json | 2 +- src/fleet/coord/text_check.rs | 68 ++++++++++++++++--- tests/test_fleet_coord.rs | 20 +++++- 6 files changed, 89 insertions(+), 18 deletions(-) diff --git a/fleet/PROTOCOL.md b/fleet/PROTOCOL.md index 64ab2a8f5..3460bfebf 100644 --- a/fleet/PROTOCOL.md +++ b/fleet/PROTOCOL.md @@ -179,7 +179,10 @@ Amazon Bedrock whether the text may be shown publicly, read together with every in the sky (so words cannot combine into something unacceptable). The model id and region come from the coordinator's configuration (`--text-check-model`, `--text-check-region`). Only an explicit allow commits the word; a refusal, an error, a timeout (5 s) or a -coordinator started without a text check (`--text-check none`) refuses it. A word checked +coordinator started without a text check (`--text-check none`) refuses it. An error's +reason, in the receipt, its `check` and `actions`, is one of two fixed sentences, "words are +turned off right now" (`--text-check none`) or "couldn't check that word, try again" +(anything else); what went wrong is in the coordinator's log. A word checked against a sky that changed meanwhile is refused (send it again). A check that cannot run marks the coordinator degraded (`failures`) until one runs again. The scene keeps the latest 16 words. A refused word's text appears nowhere: not in the scene, not in diff --git a/fleet/core/src/apply.rs b/fleet/core/src/apply.rs index e2899bcff..87fdc9562 100644 --- a/fleet/core/src/apply.rs +++ b/fleet/core/src/apply.rs @@ -675,9 +675,9 @@ fn intent_effect( word_rate(state, c.client, at)?; match check { None => return Err("the text check did not run".into()), - Some(k) if k.verdict == "error" => { - return Err(format!("the text check failed: {}", k.reason)) - } + // The coordinator words an error's reason for the viewer ("words are + // turned off right now"); the detail is in its log. + Some(k) if k.verdict == "error" => return Err(k.reason.clone()), Some(k) if !k.allowed() => { return Err(format!("the text check refused it: {}", k.reason)) } diff --git a/fleet/core/tests/vectors/07_words.json b/fleet/core/tests/vectors/07_words.json index d784e011b..4dc68cb6c 100644 --- a/fleet/core/tests/vectors/07_words.json +++ b/fleet/core/tests/vectors/07_words.json @@ -71,7 +71,7 @@ }, "at": 9000, "check": { - "reason": "text check unavailable", + "reason": "couldn't check that word, try again", "sky": [ "hello" ], @@ -185,7 +185,7 @@ { "after": 5, "checks": { - "/receipts/w4/reason": "the text check failed: text check unavailable" + "/receipts/w4/reason": "couldn't check that word, try again" } }, { @@ -255,7 +255,7 @@ "id": "w4", "name": "word", "outcome": "refused", - "summary": "the text check failed: text check unavailable", + "summary": "couldn't check that word, try again", "viewer": "v2" }, { diff --git a/fleet/web/core/test/fixtures/coordinator.json b/fleet/web/core/test/fixtures/coordinator.json index a1c1e76de..2bf7c1df6 100644 --- a/fleet/web/core/test/fixtures/coordinator.json +++ b/fleet/web/core/test/fixtures/coordinator.json @@ -1 +1 @@ -{"source":{"pr":1363,"commit":"b0e526d8","dir":"fleet/core/tests/vectors"},"intents":{"known":["claim_camera","claim_gpu","light","nudge","relaunch","release_gpu","restart_now","word"],"unknown":["bogus","camera","set_auto_relaunch","time_of_day"]},"views":{"01_intent_idempotency":{"actions":[{"args":{"tile":"slot-1"},"at":2000,"id":"i-1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"fcvm:9"},"at":2200,"id":"i-2","name":"relaunch","outcome":"refused","summary":"no GPU behind fcvm:9","viewer":"v1"},{"args":{},"at":2300,"id":"i-3","name":"bogus","outcome":"refused","summary":"unknown intent \"bogus\"","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":2000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"02_relaunch_generations":{"actions":[{"args":{"tile":"slot-2"},"at":10000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-2","viewer":"v1"},{"args":{},"at":20000,"id":"ra","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":80000,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":24000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":23000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":24000,"generation":3,"phases":{"container_ms":500,"first_frame_ms":400,"gpu_ms":3000,"restore_ms":900,"teardown_ms":300},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":20000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":20000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":24000,"fcvm_gpus":2,"fcvm_ms":4000,"fcvm_pending":0,"race":1,"started_at":20000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":4000,"last_race":1,"measured_on":2,"median_ms":4000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":20000,"first_frame_at":23000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v2","clicked_at":20000,"first_frame_at":24000,"phases":{"container_ms":500,"first_frame_ms":400,"gpu_ms":3000,"restore_ms":900,"teardown_ms":300}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"03_no_ec2_tiles":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":100,"n":3,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-2","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-3","vm":null}],"viewers":{"active":0,"count":0}},"04_retile_join_leave":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"hC","id":"agg:hC","rect":{"h":0.25,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]},{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"hB","id":"agg:hB","rect":{"h":0.75,"w":1,"x":0,"y":0.25},"tiles":["fcvm:2","fcvm:3","fcvm:4"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[{"host":"hA","kind":"host_reclaimed","retiling":true,"since":3000}],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":4,"gpus":4,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"hB"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"hC"}],"max_tiles":100,"n":4,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":168,"from_t":5.6,"generation":6},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hC:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"hC","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":4100,"first_frame_at":4100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":1,"x":0,"y":0},"rect":{"h":0.25,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20003/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:2","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":1,"x":0,"y":0},"rect":{"h":0.375,"w":1,"x":0,"y":0.25},"slot":"slot-3","vm":"fleet-slot-3"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20004/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:3","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"rect":{"h":0.375,"w":0.5,"x":0,"y":0.625},"slot":"slot-4","vm":"fleet-slot-4"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20005/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:4","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"rect":{"h":0.375,"w":0.5,"x":0.5,"y":0.625},"slot":"slot-5","vm":"fleet-slot-5"}],"viewers":{"active":0,"count":0}},"05_draining":{"actions":[{"args":{},"at":2500,"id":"ra1","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":4500,"id":"r1","name":"relaunch","outcome":"refused","summary":"slot-1 is on a draining host","viewer":"v1"},{"args":{},"at":5000,"id":"ra2","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h2","id":"agg:h2","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":6000,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h1","kind":"host_reclaimed","retiling":true,"since":3000}],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":3,"hosts":[{"arch":"x86_64","draining":true,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h2"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":135,"from_t":4.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":4000,"dropped":2,"last":"slot-3","sitting_out":[],"tiles":{"slot-3":{"capacity_wait_ms":0,"first_frame_at":4000,"generation":2,"phases":{},"tile":"fcvm:3","waiting_since":null}},"time_ms":1500},"id":1,"started_at":2500},{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-3":{"capacity_wait_ms":0,"first_frame_at":null,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":2,"started_at":5000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":5000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":1,"fcvm_ms":null,"fcvm_pending":1,"race":2,"started_at":5000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":1500,"last_race":1,"measured_on":1,"median_ms":1500,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h2:20003/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h2","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":5000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":{"h":0.33333333333333337,"w":1,"x":0,"y":0.6666666666666666},"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-3","vm":"fleet-slot-3"}],"viewers":{"active":0,"count":0}},"06_race_cadence":{"actions":[{"args":{},"at":40000,"id":"rn1","name":"restart_now","outcome":"refused","summary":"restart is cooling down for another 10 s","viewer":"v1"},{"args":{},"at":50000,"id":"rn2","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v1"},{"args":{"secs":10},"at":50500,"id":"sa","name":"set_auto_relaunch","outcome":"refused","summary":"unknown intent \"set_auto_relaunch\"","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":100000,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h1"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":110000,"nudges":{"arrive_next_at":null,"closes_at":85000,"holding":"peak-seam-low-sun","holding_label":"The peak at a low sun","leading":"peak-close","next":null,"next_hold_at":87000,"next_label":null,"opened_at":79999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[{"by":"schedule","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":1,"started_at":30000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":2,"started_at":50000},{"by":"schedule","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":4,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":3,"started_at":80000}],"relaunch_all":{"by":"schedule","ec2_capacity_wait_ms":0,"ec2_done_at":80000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":1,"fcvm_ms":null,"fcvm_pending":1,"race":3,"started_at":80000},"scene":{"camera":[{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"schedule","clicked_at":80000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-1","vm":null}],"viewers":{"active":0,"count":0}},"07_words":{"actions":[{"args":{"text":"hello"},"at":2000,"id":"w1","name":"word","outcome":"committed","summary":"word \"hello\"","viewer":"v1"},{"args":{},"at":3000,"id":"w2","name":"word","outcome":"refused","summary":"one word per 5 s per client","viewer":"v1"},{"args":{},"at":3000,"id":"w3","name":"word","outcome":"refused","summary":"the text check refused it: not suitable","viewer":"v2"},{"args":{},"at":9000,"id":"w4","name":"word","outcome":"refused","summary":"the text check failed: text check unavailable","viewer":"v2"},{"args":{},"at":9100,"id":"w5","name":"word","outcome":"refused","summary":"the text check did not run","viewer":"v2"},{"args":{},"at":9150,"id":"w6","name":"word","outcome":"refused","summary":"text is 29 characters; the limit is 28","viewer":"v2"},{"args":{"text":" two words "},"at":9200,"id":"w7","name":"word","outcome":"committed","summary":"word \"two words\"","viewer":"v2"},{"args":{},"at":9300,"id":"w8","name":"word","outcome":"refused","summary":"the words in the sky changed while this one was checked; send it again","viewer":"v3"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[{"t":2,"text":"hello","text_style":"lockup"},{"t":9.2,"text":"two words","text_style":"lockup"}]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}},"08_max_tiles_and_capacity":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3","fcvm:4"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":8,"gpus":8,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"},{"name":"Tesla T4","sbdf":"0000:04:00.0"},{"name":"Tesla T4","sbdf":"0000:05:00.0"},{"name":"Tesla T4","sbdf":"0000:06:00.0"}],"name":"h1"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h2"}],"max_tiles":4,"n":4,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-5","slot-6","slot-7","slot-8"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-3","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:04:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:4","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-4","vm":null}],"viewers":{"active":0,"count":0}},"09_ec2_half":{"actions":[{"args":{},"at":3000,"id":"ra","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":63000,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h1","kind":"host_reclaimed","retiling":true,"since":20000}],"fleet":{"ec2_instances":3,"ec2_n":3,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":{"h":1,"w":1,"x":0,"y":0},"fcvm":null,"from_frame":645,"from_t":21.5,"generation":2},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":6000,"done_at":10000,"dropped":0,"last":"ec2-3","sitting_out":[],"tiles":{"ec2-1":{"capacity_wait_ms":0,"first_frame_at":9000,"generation":2,"phases":{"capacity_wait_ms":0,"launch_ms":1500,"ready_ms":4500},"tile":"ec2:1","waiting_since":null},"ec2-2":{"capacity_wait_ms":6000,"first_frame_at":15000,"generation":2,"phases":{"capacity_wait_ms":6000,"launch_ms":1200,"ready_ms":4800},"tile":"ec2:2","waiting_since":4000},"ec2-3":{"capacity_wait_ms":0,"first_frame_at":10000,"generation":2,"phases":{"capacity_wait_ms":0,"launch_ms":1600,"ready_ms":5000},"tile":"ec2:3","waiting_since":null}},"time_ms":7000},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":3,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"id":1,"started_at":3000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":6000,"ec2_done_at":10000,"ec2_gpus":3,"ec2_ms":7000,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":3000,"fcvm_gpus":0,"fcvm_ms":null,"fcvm_pending":0,"race":1,"started_at":3000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":{"last_capacity_wait_ms":6000,"last_ms":7000,"last_race":1,"measured_on":3,"median_ms":7000,"races":1},"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-1.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:1","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":9000,"phases":{"capacity_wait_ms":0,"launch_ms":1500,"ready_ms":4500}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0.5,"y":0},"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"ec2-1","vm":"instance-2"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-2.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:2","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":15000,"phases":{"capacity_wait_ms":6000,"launch_ms":1200,"ready_ms":4800}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.25,"x":0.5,"y":0.5},"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"ec2-2","vm":"instance-3"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-3.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:3","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":10000,"phases":{"capacity_wait_ms":0,"launch_ms":1600,"ready_ms":5000}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.25,"x":0.75,"y":0.5},"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"ec2-3","vm":"instance-4"}],"viewers":{"active":0,"count":0}},"10_director":{"actions":[{"args":{"drag":{"pitch_deg":0,"yaw_deg":5}},"at":30010,"id":"c1","name":"camera","outcome":"refused","summary":"unknown intent \"camera\"","viewer":"v1"},{"args":{"v":0.5},"at":30020,"id":"t1","name":"time_of_day","outcome":"refused","summary":"unknown intent \"time_of_day\"","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":35000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"dawn-valley","next":null,"next_hold_at":37000,"next_label":null,"opened_at":5000,"targets":[{"arrive_at":55000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":55000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":55000,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":1,"count":1}},"11_relaunch_rate":{"actions":[{"args":{"tile":"slot-1"},"at":1000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-2"},"at":2000,"id":"r2","name":"relaunch","outcome":"refused","summary":"one relaunch per 2 s per client","viewer":"v2"},{"args":{"tile":"slot-2"},"at":2000,"id":"r3","name":"relaunch","outcome":"committed","summary":"relaunch slot-2","viewer":"v3"},{"args":{"tile":"slot-1"},"at":2300,"id":"r4","name":"relaunch","outcome":"refused","summary":"one relaunch per 2 s per client","viewer":"v4"},{"args":{"tile":"slot-1"},"at":3000,"id":"r5","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v4"},{"args":{"tile":"fcvm:9"},"at":3100,"id":"r6","name":"relaunch","outcome":"refused","summary":"no GPU behind fcvm:9","viewer":"v5"},{"args":{"tile":"slot-1"},"at":3200,"id":"r7","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v5"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v5","clicked_at":3200,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v3","clicked_at":2000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"12_nudges_and_actions":{"actions":[{"args":{"target":"dawn-valley"},"at":1000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v1"},{"args":{"target":"peak-seam-low-sun"},"at":1100,"id":"n2","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v2"},{"args":{"target":"dawn-valley"},"at":1200,"id":"n3","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v3"},{"args":{"target":"nowhere"},"at":1300,"id":"n4","name":"nudge","outcome":"refused","summary":"\"nowhere\" is not offered; the choices are [\"horizon\", \"dawn-valley\", \"peak-seam-low-sun\"]","viewer":"v1"},{"args":{"target":"peak-close"},"at":5100,"id":"n5","name":"nudge","outcome":"committed","summary":"nudge peak-close","viewer":"v2"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":25000,"closes_at":30000,"holding":"coast","holding_label":"From the coast","leading":"peak-close","next":"dawn-valley","next_hold_at":7000,"next_label":"Dawn in the valley","opened_at":5000,"targets":[{"arrive_at":50000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":50000,"id":"peak-close","label":"Close to the peak","votes":1},{"arrive_at":50000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7},{"fov_deg":62,"intent":"n1","look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":25},{"fov_deg":62,"intent":"n1","look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":32}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":3,"count":3}},"13_failures":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h2","id":"agg:h2","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h2"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":195,"from_t":6.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":80000,"closes_at":85000,"holding":null,"holding_label":null,"leading":"peak-close","next":"peak-seam-low-sun","next_hold_at":null,"next_label":"The peak at a low sun","opened_at":65999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":"fcvm exited (exit status: 1)","failure":{"kind":"relaunch_failed","message":"fcvm exited (exit status: 1)","retry_at":6000},"fps":null,"frame_url":"ws://h2:20002/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h2","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":1500,"first_frame_at":1500,"phases":{}},"phase":"failed","place":null,"previous_rect":{"h":0.5,"w":1,"x":0,"y":0.5},"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"14_claim_gpu":{"actions":[{"args":{},"at":3000,"id":"c1","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-3","viewer":"v1"},{"args":{},"at":9500,"id":"c2","name":"claim_gpu","outcome":"refused","summary":"one GPU per client at a time","viewer":"v1"},{"args":{},"at":9600,"id":"c3","name":"claim_gpu","outcome":"refused","summary":"no wall tile can be lent now: every tile is relaunching or racing","viewer":"v2"},{"args":{"drag":{"dive":1,"pitch_deg":0,"yaw_deg":20}},"at":10000,"id":"s1","name":"claim_camera","outcome":"committed","summary":"claim camera","viewer":"v1"},{"args":{"drag":{"pitch_deg":0,"yaw_deg":20}},"at":10100,"id":"s2","name":"claim_camera","outcome":"refused","summary":"claim_camera needs a GPU of your own (claim_gpu)","viewer":"v2"},{"args":{},"at":40000,"id":"c4","name":"claim_gpu","outcome":"refused","summary":"your last GPU was returned 7 s ago; claim again in 113 s","viewer":"v1"},{"args":{},"at":40100,"id":"c5","name":"claim_gpu","outcome":"committed","summary":"claim-2 on slot-3","viewer":"v2"},{"args":{"claim":"claim-9"},"at":40500,"id":"rel","name":"release_gpu","outcome":"refused","summary":"\"claim-9\" is not this viewer's claim","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":2,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-3"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":35000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"dawn-valley","next":null,"next_hold_at":37000,"next_label":null,"opened_at":32999,"targets":[{"arrive_at":55000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":55000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":55000,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"15_scoreboard":{"actions":[{"args":{},"at":2000,"id":"a","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"},{"args":{},"at":10000,"id":"b","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v1"},{"args":{},"at":20000,"id":"c","name":"restart_now","outcome":"committed","summary":"race 3","viewer":"v1"},{"args":{},"at":30000,"id":"d","name":"restart_now","outcome":"committed","summary":"race 4","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":31000,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":6000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":5000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":6000,"generation":2,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":2000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":17000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":12000,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":17000,"generation":3,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":7000},"id":2,"started_at":10000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":25000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":23000,"generation":4,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":25000,"generation":4,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":5000},"id":3,"started_at":20000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":5,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":null,"generation":5,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":null},"id":4,"started_at":30000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":0,"ec2_done_at":30000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":2,"fcvm_ms":null,"fcvm_pending":2,"race":4,"started_at":30000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":5000,"last_race":3,"measured_on":2,"median_ms":5000,"races":3}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":5,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":30000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":5,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v1","clicked_at":30000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"16_lights":{"actions":[{"args":{"color":[5,0,-1],"pos":[1,2,3]},"at":1000,"id":"l1","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[1,1,1],"pos":[2,2,3]},"at":2000,"id":"l2","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0.2,0.2,0.2],"pos":[3,2,3]},"at":3000,"id":"l3","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0,1,0],"pos":[9,2,3]},"at":3500,"id":"l4","name":"light","outcome":"committed","summary":"light","viewer":"v2"},{"args":{"color":[0.1,0.1,0.1],"pos":[4,2,3]},"at":4000,"id":"l5","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0.1,0.1,0.1],"pos":[5,2,3]},"at":4100,"id":"l6","name":"light","outcome":"refused","summary":"one light per 1 s per client","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[{"color":[0.6666666666666666,0.6666666666666666,0.6666666666666666],"pos":[2,2,3],"t":2},{"color":[0.2,0.2,0.2],"pos":[3,2,3],"t":3},{"color":[0,1,0],"pos":[9,2,3],"t":3.5},{"color":[0.1,0.1,0.1],"pos":[4,2,3],"t":4}],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":0}},"17_borrow_a_tile":{"actions":[{"args":{"tile":"slot-3"},"at":2000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-3","viewer":"v2"},{"args":{},"at":3000,"id":"c1","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-2","viewer":"v1"},{"args":{},"at":3200,"id":"c2","name":"claim_gpu","outcome":"refused","summary":"1 of 1 wall tiles are already lent to visitors; try again when one returns","viewer":"v2"},{"args":{},"at":6000,"id":"rn","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":66000,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":100,"n":3,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":80000,"closes_at":85000,"holding":null,"holding_label":null,"leading":"peak-close","next":"peak-seam-low-sun","next_hold_at":null,"next_label":"The peak at a low sun","opened_at":62999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":9000,"dropped":0,"last":"slot-3","sitting_out":["fcvm:2"],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":8000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-3":{"capacity_wait_ms":0,"first_frame_at":9000,"generation":3,"phases":{},"tile":"fcvm:3","waiting_since":null}},"time_ms":3000},"id":1,"started_at":6000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":6000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":9000,"fcvm_gpus":2,"fcvm_ms":3000,"fcvm_pending":0,"race":1,"started_at":6000},"scene":{"camera":[{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":3000,"last_race":1,"measured_on":2,"median_ms":3000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":6000,"first_frame_at":8000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"return:claim-1","clicked_at":63000,"first_frame_at":66000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-2","vm":"fleet-slot-2"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20003/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":{"by":"viewer:v2","clicked_at":6000,"first_frame_at":9000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-3","vm":"fleet-slot-3"}],"viewers":{"active":2,"count":2}},"18_counters":{"actions":[{"args":{},"at":3000,"id":"rn","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":63000,"counters":{"restores":4,"restores_last_hour":0,"scrub_failures":1,"scrubs":2},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":3604999,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":3606999,"next_label":null,"opened_at":0,"targets":[{"arrive_at":3629999,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":3624999,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":3620999,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":7000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":6000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":7000,"generation":2,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":3000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":0,"ec2_done_at":3000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":7000,"fcvm_gpus":2,"fcvm_ms":4000,"fcvm_pending":0,"race":1,"started_at":3000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":3599.999},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":3606.999}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":4000,"last_race":1,"measured_on":2,"median_ms":4000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":6000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":7000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"19_standby_and_quarantine":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":2,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-3"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":210,"from_t":7,"generation":6},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":5500,"first_frame_at":5500,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":1,"x":0,"y":0},"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":{"h":1,"w":0.5,"x":0,"y":0},"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":0,"count":0}},"20_power":{"actions":[{"args":{},"at":64000,"id":"rn","name":"restart_now","outcome":"refused","summary":"the GPUs are asleep","viewer":"v1"},{"args":{},"at":500200,"id":"c","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-2","viewer":"v2"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":0,"power":{"asleep_since":620200,"hosts":[],"idle_since":500300,"last_wake":{"awake_at":481000,"gpus":4,"ms":81000,"started_at":400000},"state":"asleep","wakes":[{"awake_at":481000,"gpus":4,"ms":81000,"started_at":400000}],"woke_at":null},"standby":["slot-1","slot-2"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":18651,"from_t":621.7,"generation":5},"next_race_at":null,"nudges":{"arrive_next_at":635300,"closes_at":640300,"holding":null,"holding_label":null,"leading":"dawn-valley","next":"horizon","next_hold_at":null,"next_label":"High above: the curve of the horizon","opened_at":620200,"targets":[{"arrive_at":660300,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":660300,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":660300,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":605.3},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":612.3},{"fov_deg":65,"look":[-75,28,-170],"pos":[-40,20,-80],"t":623.8},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":635.3},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":642.3}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":1}},"21_agent_reconnect":{"actions":[{"args":{"tile":"slot-1"},"at":2000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":5000,"id":"r2","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":9000,"id":"r3","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h1"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":285,"from_t":9.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":9000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"}],"viewers":{"active":0,"count":0}},"22_layout_generations":{"actions":[],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":70000,"hosts":[],"idle_since":9000,"last_wake":null,"state":"asleep","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":2145,"from_t":71.5,"generation":5},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":75000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":77000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":100000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":95000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":91000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":70},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":77}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":1}},"23_grid_budget":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":0.6666666666666666,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":707107,"budget_from_t":20,"claims":[],"cooldown_until":null,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h2","kind":"host_reclaimed","retiling":true,"since":30500}],"fleet":{"ec2_instances":1,"ec2_n":1,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":{"h":1,"w":0.33333333333333337,"x":0.6666666666666666,"y":0},"fcvm":{"h":1,"w":0.6666666666666666,"x":0,"y":0},"from_frame":960,"from_t":32,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":55000,"closes_at":60000,"holding":null,"holding_label":null,"leading":"peak-seam-low-sun","next":"dawn-valley","next_hold_at":null,"next_label":"Dawn in the valley","opened_at":36000,"targets":[{"arrive_at":80000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":80000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":80000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.265,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":1100,"first_frame_at":1100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.6666666666666666,"w":0.375,"x":0,"y":0},"rect":{"h":0.5,"w":0.6666666666666666,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"wake","clicked_at":1200,"first_frame_at":1200,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.6666666666666666,"w":0.375,"x":0.375,"y":0},"rect":{"h":0.5,"w":0.6666666666666666,"x":0,"y":0.5},"slot":"slot-2","vm":"fleet-slot-2"},{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"wss://ec2-1.example/frames","frame_url_v4":null,"generation":1,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:1","last_relaunch":{"by":"wake","clicked_at":1300,"first_frame_at":1300,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":0.25,"x":0.75,"y":0},"rect":{"h":1,"w":0.33333333333333337,"x":0.6666666666666666,"y":0},"slot":"ec2-1","vm":"i-1"}],"viewers":{"active":0,"count":0}},"24_text_style":{"actions":[{"args":{"text":"one"},"at":2000,"id":"s1","name":"word","outcome":"committed","summary":"word \"one\"","viewer":"v1"},{"args":{"text":"two","text_style":"skywriting"},"at":2100,"id":"s2","name":"word","outcome":"committed","summary":"word \"two\"","viewer":"v2"},{"args":{"text":"three","text_style":"landmark"},"at":2200,"id":"s3","name":"word","outcome":"committed","summary":"word \"three\"","viewer":"v3"},{"args":{"text":"four","text_style":"lockup"},"at":2300,"id":"s4","name":"word","outcome":"committed","summary":"word \"four\"","viewer":"v4"},{"args":{},"at":2400,"id":"s5","name":"word","outcome":"refused","summary":"args.text_style must be one of lockup, skywriting, landmark","viewer":"v5"},{"args":{},"at":2500,"id":"s6","name":"word","outcome":"refused","summary":"args.text_style must be one of lockup, skywriting, landmark","viewer":"v5"},{"args":{},"at":2600,"id":"s7","name":"word","outcome":"refused","summary":"one word per 5 s per client","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[{"t":2,"text":"one","text_style":"lockup"},{"t":2.1,"text":"two","text_style":"skywriting"},{"t":2.2,"text":"three","text_style":"landmark"},{"t":2.3,"text":"four","text_style":"lockup"}]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":5,"count":5}},"25_stop_choice_after_decision":{"actions":[{"args":{"target":"horizon"},"at":5500,"id":"n0","name":"nudge","outcome":"committed","summary":"nudge horizon","viewer":"v2"},{"args":{"target":"peak-close"},"at":6000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge peak-close","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":55000,"closes_at":60000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"peak-seam-low-sun","next":"peak-close","next_hold_at":37000,"next_label":"Close to the peak","opened_at":35000,"targets":[{"arrive_at":70000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":80000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":70000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37},{"fov_deg":50,"intent":"n1","look":[-56,2.9,-114.75],"pos":[-62,3.5,-105],"t":55},{"fov_deg":50,"intent":"n1","look":[-56,2.9,-114.75],"pos":[-62,3.5,-105],"t":62}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}},"26_stop_choice_fairness":{"actions":[{"args":{"target":"peak-seam-low-sun"},"at":1000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v1"},{"args":{"target":"dawn-valley"},"at":2000,"id":"n2","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v2"},{"args":{"target":"peak-seam-low-sun"},"at":3000,"id":"n3","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":21000,"closes_at":26000,"holding":"coast","holding_label":"From the coast","leading":"dawn-valley","next":"peak-seam-low-sun","next_hold_at":7000,"next_label":"The peak at a low sun","opened_at":5000,"targets":[{"arrive_at":46000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":36000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":46000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7},{"fov_deg":55,"intent":"n1","look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":21},{"fov_deg":55,"intent":"n1","look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":28}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}}}} +{"source":{"pr":1363,"commit":"b0e526d8","dir":"fleet/core/tests/vectors"},"intents":{"known":["claim_camera","claim_gpu","light","nudge","relaunch","release_gpu","restart_now","word"],"unknown":["bogus","camera","set_auto_relaunch","time_of_day"]},"views":{"01_intent_idempotency":{"actions":[{"args":{"tile":"slot-1"},"at":2000,"id":"i-1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"fcvm:9"},"at":2200,"id":"i-2","name":"relaunch","outcome":"refused","summary":"no GPU behind fcvm:9","viewer":"v1"},{"args":{},"at":2300,"id":"i-3","name":"bogus","outcome":"refused","summary":"unknown intent \"bogus\"","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":2000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"02_relaunch_generations":{"actions":[{"args":{"tile":"slot-2"},"at":10000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-2","viewer":"v1"},{"args":{},"at":20000,"id":"ra","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":80000,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":24000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":23000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":24000,"generation":3,"phases":{"container_ms":500,"first_frame_ms":400,"gpu_ms":3000,"restore_ms":900,"teardown_ms":300},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":20000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":20000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":24000,"fcvm_gpus":2,"fcvm_ms":4000,"fcvm_pending":0,"race":1,"started_at":20000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":4000,"last_race":1,"measured_on":2,"median_ms":4000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":20000,"first_frame_at":23000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v2","clicked_at":20000,"first_frame_at":24000,"phases":{"container_ms":500,"first_frame_ms":400,"gpu_ms":3000,"restore_ms":900,"teardown_ms":300}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"03_no_ec2_tiles":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":100,"n":3,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-2","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-3","vm":null}],"viewers":{"active":0,"count":0}},"04_retile_join_leave":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"hC","id":"agg:hC","rect":{"h":0.25,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]},{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"hB","id":"agg:hB","rect":{"h":0.75,"w":1,"x":0,"y":0.25},"tiles":["fcvm:2","fcvm:3","fcvm:4"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[{"host":"hA","kind":"host_reclaimed","retiling":true,"since":3000}],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":4,"gpus":4,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"hB"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"hC"}],"max_tiles":100,"n":4,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":168,"from_t":5.6,"generation":6},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hC:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"hC","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":4100,"first_frame_at":4100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":1,"x":0,"y":0},"rect":{"h":0.25,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20003/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:2","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":1,"x":0,"y":0},"rect":{"h":0.375,"w":1,"x":0,"y":0.25},"slot":"slot-3","vm":"fleet-slot-3"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20004/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:3","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"rect":{"h":0.375,"w":0.5,"x":0,"y":0.625},"slot":"slot-4","vm":"fleet-slot-4"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://hB:20005/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"hB","id":"fcvm:4","last_relaunch":{"by":"wake","clicked_at":2100,"first_frame_at":2100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"rect":{"h":0.375,"w":0.5,"x":0.5,"y":0.625},"slot":"slot-5","vm":"fleet-slot-5"}],"viewers":{"active":0,"count":0}},"05_draining":{"actions":[{"args":{},"at":2500,"id":"ra1","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":4500,"id":"r1","name":"relaunch","outcome":"refused","summary":"slot-1 is on a draining host","viewer":"v1"},{"args":{},"at":5000,"id":"ra2","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h2","id":"agg:h2","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":6000,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h1","kind":"host_reclaimed","retiling":true,"since":3000}],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":3,"hosts":[{"arch":"x86_64","draining":true,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h2"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":135,"from_t":4.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":4000,"dropped":2,"last":"slot-3","sitting_out":[],"tiles":{"slot-3":{"capacity_wait_ms":0,"first_frame_at":4000,"generation":2,"phases":{},"tile":"fcvm:3","waiting_since":null}},"time_ms":1500},"id":1,"started_at":2500},{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-3":{"capacity_wait_ms":0,"first_frame_at":null,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":2,"started_at":5000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":5000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":1,"fcvm_ms":null,"fcvm_pending":1,"race":2,"started_at":5000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":1500,"last_race":1,"measured_on":1,"median_ms":1500,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h2:20003/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h2","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":5000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":{"h":0.33333333333333337,"w":1,"x":0,"y":0.6666666666666666},"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-3","vm":"fleet-slot-3"}],"viewers":{"active":0,"count":0}},"06_race_cadence":{"actions":[{"args":{},"at":40000,"id":"rn1","name":"restart_now","outcome":"refused","summary":"restart is cooling down for another 10 s","viewer":"v1"},{"args":{},"at":50000,"id":"rn2","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v1"},{"args":{"secs":10},"at":50500,"id":"sa","name":"set_auto_relaunch","outcome":"refused","summary":"unknown intent \"set_auto_relaunch\"","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":100000,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h1"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":110000,"nudges":{"arrive_next_at":null,"closes_at":85000,"holding":"peak-seam-low-sun","holding_label":"The peak at a low sun","leading":"peak-close","next":null,"next_hold_at":87000,"next_label":null,"opened_at":79999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[{"by":"schedule","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":1,"started_at":30000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":2,"started_at":50000},{"by":"schedule","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":4,"phases":{},"tile":"fcvm:1","waiting_since":null}},"time_ms":null},"id":3,"started_at":80000}],"relaunch_all":{"by":"schedule","ec2_capacity_wait_ms":0,"ec2_done_at":80000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":1,"fcvm_ms":null,"fcvm_pending":1,"race":3,"started_at":80000},"scene":{"camera":[{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"schedule","clicked_at":80000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-1","vm":null}],"viewers":{"active":0,"count":0}},"07_words":{"actions":[{"args":{"text":"hello"},"at":2000,"id":"w1","name":"word","outcome":"committed","summary":"word \"hello\"","viewer":"v1"},{"args":{},"at":3000,"id":"w2","name":"word","outcome":"refused","summary":"one word per 5 s per client","viewer":"v1"},{"args":{},"at":3000,"id":"w3","name":"word","outcome":"refused","summary":"the text check refused it: not suitable","viewer":"v2"},{"args":{},"at":9000,"id":"w4","name":"word","outcome":"refused","summary":"couldn't check that word, try again","viewer":"v2"},{"args":{},"at":9100,"id":"w5","name":"word","outcome":"refused","summary":"the text check did not run","viewer":"v2"},{"args":{},"at":9150,"id":"w6","name":"word","outcome":"refused","summary":"text is 29 characters; the limit is 28","viewer":"v2"},{"args":{"text":" two words "},"at":9200,"id":"w7","name":"word","outcome":"committed","summary":"word \"two words\"","viewer":"v2"},{"args":{},"at":9300,"id":"w8","name":"word","outcome":"refused","summary":"the words in the sky changed while this one was checked; send it again","viewer":"v3"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[{"t":2,"text":"hello","text_style":"lockup"},{"t":9.2,"text":"two words","text_style":"lockup"}]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}},"08_max_tiles_and_capacity":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3","fcvm:4"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":8,"gpus":8,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"},{"name":"Tesla T4","sbdf":"0000:04:00.0"},{"name":"Tesla T4","sbdf":"0000:05:00.0"},{"name":"Tesla T4","sbdf":"0000:06:00.0"}],"name":"h1"},{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h2"}],"max_tiles":4,"n":4,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-5","slot-6","slot-7","slot-8"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-3","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:04:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:4","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-4","vm":null}],"viewers":{"active":0,"count":0}},"09_ec2_half":{"actions":[{"args":{},"at":3000,"id":"ra","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":63000,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h1","kind":"host_reclaimed","retiling":true,"since":20000}],"fleet":{"ec2_instances":3,"ec2_n":3,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":{"h":1,"w":1,"x":0,"y":0},"fcvm":null,"from_frame":645,"from_t":21.5,"generation":2},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":6000,"done_at":10000,"dropped":0,"last":"ec2-3","sitting_out":[],"tiles":{"ec2-1":{"capacity_wait_ms":0,"first_frame_at":9000,"generation":2,"phases":{"capacity_wait_ms":0,"launch_ms":1500,"ready_ms":4500},"tile":"ec2:1","waiting_since":null},"ec2-2":{"capacity_wait_ms":6000,"first_frame_at":15000,"generation":2,"phases":{"capacity_wait_ms":6000,"launch_ms":1200,"ready_ms":4800},"tile":"ec2:2","waiting_since":4000},"ec2-3":{"capacity_wait_ms":0,"first_frame_at":10000,"generation":2,"phases":{"capacity_wait_ms":0,"launch_ms":1600,"ready_ms":5000},"tile":"ec2:3","waiting_since":null}},"time_ms":7000},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":3,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"id":1,"started_at":3000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":6000,"ec2_done_at":10000,"ec2_gpus":3,"ec2_ms":7000,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":3000,"fcvm_gpus":0,"fcvm_ms":null,"fcvm_pending":0,"race":1,"started_at":3000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":{"last_capacity_wait_ms":6000,"last_ms":7000,"last_race":1,"measured_on":3,"median_ms":7000,"races":1},"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-1.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:1","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":9000,"phases":{"capacity_wait_ms":0,"launch_ms":1500,"ready_ms":4500}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.5,"x":0.5,"y":0},"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"ec2-1","vm":"instance-2"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-2.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:2","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":15000,"phases":{"capacity_wait_ms":6000,"launch_ms":1200,"ready_ms":4800}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.25,"x":0.5,"y":0.5},"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"ec2-2","vm":"instance-3"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://ec2-3.example:443/frames","frame_url_v4":null,"generation":2,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:3","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":10000,"phases":{"capacity_wait_ms":0,"launch_ms":1600,"ready_ms":5000}},"phase":"running","place":null,"previous_rect":{"h":0.5,"w":0.25,"x":0.75,"y":0.5},"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"ec2-3","vm":"instance-4"}],"viewers":{"active":0,"count":0}},"10_director":{"actions":[{"args":{"drag":{"pitch_deg":0,"yaw_deg":5}},"at":30010,"id":"c1","name":"camera","outcome":"refused","summary":"unknown intent \"camera\"","viewer":"v1"},{"args":{"v":0.5},"at":30020,"id":"t1","name":"time_of_day","outcome":"refused","summary":"unknown intent \"time_of_day\"","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":35000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"dawn-valley","next":null,"next_hold_at":37000,"next_label":null,"opened_at":5000,"targets":[{"arrive_at":55000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":55000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":55000,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":1,"count":1}},"11_relaunch_rate":{"actions":[{"args":{"tile":"slot-1"},"at":1000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-2"},"at":2000,"id":"r2","name":"relaunch","outcome":"refused","summary":"one relaunch per 2 s per client","viewer":"v2"},{"args":{"tile":"slot-2"},"at":2000,"id":"r3","name":"relaunch","outcome":"committed","summary":"relaunch slot-2","viewer":"v3"},{"args":{"tile":"slot-1"},"at":2300,"id":"r4","name":"relaunch","outcome":"refused","summary":"one relaunch per 2 s per client","viewer":"v4"},{"args":{"tile":"slot-1"},"at":3000,"id":"r5","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v4"},{"args":{"tile":"fcvm:9"},"at":3100,"id":"r6","name":"relaunch","outcome":"refused","summary":"no GPU behind fcvm:9","viewer":"v5"},{"args":{"tile":"slot-1"},"at":3200,"id":"r7","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v5"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v5","clicked_at":3200,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v3","clicked_at":2000,"first_frame_at":null,"phases":{}},"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"12_nudges_and_actions":{"actions":[{"args":{"target":"dawn-valley"},"at":1000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v1"},{"args":{"target":"peak-seam-low-sun"},"at":1100,"id":"n2","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v2"},{"args":{"target":"dawn-valley"},"at":1200,"id":"n3","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v3"},{"args":{"target":"nowhere"},"at":1300,"id":"n4","name":"nudge","outcome":"refused","summary":"\"nowhere\" is not offered; the choices are [\"horizon\", \"dawn-valley\", \"peak-seam-low-sun\"]","viewer":"v1"},{"args":{"target":"peak-close"},"at":5100,"id":"n5","name":"nudge","outcome":"committed","summary":"nudge peak-close","viewer":"v2"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":25000,"closes_at":30000,"holding":"coast","holding_label":"From the coast","leading":"peak-close","next":"dawn-valley","next_hold_at":7000,"next_label":"Dawn in the valley","opened_at":5000,"targets":[{"arrive_at":50000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":50000,"id":"peak-close","label":"Close to the peak","votes":1},{"arrive_at":50000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7},{"fov_deg":62,"intent":"n1","look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":25},{"fov_deg":62,"intent":"n1","look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":32}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":3,"count":3}},"13_failures":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h2","id":"agg:h2","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h2"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":195,"from_t":6.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":80000,"closes_at":85000,"holding":null,"holding_label":null,"leading":"peak-close","next":"peak-seam-low-sun","next_hold_at":null,"next_label":"The peak at a low sun","opened_at":65999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":"fcvm exited (exit status: 1)","failure":{"kind":"relaunch_failed","message":"fcvm exited (exit status: 1)","retry_at":6000},"fps":null,"frame_url":"ws://h2:20002/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h2","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":1500,"first_frame_at":1500,"phases":{}},"phase":"failed","place":null,"previous_rect":{"h":0.5,"w":1,"x":0,"y":0.5},"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"14_claim_gpu":{"actions":[{"args":{},"at":3000,"id":"c1","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-3","viewer":"v1"},{"args":{},"at":9500,"id":"c2","name":"claim_gpu","outcome":"refused","summary":"one GPU per client at a time","viewer":"v1"},{"args":{},"at":9600,"id":"c3","name":"claim_gpu","outcome":"refused","summary":"no wall tile can be lent now: every tile is relaunching or racing","viewer":"v2"},{"args":{"drag":{"dive":1,"pitch_deg":0,"yaw_deg":20}},"at":10000,"id":"s1","name":"claim_camera","outcome":"committed","summary":"claim camera","viewer":"v1"},{"args":{"drag":{"pitch_deg":0,"yaw_deg":20}},"at":10100,"id":"s2","name":"claim_camera","outcome":"refused","summary":"claim_camera needs a GPU of your own (claim_gpu)","viewer":"v2"},{"args":{},"at":40000,"id":"c4","name":"claim_gpu","outcome":"refused","summary":"your last GPU was returned 7 s ago; claim again in 113 s","viewer":"v1"},{"args":{},"at":40100,"id":"c5","name":"claim_gpu","outcome":"committed","summary":"claim-2 on slot-3","viewer":"v2"},{"args":{"claim":"claim-9"},"at":40500,"id":"rel","name":"release_gpu","outcome":"refused","summary":"\"claim-9\" is not this viewer's claim","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":2,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-3"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":35000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"dawn-valley","next":null,"next_hold_at":37000,"next_label":null,"opened_at":32999,"targets":[{"arrive_at":55000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":55000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":55000,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":null},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":1,"count":1}},"15_scoreboard":{"actions":[{"args":{},"at":2000,"id":"a","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"},{"args":{},"at":10000,"id":"b","name":"restart_now","outcome":"committed","summary":"race 2","viewer":"v1"},{"args":{},"at":20000,"id":"c","name":"restart_now","outcome":"committed","summary":"race 3","viewer":"v1"},{"args":{},"at":30000,"id":"d","name":"restart_now","outcome":"committed","summary":"race 4","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":31000,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":6000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":5000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":6000,"generation":2,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":2000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":17000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":12000,"generation":3,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":17000,"generation":3,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":7000},"id":2,"started_at":10000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":25000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":23000,"generation":4,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":25000,"generation":4,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":5000},"id":3,"started_at":20000},{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":null,"generation":5,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":null,"generation":5,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":null},"id":4,"started_at":30000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":0,"ec2_done_at":30000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":null,"fcvm_gpus":2,"fcvm_ms":null,"fcvm_pending":2,"race":4,"started_at":30000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":5000,"last_race":3,"measured_on":2,"median_ms":5000,"races":3}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":5,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":30000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":5,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v1","clicked_at":30000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"16_lights":{"actions":[{"args":{"color":[5,0,-1],"pos":[1,2,3]},"at":1000,"id":"l1","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[1,1,1],"pos":[2,2,3]},"at":2000,"id":"l2","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0.2,0.2,0.2],"pos":[3,2,3]},"at":3000,"id":"l3","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0,1,0],"pos":[9,2,3]},"at":3500,"id":"l4","name":"light","outcome":"committed","summary":"light","viewer":"v2"},{"args":{"color":[0.1,0.1,0.1],"pos":[4,2,3]},"at":4000,"id":"l5","name":"light","outcome":"committed","summary":"light","viewer":"v1"},{"args":{"color":[0.1,0.1,0.1],"pos":[5,2,3]},"at":4100,"id":"l6","name":"light","outcome":"refused","summary":"one light per 1 s per client","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[{"color":[0.6666666666666666,0.6666666666666666,0.6666666666666666],"pos":[2,2,3],"t":2},{"color":[0.2,0.2,0.2],"pos":[3,2,3],"t":3},{"color":[0,1,0],"pos":[9,2,3],"t":3.5},{"color":[0.1,0.1,0.1],"pos":[4,2,3],"t":4}],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":0}},"17_borrow_a_tile":{"actions":[{"args":{"tile":"slot-3"},"at":2000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-3","viewer":"v2"},{"args":{},"at":3000,"id":"c1","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-2","viewer":"v1"},{"args":{},"at":3200,"id":"c2","name":"claim_gpu","outcome":"refused","summary":"1 of 1 wall tiles are already lent to visitors; try again when one returns","viewer":"v2"},{"args":{},"at":6000,"id":"rn","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v2"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2","fcvm:3"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":66000,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":100,"n":3,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":80000,"closes_at":85000,"holding":null,"holding_label":null,"leading":"peak-close","next":"peak-seam-low-sun","next_hold_at":null,"next_label":"The peak at a low sun","opened_at":62999,"targets":[{"arrive_at":95000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":105000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":95000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[{"by":"viewer:v2","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":9000,"dropped":0,"last":"slot-3","sitting_out":["fcvm:2"],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":8000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-3":{"capacity_wait_ms":0,"first_frame_at":9000,"generation":3,"phases":{},"tile":"fcvm:3","waiting_since":null}},"time_ms":3000},"id":1,"started_at":6000}],"relaunch_all":{"by":"viewer:v2","ec2_capacity_wait_ms":0,"ec2_done_at":6000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":9000,"fcvm_gpus":2,"fcvm_ms":3000,"fcvm_pending":0,"race":1,"started_at":6000},"scene":{"camera":[{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":80},{"fov_deg":55,"look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":87}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.255,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":3000,"last_race":1,"measured_on":2,"median_ms":3000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v2","clicked_at":6000,"first_frame_at":8000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"return:claim-1","clicked_at":63000,"first_frame_at":66000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0,"y":0.5},"slot":"slot-2","vm":"fleet-slot-2"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20003/frames","frame_url_v4":null,"generation":3,"gpu":{"name":"Tesla T4","sbdf":"0000:03:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:3","last_relaunch":{"by":"viewer:v2","clicked_at":6000,"first_frame_at":9000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":0.5,"w":0.5,"x":0.5,"y":0.5},"slot":"slot-3","vm":"fleet-slot-3"}],"viewers":{"active":2,"count":2}},"18_counters":{"actions":[{"args":{},"at":3000,"id":"rn","name":"restart_now","outcome":"committed","summary":"race 1","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":63000,"counters":{"restores":4,"restores_last_hour":0,"scrub_failures":1,"scrubs":2},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":75,"from_t":2.5,"generation":1},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":3604999,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":3606999,"next_label":null,"opened_at":0,"targets":[{"arrive_at":3629999,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":3624999,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":3620999,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[{"by":"viewer:v1","ec2":{"capacity_wait_ms":0,"done_at":null,"dropped":0,"last":null,"sitting_out":[],"tiles":{},"time_ms":null},"fcvm":{"capacity_wait_ms":0,"done_at":7000,"dropped":0,"last":"slot-2","sitting_out":[],"tiles":{"slot-1":{"capacity_wait_ms":0,"first_frame_at":6000,"generation":2,"phases":{},"tile":"fcvm:1","waiting_since":null},"slot-2":{"capacity_wait_ms":0,"first_frame_at":7000,"generation":2,"phases":{},"tile":"fcvm:2","waiting_since":null}},"time_ms":4000},"id":1,"started_at":3000}],"relaunch_all":{"by":"viewer:v1","ec2_capacity_wait_ms":0,"ec2_done_at":3000,"ec2_gpus":0,"ec2_ms":null,"ec2_pending":0,"ec2_waiting_since":null,"fcvm_done_at":7000,"fcvm_gpus":2,"fcvm_ms":4000,"fcvm_pending":0,"race":1,"started_at":3000},"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":3599.999},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":3606.999}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":{"last_capacity_wait_ms":0,"last_ms":4000,"last_race":1,"measured_on":2,"median_ms":4000,"races":1}},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":6000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":2,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"viewer:v1","clicked_at":3000,"first_frame_at":7000,"phases":{}},"phase":"running","place":null,"previous_rect":null,"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":"fleet-slot-2"}],"viewers":{"active":0,"count":0}},"19_standby_and_quarantine":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":3,"gpus":3,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"},{"name":"Tesla T4","sbdf":"0000:03:00.0"}],"name":"h1"}],"max_tiles":2,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":["slot-3"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":210,"from_t":7,"generation":6},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":5500,"first_frame_at":5500,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":1,"x":0,"y":0},"rect":{"h":1,"w":0.5,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":null,"frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":null,"phase":"pending","place":null,"previous_rect":{"h":1,"w":0.5,"x":0,"y":0},"rect":{"h":1,"w":0.5,"x":0.5,"y":0},"slot":"slot-2","vm":null}],"viewers":{"active":0,"count":0}},"20_power":{"actions":[{"args":{},"at":64000,"id":"rn","name":"restart_now","outcome":"refused","summary":"the GPUs are asleep","viewer":"v1"},{"args":{},"at":500200,"id":"c","name":"claim_gpu","outcome":"committed","summary":"claim-1 on slot-2","viewer":"v2"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":4,"restores_last_hour":4,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":0,"power":{"asleep_since":620200,"hosts":[],"idle_since":500300,"last_wake":{"awake_at":481000,"gpus":4,"ms":81000,"started_at":400000},"state":"asleep","wakes":[{"awake_at":481000,"gpus":4,"ms":81000,"started_at":400000}],"woke_at":null},"standby":["slot-1","slot-2"],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":18651,"from_t":621.7,"generation":5},"next_race_at":null,"nudges":{"arrive_next_at":635300,"closes_at":640300,"holding":null,"holding_label":null,"leading":"dawn-valley","next":"horizon","next_hold_at":null,"next_label":"High above: the curve of the horizon","opened_at":620200,"targets":[{"arrive_at":660300,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":660300,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":660300,"id":"peak-close","label":"Close to the peak","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":605.3},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":612.3},{"fov_deg":65,"look":[-75,28,-170],"pos":[-40,20,-80],"t":623.8},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":635.3},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":642.3}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":1}},"21_agent_reconnect":{"actions":[{"args":{"tile":"slot-1"},"at":2000,"id":"r1","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":5000,"id":"r2","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"},{"args":{"tile":"slot-1"},"at":9000,"id":"r3","name":"relaunch","outcome":"committed","summary":"relaunch slot-1","viewer":"v1"}],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":1,"x":0,"y":0},"tiles":["fcvm:1"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":2,"restores_last_hour":2,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":1,"gpus":1,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"}],"name":"h1"}],"max_tiles":100,"n":1,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":{"h":1,"w":1,"x":0,"y":0},"from_frame":285,"from_t":9.5,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":null,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":4,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"viewer:v1","clicked_at":9000,"first_frame_at":null,"phases":{}},"phase":"relaunching","place":null,"previous_rect":null,"rect":{"h":1,"w":1,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"}],"viewers":{"active":0,"count":0}},"22_layout_generations":{"actions":[],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":1,"restores_last_hour":1,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":70000,"hosts":[],"idle_since":9000,"last_wake":null,"state":"asleep","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":2145,"from_t":71.5,"generation":5},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":75000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":77000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":100000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":95000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":91000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":70},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":77}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":0,"count":1}},"23_grid_budget":{"actions":[],"aggregates":[{"frame_url":null,"frame_url_v4":null,"half":"fcvm","host":"h1","id":"agg:h1","rect":{"h":1,"w":0.6666666666666666,"x":0,"y":0},"tiles":["fcvm:1","fcvm:2"]}],"borrow":{"cap":1,"lent":0,"next_return_at":null},"budget":707107,"budget_from_t":20,"claims":[],"cooldown_until":null,"counters":{"restores":3,"restores_last_hour":3,"scrub_failures":0,"scrubs":0},"failures":[{"host":"h2","kind":"host_reclaimed","retiling":true,"since":30500}],"fleet":{"ec2_instances":1,"ec2_n":1,"ec2_waiting":0,"fcvm_gpus":2,"gpus":2,"hosts":[{"arch":"x86_64","draining":false,"gpus":[{"name":"Tesla T4","sbdf":"0000:01:00.0"},{"name":"Tesla T4","sbdf":"0000:02:00.0"}],"name":"h1"}],"max_tiles":100,"n":2,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":{"h":1,"w":0.33333333333333337,"x":0.6666666666666666,"y":0},"fcvm":{"h":1,"w":0.6666666666666666,"x":0,"y":0},"from_frame":960,"from_t":32,"generation":3},"next_race_at":null,"nudges":{"arrive_next_at":55000,"closes_at":60000,"holding":null,"holding_label":null,"leading":"peak-seam-low-sun","next":"dawn-valley","next_hold_at":null,"next_label":"Dawn in the valley","opened_at":36000,"targets":[{"arrive_at":80000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":80000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":80000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":55},{"fov_deg":62,"look":[-10.75,2.6,-105.5],"pos":[-18,1.8,-92],"t":62}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.265,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"ws://h1:20001/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:01:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:1","last_relaunch":{"by":"wake","clicked_at":1100,"first_frame_at":1100,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.6666666666666666,"w":0.375,"x":0,"y":0},"rect":{"h":0.5,"w":0.6666666666666666,"x":0,"y":0},"slot":"slot-1","vm":"fleet-slot-1"},{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"ws://h1:20002/frames","frame_url_v4":null,"generation":1,"gpu":{"name":"Tesla T4","sbdf":"0000:02:00.0"},"gpu_stats":null,"half":"fcvm","host":"h1","id":"fcvm:2","last_relaunch":{"by":"wake","clicked_at":1200,"first_frame_at":1200,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":0.6666666666666666,"w":0.375,"x":0.375,"y":0},"rect":{"h":0.5,"w":0.6666666666666666,"x":0,"y":0.5},"slot":"slot-2","vm":"fleet-slot-2"},{"borrowed":null,"error":null,"failure":null,"fps":30,"frame_url":"wss://ec2-1.example/frames","frame_url_v4":null,"generation":1,"gpu":null,"gpu_stats":null,"half":"ec2","host":null,"id":"ec2:1","last_relaunch":{"by":"wake","clicked_at":1300,"first_frame_at":1300,"phases":{}},"phase":"running","place":null,"previous_rect":{"h":1,"w":0.25,"x":0.75,"y":0},"rect":{"h":1,"w":0.33333333333333337,"x":0.6666666666666666,"y":0},"slot":"ec2-1","vm":"i-1"}],"viewers":{"active":0,"count":0}},"24_text_style":{"actions":[{"args":{"text":"one"},"at":2000,"id":"s1","name":"word","outcome":"committed","summary":"word \"one\"","viewer":"v1"},{"args":{"text":"two","text_style":"skywriting"},"at":2100,"id":"s2","name":"word","outcome":"committed","summary":"word \"two\"","viewer":"v2"},{"args":{"text":"three","text_style":"landmark"},"at":2200,"id":"s3","name":"word","outcome":"committed","summary":"word \"three\"","viewer":"v3"},{"args":{"text":"four","text_style":"lockup"},"at":2300,"id":"s4","name":"word","outcome":"committed","summary":"word \"four\"","viewer":"v4"},{"args":{},"at":2400,"id":"s5","name":"word","outcome":"refused","summary":"args.text_style must be one of lockup, skywriting, landmark","viewer":"v5"},{"args":{},"at":2500,"id":"s6","name":"word","outcome":"refused","summary":"args.text_style must be one of lockup, skywriting, landmark","viewer":"v5"},{"args":{},"at":2600,"id":"s7","name":"word","outcome":"refused","summary":"one word per 5 s per client","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":null,"closes_at":5000,"holding":"coast","holding_label":"From the coast","leading":"horizon","next":null,"next_hold_at":7000,"next_label":null,"opened_at":0,"targets":[{"arrive_at":30000,"id":"horizon","label":"High above: the curve of the horizon","votes":0},{"arrive_at":25000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":21000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[{"t":2,"text":"one","text_style":"lockup"},{"t":2.1,"text":"two","text_style":"skywriting"},{"t":2.2,"text":"three","text_style":"landmark"},{"t":2.3,"text":"four","text_style":"lockup"}]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":5,"count":5}},"25_stop_choice_after_decision":{"actions":[{"args":{"target":"horizon"},"at":5500,"id":"n0","name":"nudge","outcome":"committed","summary":"nudge horizon","viewer":"v2"},{"args":{"target":"peak-close"},"at":6000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge peak-close","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":55000,"closes_at":60000,"holding":"horizon","holding_label":"High above: the curve of the horizon","leading":"peak-seam-low-sun","next":"peak-close","next_hold_at":37000,"next_label":"Close to the peak","opened_at":35000,"targets":[{"arrive_at":70000,"id":"peak-seam-low-sun","label":"The peak at a low sun","votes":0},{"arrive_at":80000,"id":"high-lakes","label":"The high lakes","votes":0},{"arrive_at":70000,"id":"peak-seam","label":"The peak across the seam","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":30},{"fov_deg":70,"look":[-75,28,-170],"pos":[-30,38,-60],"t":37},{"fov_deg":50,"intent":"n1","look":[-56,2.9,-114.75],"pos":[-62,3.5,-105],"t":55},{"fov_deg":50,"intent":"n1","look":[-56,2.9,-114.75],"pos":[-62,3.5,-105],"t":62}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.45,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}},"26_stop_choice_fairness":{"actions":[{"args":{"target":"peak-seam-low-sun"},"at":1000,"id":"n1","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v1"},{"args":{"target":"dawn-valley"},"at":2000,"id":"n2","name":"nudge","outcome":"committed","summary":"nudge dawn-valley","viewer":"v2"},{"args":{"target":"peak-seam-low-sun"},"at":3000,"id":"n3","name":"nudge","outcome":"committed","summary":"nudge peak-seam-low-sun","viewer":"v1"}],"aggregates":[],"borrow":{"cap":0,"lent":0,"next_return_at":null},"budget":null,"budget_from_t":null,"claims":[],"cooldown_until":null,"counters":{"restores":0,"restores_last_hour":0,"scrub_failures":0,"scrubs":0},"failures":[],"fleet":{"ec2_instances":0,"ec2_n":0,"ec2_waiting":0,"fcvm_gpus":0,"gpus":0,"hosts":[],"max_tiles":100,"n":0,"power":{"asleep_since":null,"hosts":[],"idle_since":null,"last_wake":null,"state":"awake","wakes":[],"woke_at":null},"standby":[],"standby_ready":0,"warming":[]},"image_aspect":1.7777777777777777,"layout":{"ec2":null,"fcvm":null,"from_frame":0,"from_t":0,"generation":0},"next_race_at":null,"nudges":{"arrive_next_at":21000,"closes_at":26000,"holding":"coast","holding_label":"From the coast","leading":"dawn-valley","next":"peak-seam-low-sun","next_hold_at":7000,"next_label":"The peak at a low sun","opened_at":5000,"targets":[{"arrive_at":46000,"id":"dawn-valley","label":"Dawn in the valley","votes":0},{"arrive_at":36000,"id":"peak-close","label":"Close to the peak","votes":0},{"arrive_at":46000,"id":"high-lakes","label":"The high lakes","votes":0}]},"races":[],"relaunch_all":null,"scene":{"camera":[{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":0},{"fov_deg":60,"look":[-63.5,2.2,-87],"pos":[-100,0.35,-90],"t":7},{"fov_deg":55,"intent":"n1","look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":21},{"fov_deg":55,"intent":"n1","look":[-56,2.75,-114.75],"pos":[-66,2.5,-101],"t":28}],"epoch_unix_ms":0,"lights":[],"time_of_day":0.33,"words":[]},"scoreboard":{"ec2":null,"fcvm":null},"tiles":[],"viewers":{"active":2,"count":2}}}} diff --git a/src/fleet/coord/text_check.rs b/src/fleet/coord/text_check.rs index d568524dc..45e4ce8b4 100644 --- a/src/fleet/coord/text_check.rs +++ b/src/fleet/coord/text_check.rs @@ -13,6 +13,26 @@ use std::time::Duration; /// How long a word may wait for its check. pub const CHECK_TIMEOUT: Duration = Duration::from_secs(5); +/// What a viewer reads when no checker is configured (`--text-check none`). +pub const WORDS_OFF: &str = "words are turned off right now"; + +/// What a viewer reads when the check could not answer (an error or a timeout). The +/// reason itself goes to the coordinator log only: it can name the model, the region or +/// an AWS error, and the receipt and the actions list are public. +pub const CHECK_FAILED: &str = "couldn't check that word, try again"; + +/// The error `NoCheck` answers with, so the verdict can tell "turned off" from "failed". +#[derive(Debug)] +pub struct Disabled; + +impl std::fmt::Display for Disabled { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("no text check is configured") + } +} + +impl std::error::Error for Disabled {} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum Verdict { Allow, @@ -46,19 +66,30 @@ async fn check_inner(checker: &dyn TextChecker, sky: &[String], text: &str) -> T reason, sky: vec![], }, + Ok(Err(e)) if e.downcast_ref::().is_some() => TextCheck { + verdict: "error".into(), + reason: WORDS_OFF.into(), + sky: vec![], + }, Ok(Err(e)) => { tracing::warn!(error = %format!("{e:#}"), "text check failed"); TextCheck { verdict: "error".into(), - reason: format!("{e:#}"), + reason: CHECK_FAILED.into(), + sky: vec![], + } + } + Err(_) => { + tracing::warn!( + timeout_s = CHECK_TIMEOUT.as_secs(), + "text check failed: no answer in time" + ); + TextCheck { + verdict: "error".into(), + reason: CHECK_FAILED.into(), sky: vec![], } } - Err(_) => TextCheck { - verdict: "error".into(), - reason: format!("no answer within {} s", CHECK_TIMEOUT.as_secs()), - sky: vec![], - }, } } @@ -68,7 +99,7 @@ pub struct NoCheck; #[async_trait] impl TextChecker for NoCheck { async fn check(&self, _sky: &[String], _text: &str) -> Result { - bail!("no text check is configured") + Err(Disabled.into()) } } @@ -289,6 +320,27 @@ mod tests { }; let v = check_text(&slow, &[], "hi").await; assert_eq!(v.verdict, "error"); - assert!(v.reason.contains("no answer"), "{v:?}"); + assert_eq!(v.reason, CHECK_FAILED, "{v:?}"); + } + + /// The reason travels to the viewer (receipt, actions list, coordinator status), so it + /// is one of two fixed sentences and never the checker's own error. + #[tokio::test] + async fn viewers_read_a_fixed_reason_not_the_checker_error() { + let off = check_text(&NoCheck, &[], "Hi!").await; + assert_eq!( + (off.verdict.as_str(), off.reason.as_str()), + ("error", WORDS_OFF) + ); + let down = FakeCheck { + refuse: vec![], + fail: true, + delay: Duration::ZERO, + }; + let failed = check_text(&down, &[], "Hi!").await; + assert_eq!( + (failed.verdict.as_str(), failed.reason.as_str()), + ("error", CHECK_FAILED) + ); } } diff --git a/tests/test_fleet_coord.rs b/tests/test_fleet_coord.rs index 15936739a..b0672ce14 100644 --- a/tests/test_fleet_coord.rs +++ b/tests/test_fleet_coord.rs @@ -5,7 +5,7 @@ use fcvm::fleet::aggregate::codec::Raw; use fcvm::fleet::aggregate::frame::Frame; -use fcvm::fleet::coord::text_check::{FakeCheck, TextChecker}; +use fcvm::fleet::coord::text_check::{FakeCheck, NoCheck, TextChecker, CHECK_FAILED, WORDS_OFF}; use fcvm::fleet::coord::turnstile::{self, Verifier}; use fcvm::fleet::coord::{agents, ec2, live, poster, start, store, Handle}; use fcvm::fleet::lines::{spawn_writer, LineReader}; @@ -521,7 +521,23 @@ async fn a_failing_text_check_degrades_the_coordinator() { .unwrap() .clone(); assert_eq!(f["component"], "text_check"); - assert!(f["reason"].as_str().unwrap().contains("down"), "{f}"); + // The receipt, its check and the failures list are public: they carry a fixed + // sentence, and the checker's own error ("fake text check is down") stays in the log. + assert_eq!(f["reason"], CHECK_FAILED, "{f}"); + assert_eq!(r["reason"], CHECK_FAILED, "{r}"); + assert_eq!(r["check"]["reason"], CHECK_FAILED, "{r}"); +} + +/// A coordinator started with `--text-check none` tells the viewer words are off, not +/// that a check failed. +#[tokio::test] +async fn without_a_text_check_viewers_read_that_words_are_off() { + let (url, _h) = coordinator(Arc::new(NoCheck)).await; + let mut c = Viewer::connect(&url).await; + c.intent("w", "word", json!({"text": "Hi!"})).await; + let r = c.receipt("w").await; + assert_eq!(r["outcome"], "refused"); + assert_eq!(r["reason"], WORDS_OFF, "{r}"); } /// Limits are per client address: `CF-Connecting-IP` when the coordinator trusts it (it