From ccc53c1174d09d84359a9edbb8689788813586f2 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 09:36:16 -0400 Subject: [PATCH 01/13] protocols: color optomotor = CW/CCW pairs (15 s each, no gap, 10 s dark between pairs) per Grace; smoke variant same shape The full run is now 24 pairs (3 duties x 4 colors x 2 reps), ~16 min; the 10 s pause is kept across block boundaries. The reversal is phase-continuous (15 s x 80 fps = 1200 frames, a multiple of the 20-frame period) and the camera timestamp moves to the start of each pair so its round trip never stretches the CW half. Co-Authored-By: Claude Opus 5.5 --- protocols/g6_color_optomotor_smoke.yaml | 96 ++++----- protocols/g6_color_optomotor_test.yaml | 267 ++++++++++-------------- 2 files changed, 152 insertions(+), 211 deletions(-) diff --git a/protocols/g6_color_optomotor_smoke.yaml b/protocols/g6_color_optomotor_smoke.yaml index 576ee64b..afaa8ee8 100644 --- a/protocols/g6_color_optomotor_smoke.yaml +++ b/protocols/g6_color_optomotor_smoke.yaml @@ -1,14 +1,16 @@ version: 3 # ============================================================================ -# SMOKE variant of g6_color_optomotor_test.yaml — same patterns, same trial shape, cut to -# ~2 minutes: 10 s trials, the four colors clockwise + counterclockwise at duty 100 % only, -# one repetition, 2 s dark ITIs, camera bracket as in the full protocol. For checking a rig -# (patterns on the card, direction, BIAS recording) before the full ~18-minute run. +# SMOKE variant of g6_color_optomotor_test.yaml — same patterns, same CW/CCW pair shape +# (CW then CCW with no pause, dark between pairs), cut to ~2 minutes: 10 s per rotation, +# 5 s dark between pairs, the four colors at duty 100 % only, one repetition, camera +# bracket as in the full protocol. For checking a rig (patterns on the card, direction, +# BIAS recording) before the full ~16-minute run. 10 s × 80 frames/s = 800 frames, a +# whole number of 20-frame periods, so the reversal is phase-continuous here too. # ============================================================================ experiment_info: - name: "Color optomotor SMOKE — 10 s per pattern, duty 100 %, BIAS check" + name: "Color optomotor SMOKE — CW/CCW pairs, 10 s per rotation, duty 100 %, BIAS check" author: "Reiser Lab" rig: "./configs/rigs/cshl_g6_2x10.yaml" @@ -20,9 +22,9 @@ plugins: class: "BiasPlugin" variables: - trial_dur: &trial_dur 10 # s — trialParams duration AND its wait (one anchor, never drift) - iti_dur: &iti_dur 2 # s — dark between trials - pre_dark: &pre_dark 2 # s — dark before the first trial (originals: 300 s adaptation) + rot_dur: &rot_dur 10 # s — each rotation (CW, then CCW): trialParams duration AND its wait + pause_dur: &pause_dur 5 # s — dark between CW/CCW pairs + pre_dark: &pre_dark 2 # s — dark before the first pair (originals: 300 s adaptation) fps_cw: &fps_cw 80 # frames/s = 4 Hz temporal frequency at 20 px per period fps_ccw: &fps_ccw -80 duty_100: &duty_100 255 # 100 % @@ -30,11 +32,10 @@ variables: experiment: - "cam_start" - name: "duty_100_block" - trials: ["violet_cw_d100", "violet_ccw_d100", "blue_cw_d100", "blue_ccw_d100", - "green_cw_d100", "green_ccw_d100", "yellow_cw_d100", "yellow_ccw_d100"] + trials: ["violet_pair_d100", "blue_pair_d100", "green_pair_d100", "yellow_pair_d100"] repetitions: 1 - intertrial: "dark_iti" - - "cam_stop" + intertrial: "dark_pause" + - "cam_stop" # its own 10 s dark, then stopRecording conditions: # ── camera bracket (one ufmf for the whole run) ───────────────────────────── @@ -61,7 +62,7 @@ conditions: - type: "controller" command_name: "allOff" - type: "wait" - duration: *iti_dur + duration: *pause_dur - type: "plugin" plugin_name: "camera" command_name: "getTimestamp" @@ -69,15 +70,15 @@ conditions: plugin_name: "camera" command_name: "stopRecording" - - name: "dark_iti" + - name: "dark_pause" commands: - type: "controller" command_name: "allOff" - type: "wait" - duration: *iti_dur + duration: *pause_dur - # ── duty 100 % ────────────────────────────────────────────────────────────── - - name: "violet_cw_d100" + # ── duty 100 % ─────────────────────────────────────────────────────────────── + - name: "violet_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -86,32 +87,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "violet_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "blue_cw_d100" + duration: *rot_dur + + - name: "blue_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -120,32 +117,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "blue_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "green_cw_d100" + duration: *rot_dur + + - name: "green_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -154,32 +147,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "green_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "yellow_cw_d100" + duration: *rot_dur + + - name: "yellow_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -188,28 +177,23 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "yellow_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur + duration: *rot_dur diff --git a/protocols/g6_color_optomotor_test.yaml b/protocols/g6_color_optomotor_test.yaml index 6c9891a5..e91ac70b 100644 --- a/protocols/g6_color_optomotor_test.yaml +++ b/protocols/g6_color_optomotor_test.yaml @@ -1,32 +1,41 @@ version: 3 # ============================================================================ -# Color optomotor test — G6 2×10 four-color arena, web runner, BIAS camera. +# Color optomotor test — G6 2×10 four-color arena, web runner, BIAS camera +# (okey-cokey / visual motion centering rig: freely walking flies, no FicTrac). # # Modeled on the G4.1 MATLAB protocols 11 and 13 (Reiser Lab, 2026-04-21): open-loop -# rotating square-wave gratings, clockwise then counterclockwise, a camera timestamp at -# the start of every trial, one video for the whole run. What changes here: +# rotating square-wave gratings, clockwise then counterclockwise, one video for the run. +# What changes here: # * ONE LED color at a time. The four-color G6 panels put four colors in each 2×2 pixel # cell; each pattern lights only one color's pixels, on the TOP panel row only # (protocols/g6_color_optomotor_test_patterns/, built by # scripts/make-color-optomotor-patterns.js). Bank 0 violet 405 nm, 1 blue 470 nm, # 2 green 525 nm, 3 yellow-orange 590 nm. # * ONE spatial period (20 px = 36°), ONE speed (frame_rate 80 → 4 Hz temporal -# frequency, the originals' 4 Hz), both directions, 20 s trials. +# frequency, the originals' 4 Hz). # * Brightness by the per-trial `duty` override (fw #33, web-only): 25 %, 50 %, 100 % -# of 255 — one block per duty, the four colors × two directions inside, 2 repetitions. -# * No backlight, no thermometer (not driven by the web runner); the camera plugin -# (BiasPlugin) IS driven by the web runner through the bridge's BIAS connector -# (Studio v0.95, LAB-303), so startRecording / getTimestamp / stopRecording run. +# of 255 — one block per duty, the four colors inside, 2 repetitions. +# * No backlight, no thermometer (not driven by the web runner; turn the IR backlight on +# by hand). The camera plugin (BiasPlugin) IS driven by the web runner through the +# bridge's BIAS connector (Studio v0.95+, LAB-303): startRecording / getTimestamp / +# stopRecording run. # -# TIMING: 3 duties × 4 colors × 2 directions × 2 repetitions = 48 trials × 20 s = 16 min -# of stimulus, plus 48 × iti_dur of dark between trials. To make it 8 min set -# `repetitions: 1` on each block (24 trials) or trial_dur 10. All knobs are in `variables:`. +# TRIAL SHAPE (Grace, 2026-10-21): each trial is a CW/CCW PAIR — 15 s clockwise, then +# 15 s counterclockwise with NO pause between them, then 10 s dark before the next pair +# (also across block boundaries). The reversal is phase-continuous: 15 s × 80 frames/s = +# 1200 frames = a whole number of 20-frame periods, so CW ends on frame 0, where CCW +# starts. The camera timestamp is taken at the start of each pair (in the dark), never +# between CW and CCW, so its round trip cannot stretch the clockwise half. # -# DIRECTION: frame_rate +80 steps the pattern toward increasing column index (the -# generator's convention); "cw" below assumes that is clockwise seen from above, as fresh -# Designer patterns are. Confirm once on the arena (Console → Test) and swap the two -# anchors if it is reversed. +# TIMING: 3 duties × 4 colors × 2 repetitions = 24 pairs × 30 s = 12 min of stimulus, +# plus 24 × 10 s of dark between pairs and the 5 s pre-dark → ~16 min. For ~8 min set +# `repetitions: 1` on each block. All knobs are in `variables:`; keep rot_dur × 80 a +# multiple of 20 (rot_dur a multiple of 0.25 s) to keep the reversal phase-continuous. +# +# DIRECTION: frame_rate +80 steps the pattern toward increasing column index; on the +# four-color arena that is clockwise seen from above (confirmed on the smoke run, +# ww18 2026-10-15). If a rig shows it reversed, swap the fps_cw / fps_ccw values. # # BEFORE RUNNING: upload the patterns folder from the Console (Patterns → Add ▾), check # the SD listing shows the four names with 20 frames; BIAS must be running with its @@ -36,7 +45,7 @@ version: 3 # ============================================================================ experiment_info: - name: "Color optomotor test — one LED color at a time, top row, duty sweep" + name: "Color optomotor test — one LED color at a time, top row, CW/CCW pairs, duty sweep" author: "Reiser Lab" rig: "./configs/rigs/cshl_g6_2x10.yaml" @@ -48,9 +57,9 @@ plugins: class: "BiasPlugin" variables: - trial_dur: &trial_dur 20 # s — trialParams duration AND its wait (one anchor, never drift) - iti_dur: &iti_dur 3 # s — dark between trials - pre_dark: &pre_dark 5 # s — dark before the first trial (originals: 300 s adaptation) + rot_dur: &rot_dur 15 # s — each rotation (CW, then CCW): trialParams duration AND its wait + pause_dur: &pause_dur 10 # s — dark between CW/CCW pairs + pre_dark: &pre_dark 5 # s — dark before the first pair (originals: 300 s adaptation) fps_cw: &fps_cw 80 # frames/s = 4 Hz temporal frequency at 20 px per period fps_ccw: &fps_ccw -80 duty_25: &duty_25 64 # 25 % of 255 @@ -60,21 +69,20 @@ variables: experiment: - "cam_start" - name: "duty_25_block" - trials: ["violet_cw_d25", "violet_ccw_d25", "blue_cw_d25", "blue_ccw_d25", - "green_cw_d25", "green_ccw_d25", "yellow_cw_d25", "yellow_ccw_d25"] + trials: ["violet_pair_d25", "blue_pair_d25", "green_pair_d25", "yellow_pair_d25"] repetitions: 2 - intertrial: "dark_iti" + intertrial: "dark_pause" + - "dark_pause" # keep the 10 s pause across the block boundary - name: "duty_50_block" - trials: ["violet_cw_d50", "violet_ccw_d50", "blue_cw_d50", "blue_ccw_d50", - "green_cw_d50", "green_ccw_d50", "yellow_cw_d50", "yellow_ccw_d50"] + trials: ["violet_pair_d50", "blue_pair_d50", "green_pair_d50", "yellow_pair_d50"] repetitions: 2 - intertrial: "dark_iti" + intertrial: "dark_pause" + - "dark_pause" # keep the 10 s pause across the block boundary - name: "duty_100_block" - trials: ["violet_cw_d100", "violet_ccw_d100", "blue_cw_d100", "blue_ccw_d100", - "green_cw_d100", "green_ccw_d100", "yellow_cw_d100", "yellow_ccw_d100"] + trials: ["violet_pair_d100", "blue_pair_d100", "green_pair_d100", "yellow_pair_d100"] repetitions: 2 - intertrial: "dark_iti" - - "cam_stop" + intertrial: "dark_pause" + - "cam_stop" # its own 10 s dark, then stopRecording conditions: # ── camera bracket (one ufmf for the whole run) ───────────────────────────── @@ -101,7 +109,7 @@ conditions: - type: "controller" command_name: "allOff" - type: "wait" - duration: *iti_dur + duration: *pause_dur - type: "plugin" plugin_name: "camera" command_name: "getTimestamp" @@ -109,15 +117,15 @@ conditions: plugin_name: "camera" command_name: "stopRecording" - - name: "dark_iti" + - name: "dark_pause" commands: - type: "controller" command_name: "allOff" - type: "wait" - duration: *iti_dur + duration: *pause_dur - # ── duty 25 % ─────────────────────────────────────────────────────────────── - - name: "violet_cw_d25" + # ── duty 25 % ──────────────────────────────────────────────────────────────── + - name: "violet_pair_d25" commands: - type: "plugin" plugin_name: "camera" @@ -126,32 +134,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "violet_ccw_d25" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "blue_cw_d25" + duration: *rot_dur + + - name: "blue_pair_d25" commands: - type: "plugin" plugin_name: "camera" @@ -160,32 +164,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "blue_ccw_d25" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "green_cw_d25" + duration: *rot_dur + + - name: "green_pair_d25" commands: - type: "plugin" plugin_name: "camera" @@ -194,32 +194,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "green_ccw_d25" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "yellow_cw_d25" + duration: *rot_dur + + - name: "yellow_pair_d25" commands: - type: "plugin" plugin_name: "camera" @@ -228,34 +224,29 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur - - name: "yellow_ccw_d25" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_25 - type: "wait" - duration: *trial_dur + duration: *rot_dur - # ── duty 50 % ─────────────────────────────────────────────────────────────── - - name: "violet_cw_d50" + # ── duty 50 % ──────────────────────────────────────────────────────────────── + - name: "violet_pair_d50" commands: - type: "plugin" plugin_name: "camera" @@ -264,32 +255,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "violet_ccw_d50" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "blue_cw_d50" + duration: *rot_dur + + - name: "blue_pair_d50" commands: - type: "plugin" plugin_name: "camera" @@ -298,32 +285,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "blue_ccw_d50" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "green_cw_d50" + duration: *rot_dur + + - name: "green_pair_d50" commands: - type: "plugin" plugin_name: "camera" @@ -332,32 +315,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "green_ccw_d50" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "yellow_cw_d50" + duration: *rot_dur + + - name: "yellow_pair_d50" commands: - type: "plugin" plugin_name: "camera" @@ -366,34 +345,29 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur - - name: "yellow_ccw_d50" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_50 - type: "wait" - duration: *trial_dur + duration: *rot_dur - # ── duty 100 % ────────────────────────────────────────────────────────────── - - name: "violet_cw_d100" + # ── duty 100 % ─────────────────────────────────────────────────────────────── + - name: "violet_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -402,32 +376,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "violet_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_violet" pattern_ID: 1 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "blue_cw_d100" + duration: *rot_dur + + - name: "blue_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -436,32 +406,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "blue_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_blue" pattern_ID: 2 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "green_cw_d100" + duration: *rot_dur + + - name: "green_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -470,32 +436,28 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "green_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_green" pattern_ID: 3 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "yellow_cw_d100" + duration: *rot_dur + + - name: "yellow_pair_d100" commands: - type: "plugin" plugin_name: "camera" @@ -504,28 +466,23 @@ conditions: command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_cw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur - - name: "yellow_ccw_d100" - commands: - - type: "plugin" - plugin_name: "camera" - command_name: "getTimestamp" + duration: *rot_dur - type: "controller" command_name: "trialParams" pattern: "G6_2x10_toprow_grat20_yellow" pattern_ID: 4 - duration: *trial_dur + duration: *rot_dur mode: 2 frame_index: 0 frame_rate: *fps_ccw gain: 0 duty: *duty_100 - type: "wait" - duration: *trial_dur + duration: *rot_dur From 0a54863dfca98face23a8bf6a95c389bb8db06d3 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 09:38:30 -0400 Subject: [PATCH 02/13] ros2-poc: okey-cokey PC run sheet (BIAS + G6, no FicTrac) and a Foxglove camera + G6 dashboard plan Co-Authored-By: Claude Opus 5.5 --- .../okey-cokey-foxglove-dashboard-plan.md | 146 ++++++++++++++++++ ros2-poc/okey-cokey-pc-2026-10-21.md | 126 +++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md create mode 100644 ros2-poc/okey-cokey-pc-2026-10-21.md diff --git a/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md b/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md new file mode 100644 index 00000000..df28d1ff --- /dev/null +++ b/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md @@ -0,0 +1,146 @@ +# Plan: a purpose-built Foxglove dashboard for the okey-cokey rig (camera + G6, no FicTrac) + +Status: PLAN, 2026-10-21. Nothing built. Part of LAB-303 (ROS 2 graph); ticket to be filed. + +## Goal + +One Foxglove layout for a freely-walking-fly rig (overhead BIAS camera + G6 2×10 four-color arena, no +FicTrac) showing three things: + +1. **The camera.** +2. **The expected display**: what the arena should be showing right now (pattern, frame, color, duty), + rendered as an image of the unrolled arena. +3. **Health and rate** for the arena and for BIAS: is each one alive, at what rate, and has anything + dropped. + +The same layout should work **live** (foxglove_bridge during the run) and on **replay** (the run's MCAP +afterwards). + +## What exists today (release 2026-10-20, `ros2-poc/`) + +| Need | Today | Gap | +|---|---|---| +| BIAS health | `/bias/status` at 5 Hz (`bias_node.py`): reachable, capturing, logging, frameCount, framesPerSec, rtt_ms; on Atlas10 also frameIdGaps, cameraSkipped, loggerSkipped, writerSkipped, logQueueSize. `viz_node` turns these into `/viz/bias/*` and a `bias` entry in `/diagnostics`. `foxglove/arena-bias.json` shows them. | Mostly there. Missing: a rate from the frameCount slope (independent of BIAS's own estimate) and the drop counters on a plot. | +| Arena health | `viz_node`'s `arena` entry is fed only by `/arena/frame`, which only FicTrac lines produce. | **Always "never seen" on this rig.** | +| What the arena is doing | Every Studio message (trialParams `a` echoes, runner status events with `conditionName` / `params` / `durationSec`, controller telemetry `cc`/`cf`/`cs` row batches) reaches `bridge_node`. | It is **written to the bag only** (`/arena/browser`, `/arena/event` via `rec.write_raw`), never published, so nothing live can subscribe. The `a` hex is truncated to 8 bytes (no gain/duration). | +| Expected display image | `tools/patfile.py` reads `.pat`; `tools/tiff_run_to_mcap.py` already renders `/arena/display` as a `foxglove.CompressedImage` (200×40, upscaled, labeled RECONSTRUCTED). | Offline only, and wired to the 2P color-sweep file names and DO1 timing. Monochrome. | +| Camera image | Nothing. BIAS's HTTP API has no frame grab (only status, timestamps, frame count, video file). | **No live frames anywhere.** After the run, the `.ufmf` plus the per-frame stamps on `/bias/frames` (recorded at stopRecording) are enough to rebuild a timed image stream. | + +## Design + +### A. Make the arena's activity a live topic (bridge_node, small) + +`bridge_node` publishes what it already records: + +- `/arena/browser`: String JSON, verbatim, the same record it writes to the bag (QoS reliable, depth 50). + It stays in the bag as now. + +That one change lets any node watch the run without touching the Studio or bridge 3.4. + +### B. `stimulus_node.py`: expected display + arena health (new node, the core of the work) + +It subscribes to `/arena/browser` and keeps the current stimulus state: + +- From runner status events and the trialParams echoes: condition name, pattern name, mode, frame + rate, start frame, duty, and the trial start time (the echo's `ms`, i.e. when the command left the + browser). +- Expected frame for Mode 2: `idx = (init + round(fps · (t − t_start))) mod n_frames`. `allOff` = dark. + Mode 3 is out of scope here (it needs FicTrac). +- Observed frame: telemetry `cf` rows carry the frame the controller actually put up (idx + pattern + per frame change). Arrival is in ~10 Hz batches, so the observed stream lags by ~100 ms, but each row + has its own controller timestamp. + +It publishes: + +| Topic | Type | Rate | What | +|---|---|---|---| +| `/arena/stimulus` | String JSON (later typed) | on change | `{condition, pattern, mode, fps, init, duty, t_start}` | +| `/arena/display` | `foxglove.CompressedImage` (PNG) | 10–15 Hz | the expected display: unrolled 200×40, ×4 upscale, top row = top | +| `/viz/arena/expected_idx`, `/viz/arena/observed_idx` | Float64 | 10–15 Hz / per row | for one plot of the two indices together | +| `/viz/arena/display_rate_hz` | Float64 | 1 Hz | frame changes per second from `cf` rows; should equal \|fps\| (80) during a trial and 0 in the dark | +| `/viz/arena/lag_frames` | Float64 | 1 Hz | expected − observed (mod n), the "is the arena keeping up" number | +| `/diagnostics` entry `arena` | DiagnosticStatus | 1 Hz | OK / WARN (rate off by > 5 %, lag > 2 frames, a `display_gap` or flagged `trial_quality`) / STALE (no telemetry for > 2 s while a run is open) / ERROR (non-ok command status) | + +**Color, not green.** The four-color panels light one bank per pattern, so a green ramp would show the +wrong thing. Port the minimum of `js/panel-color.js` to Python (the layout table, and +`bankAt(row, col) = 2·((row+1)%2) + (col%2)` in host coordinates), and pin it to the JS module with a +test. Duty scales the brightness. Without that test, the two copies will drift. + +**Patterns.** The node needs the `.pat` files. It looks them up by name under a `--pattern-dir` (the +protocol's `*_patterns/` folder) and caches them. If a pattern is missing, the image shows the name on +a gray placeholder; the run is never refused. + +**If the firmware has no telemetry** (GET_FIRMWARE_VERSION flag bit 2 off, or the poller is off), there +are no `cf` rows: show the expected display only, observed = "unknown", and arena health from command +status alone. It must never claim "OK" without evidence (the same rule as `trial-quality.js`). + +### C. BIAS health additions (viz_node, small) + +- `/viz/bias/fps_from_count`: frameCount slope over the last 2 s of 5 Hz polls, next to BIAS's own + `framesPerSec`. +- `/viz/bias/drops`: the sum of the Atlas10 counters (frameIdGaps + cameraSkipped + loggerSkipped + + writerSkipped) as a plot; WARN on any increase while logging. Absent keys (Spinnaker build) mean + "not reported", not 0. +- `/viz/bias/rtt_ms` from the status polls; WARN at > 100 ms. + +### D. Camera + +- **Replay (phase 2).** `ufmf_to_mcap.py` reads the run's `.ufmf` and writes a sidecar MCAP with + `/camera/image` as `foxglove.CompressedImage` JPEG, timed by the `/bias/frames` stamps from the run + bag, downsampled (for example 10 Hz, half resolution) so the file stays small. Foxglove opens the run + bag and the sidecar together. uFMF needs a reader: write a small one (the format is documented, and + motmot's reader is Python 2-era), and test it on a ww18 recording. +- **Live (phase 3, only if wanted).** BIAS has to push frames out: a small BIAS patch (in our + atlas-bench fork) publishing a rate-limited JPEG preview over ZMQ. A node then mirrors it as + `/camera/image`, copying `scanimage_frames_node.py` (rate limit + loss stats). Until then, keep the + BIAS window beside Foxglove during the run. + +### E. The layout: `foxglove/arena-okey-cokey.json` + +``` +┌──────────────────────────────┬─────────────────────────────────────┐ +│ Image /camera/image │ Image /arena/display (expected) │ +│ (replay; live after phase 3) │ caption: condition · pattern · duty │ +├──────────────┬───────────────┼─────────────────────────────────────┤ +│ Indicators │ Plot: BIAS fps (own + from count), drops │ +│ BIAS reach/ ├─────────────────────────────────────────────────────┤ +│ capture/log; │ Plot: arena expected vs observed idx; display rate │ +│ arena state ├─────────────────────────────────────────────────────┤ +│ │ StateTransitions: /arena/stimulus condition │ +├──────────────┴──────────────────┬──────────────────────────────────┤ +│ DiagnosticSummary (/diagnostics)│ RawMessages /bias/reply │ +└─────────────────────────────────┴──────────────────────────────────┘ +``` + +### F. Replay parity + +The derived topics (B, C) do not exist in bags recorded before this work, and live ones are not +recorded unless we choose to. Do both: + +- Record `/arena/stimulus` live (it is small). Do not record `/arena/display`: it is rebuilt from + stimulus + patterns. +- Extend `nodes/foxglove_json.py` (the offline converter) to run the same `stimulus_node` logic over + the bag's `/arena/browser` records and emit `/arena/display` and the `/viz/arena/*` topics. Then the + layout works on any run, including today's. + +The logic lives in one pure module (`nodes/stimulus_core.py`) used both by the live node and by the +converter, the same split as `studio-replay.js`. + +## Phases and size + +| Phase | What | Size | Gate | +|---|---|---|---| +| 1 | A + B (no color port yet) + C + layout; harness test with the Studio emulator sending trialParams + `cf` rows | ~1 day | live on the Mac bench with the 2×10 arena | +| 1b | Python color port + test pinned to `js/panel-color.js` | ~2 h | the four banks show their colors | +| 2 | F (offline converter) + D replay (uFMF reader → sidecar MCAP) | ~1 day | today's okey-cokey run opens with camera + expected display in sync | +| 3 | D live (BIAS ZMQ preview patch + node) | ~1–2 days, BIAS C++ | only if a live camera in Foxglove is worth more than the BIAS window | + +## Open questions + +1. **Is a live camera in Foxglove needed, or is replay enough?** This decides whether phase 3 exists. +2. **Does the okey-cokey controller run telemetry-capable firmware?** (0xCB flag bit 2.) Without it, + the observed index and display rate are unavailable and arena health falls back to command status. +3. Which BIAS build is on the okey-cokey PC: the Atlas10 fork has the drop counters, the Spinnaker + build may not. +4. Typed messages: `viz_node` subscribes to `/arena/frame` as String, which breaks in the `typed` env. + That doesn't affect this rig (no `/arena/frame`), but fix it when touching viz_node. diff --git a/ros2-poc/okey-cokey-pc-2026-10-21.md b/ros2-poc/okey-cokey-pc-2026-10-21.md new file mode 100644 index 00000000..c2b9eda4 --- /dev/null +++ b/ros2-poc/okey-cokey-pc-2026-10-21.md @@ -0,0 +1,126 @@ +# Okey-cokey rig PC — BIAS + G6 color optomotor test, 2026-10-21 + +The run sheet for today's test on the new lab PC in the okey-cokey room (freely walking flies, overhead +BIAS camera, G6 2×10 four-color arena, **no FicTrac**). It is `WINDOWS-TEST.md` cut down to what this +rig needs, with the ww18 findings of 2026-10-15 (`windows-results-ww18-2026-10-15.md`) applied up front. +Record results in a sibling `okey-cokey-pc-results-2026-10-21.md` in the same shape as the ww18 file. + +Branch: `claude/okey-cokey-pairs` = the 2026-10-20 release candidate (Studio v0.97, ROS 2 graph, BIAS +connector) + the CW/CCW-pair protocols Grace asked for. The protocol change is on no Pages tier yet, +so serve the checkout locally (step 4). + +## 0. Machine (5 min, over Remote Desktop) + +- [ ] PC name, Windows version, free disk on the video drive. +- [ ] `git --version`, `pixi --version` (≥ 0.69), Chrome version. Install pixi if missing: + `powershell -ExecutionPolicy ByPass -c "irm -useb https://pixi.sh/install.ps1 | iex"`. +- [ ] Short checkout path (ww18 W2: long paths break the optional typed build; the default env does not + care, but keep it short anyway): `git clone https://github.com/reiserlab/webDisplayTools C:\w`, + then `cd C:\w` and `git switch claude/okey-cokey-pairs`. +- [ ] No bridge 3.4 running; `netstat -ano | findstr ":7447 :8765 :8092 :60000 :5010"` shows only BIAS + on 5010 (once BIAS is up). +- [ ] `cd ros2-poc` then `pixi install --locked` (default env only; skip `-e typed`). ww18: 25 s cold. +- [ ] IR backlight on by hand (Arduino). The web runner does not drive the backlight yet. + +## 1. BIAS itself (10 min) + +- [ ] **Which BIAS build?** FLIR/Spinnaker build or the atlas-bench Atlas10 fork (ww18 ran the fork). + Note the exe path and camera model. +- [ ] External-control server on, port **5010** for camera 0: + `curl.exe "http://127.0.0.1:5010/?get-status"` returns `"success" : true`. Note any extra keys. +- [ ] Logging format = **ufmf** in BIAS's own settings (BIAS writes its configured format whatever + extension it is sent; ww18 B1). +- [ ] Choose and create the **video directory**, e.g. `D:\rec\okey-cokey`. Use plain file names with + no dots or spaces (BIAS cuts the name at the first dot; ww18 B1). +- [ ] Camera preview live, frame rate as expected (note `framesPerSec`). +- [ ] Never run `bias_sim` or the harness while the real BIAS is open: on Windows both can hold 5010 and + loopback clients reach the sim (ww18 W8; the sim now refuses, but don't test it). + +## 2. BIAS through the node (5 min) + +Two terminals in `ros2-poc`: + +```powershell +pixi run router +pixi run bias-node -- --host 127.0.0.1 --port 5010 --video-dir D:\rec\okey-cokey +``` + +Third terminal: + +```powershell +pixi run bias-cmd -- connect --json +pixi run bias-cmd -- startRecording --filename nodecheck.ufmf --json +pixi run bias-cmd -- getTimestamp --json # wait ≥ 0.5 s after start: the clock reads 0 before the first frame +pixi run bias-cmd -- stopRecording --json # expect resolved file + bytes + frame-stamp count +dir D:\rec\okey-cokey +``` + +- [ ] Every reply `"outcome": "ok"` (connect can take ~2.2 s on Atlas10; the node allows 15 s). +- [ ] `stopRecording` names the file BIAS really wrote (`…_cam_0_date_…_v001.ufmf`) and its frame count + = the lines in `stamp_log_cam0.txt`. +- [ ] Unknown path: start a recording, close BIAS, `getTimestamp` returns `unknown` / `error` within + seconds, no hang; reopen BIAS and the node's status flips back to reachable. + +Stop the node and the router (Ctrl-C). + +## 3. The graph, no FicTrac (1 min) + +```powershell +pixi run rig -- --bias --bias-args "--video-dir D:\rec\okey-cokey" +``` + +Expected: `[rig] up: router, fictrac, bias, bridge — Studio → ws://localhost:8765`. The FicTrac +adapter starts but stays idle (nothing sends to UDP 60000), which is fine. Add `--foxglove` only if +`pixi install -e viz` was done (see the dashboard plan). + +## 4. Studio, served from this checkout + +```powershell +cd C:\w +python .claude\nocache-server.py 8092 +``` + +Chrome: `http://localhost:8092/arena_studio.html?p=g6_color_optomotor_smoke&rig=cshl_g6_2x10&mode=run` +(use this arena's own rig key from `configs\rigs\index.json` if it has one). Fallback if the local +server is a problem: the Next tier (`/next/arena_studio.html`, same Studio v0.97) plus +Protocol ▾ → Open… on `C:\w\protocols\g6_color_optomotor_smoke.yaml`. + +- [ ] Run view → **Bridge: Connect** → `connected`. Note any Chrome local-network prompt. +- [ ] **Connect** → the controller. Firmware line in the log; note the build label. The 2×10 controller + must run the `arena-2x10-local` build (a `main` build rejects 2×10 patterns). +- [ ] SD card mounted (Console shows a listing; ww18 H1 had an unmounted card). Upload + `protocols\g6_color_optomotor_test_patterns\` (Patterns → Add ▾); the four + `G6_2x10_toprow_grat20_*` names show 20 frames. +- [ ] **Direction**, once: Console → Test `green_pair_d100`. The first 10 s must turn clockwise seen + from above, then reverse with no gap. If reversed, swap `fps_cw` / `fps_ccw` in Edit view and + record it. + +## 5. Smoke run (2 min), then the full run (16 min) + +- [ ] Experimenter + genotype filled, **Run experiment** on `g6_color_optomotor_smoke`: 4 pairs at duty + 100 % (10 s CW → 10 s CCW, 5 s dark), camera connect / startRecording / getTimestamp / + stopRecording all `ok` in the step strip. +- [ ] Watch: top row only, one color per pair, no dark gap at the CW→CCW reversal, bottom row dark. +- [ ] Then `?p=g6_color_optomotor_test` (Grace's shape): 24 pairs of 15 s CW + 15 s CCW, 10 s dark + between pairs and across blocks, duty 25 → 50 → 100 %. Is the 25 % vs 100 % step visible? + +## 6. Collect and judge + +```powershell +cd C:\w\ros2-poc +pixi run export # unless the Studio auto-committed +pixi run python ..\scripts\runlog-check.py runlogs\exports\.jsonl +pixi run ros2 bag info runlogs\.bag +``` + +- [ ] `runlog-check` OK (smoke: 4 conditions × 2 trialParams; full: 24 × 2 = 48 trialParams). +- [ ] `/bias/reply`: smoke 1 + 4 + 1 `getTimestamp` (start, each pair, stop); full 1 + 24 + 1. +- [ ] `/bias/frames` count = the stamp log's lines; video length ≈ run length × camera fps + (full ≈ 965 s × fps). +- [ ] Copy the bag, the export and the `.ufmf` somewhere Michael can reach (group share). + +## What to send back + +The PC facts (step 0), the BIAS build and status keys, every non-ok reply verbatim, the direction +result, the two runs' `runlog-check` output, frame counts vs expected, and anything that needed a +manual step. From 1d3549f9fb636935b1be5c9a01e637f9939ca52a Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 09:39:48 -0400 Subject: [PATCH 03/13] ros2-poc: handover for the okey-cokey lab-PC session (BIAS test, then dashboard phases 1/1b/2) Co-Authored-By: Claude Opus 5.5 --- ros2-poc/HANDOVER-okey-cokey-2026-10-21.md | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 ros2-poc/HANDOVER-okey-cokey-2026-10-21.md diff --git a/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md new file mode 100644 index 00000000..ce0756b7 --- /dev/null +++ b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md @@ -0,0 +1,117 @@ +# Handover: okey-cokey rig, BIAS test + Foxglove dashboard (2026-10-21) + +For a Claude Code session running **on the okey-cokey lab PC** (Windows; the room with the G6 2×10 +four-color arena, an overhead BIAS camera, freely walking flies, no FicTrac). Michael's Mac has no +remote path into the rig PCs, so start the session there: open Claude Code in a checkout of +`reiserlab/webDisplayTools` and paste the kickoff prompt below as the first message. + +There are two jobs, in this order: + +1. **Test** today's protocol and BIAS on this PC: [`okey-cokey-pc-2026-10-21.md`](okey-cokey-pc-2026-10-21.md). +2. **Implement and test** the dashboard, phase 1 then 1b then 2: + [`docs/okey-cokey-foxglove-dashboard-plan.md`](docs/okey-cokey-foxglove-dashboard-plan.md). + +## Kickoff prompt (paste on the lab PC) + +``` +Okey-cokey rig work, LAB-303. Read ros2-poc/HANDOVER-okey-cokey-2026-10-21.md first and follow it. +Branch: git fetch && git switch claude/okey-cokey-pairs (clone to a short path such as C:\w if this +PC has no checkout yet; install pixi if missing, asking me before any admin/UAC step). +Job 1: run ros2-poc/okey-cokey-pc-2026-10-21.md in order, keeping +ros2-poc/okey-cokey-pc-results-2026-10-21.md as you go (one row per step: pass/fail, timings, +sizes, every error verbatim). Stop and ask me (Michael) before anything that needs a click: the +BIAS GUI, the Chrome serial-port picker, the IR backlight, or putting flies in. +Job 2: on a new branch claude/okey-cokey-dashboard cut from claude/okey-cokey-pairs, implement +ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md phases 1, 1b, 2 in that order, each tested +(harness first, then on this rig) and committed separately. Rules are in the handover's +"Constraints" section. Push both branches when done; open no PR. +``` + +## Where things stand + +- **Branch `claude/okey-cokey-pairs`** = release candidate 2026-10-20 rc1 (Studio v0.97, the ROS 2 + graph, the BIAS connector; on Next) plus three commits: + - `protocols/g6_color_optomotor_test.yaml` + `_smoke.yaml` now use **Grace's trial shape + (2026-10-21)**: each trial is a CW/CCW pair, 15 s CW then 15 s CCW with no gap, then 10 s dark (also + across block boundaries). The full run is 24 pairs, 16.1 min; the smoke run is 4 pairs, 10 s per + direction, 5 s dark, 1.7 min. Parsed and flattened with the real modules (no blocking errors, no + warnings); `pixi run test -- protocol` 106/106. + - The run sheet and the dashboard plan (linked above). +- **Not on any Pages tier**: Next serves the candidate *without* the new protocol shape. Serve this + checkout locally (`python .claude\nocache-server.py 8092`) or open the YAML via Protocol ▾ → Open…. +- **Background reading for job 1**: the ww18 results, `windows-results-ww18-2026-10-15.md` (every + Windows finding is in its table; most are fixed on this branch). For job 2: the plan's "What exists + today" table, which has file:line refs. +- **Not settled yet; the defaults below apply unless Michael says otherwise**: + 1. Live camera in Foxglove: **no** for now (no phase 3; the BIAS window sits beside Foxglove). Replay + camera (phase 2): yes. + 2. Controller telemetry: check it in job 1. Look for the firmware line at Connect and whether the + Studio starts the telemetry poller (GET_FIRMWARE_VERSION flag bit 2). Record the answer; phase 1 + must work either way, falling back to expected display only, with health = command status. + 3. BIAS build: record which one in job 1 (the Atlas10 fork reports drop counters, the Spinnaker + build may not). Missing keys mean "not reported", never 0. + +## Job 2: implementation notes + +Build in this order; each phase ends green before the next starts. + +**Phase 1** (plan §A, §B without color, §C, §E): +- `nodes/bridge_node.py`: also *publish* `/arena/browser` (String, reliable, depth 50), the same record + it already writes to the bag. Keep the bag write exactly as it is. +- `nodes/stimulus_core.py` (pure, no rclpy): stimulus state from `/arena/browser` records (runner + status events + trialParams `a` echoes + `cf` telemetry rows); expected index for Mode 2 + `(init + round(fps·(t − t_start))) mod n`; observed index, display rate and lag from `cf` rows; + diagnostic level. Read the row/event formats from `docs/development/telemetry-logging-reference.md` + (the format authority) and `js/arena-telemetry.js`'s header comment (the `cf` schema). Remember that + `cf` rows are 8 or 11 elements long. +- `nodes/stimulus_node.py`: a thin rclpy wrapper; `--pattern-dir`; publishes the topics in the plan's + table; `/arena/display` as `foxglove.CompressedImage` PNG reusing `tools/patfile.py` and the rendering + in `tools/tiff_run_to_mcap.py` (`display_png`), moved into a shared helper rather than copied. +- `nodes/viz_node.py`: feed the `arena` diagnostic from the new topics; add the BIAS + `fps_from_count` / `drops` / `rtt_ms` topics. +- `nodes/rig.py`: `--stimulus` (and include it with `--foxglove`), passing `--pattern-dir`. +- `foxglove/arena-okey-cokey.json`: the layout in plan §E (the camera panel can stay empty until + phase 2). +- Tests: extend `tests/test_ros2_poc.py` / `tests/test_viz_node.py`. `tests/studio_emulator.py` already + drives runs: make it send a Mode-2 trialParams + a stream of `cf` rows and assert expected vs + observed index, the rate (≈ |fps|) and the diagnostic levels (including the no-telemetry + fallback). Put unit tests for `stimulus_core` beside it. +- On the rig: `pixi run rig -- --bias --foxglove --stimulus …`, Foxglove (desktop app) → + `ws://localhost:8766` → import the layout; run the smoke protocol. Pass if the expected display + turns, reverses with no gap, goes dark in the pauses, and shows the condition name; the arena rate + reads ≈ 80 Hz during rotation and 0 in the dark (with telemetry); the BIAS fps and indicators are + live. Take a screenshot for the results file. + +**Phase 1b**: a Python port of the minimum of `js/panel-color.js` (layout table, `bankAt(row, col) = +2·((row+1)%2)+(col%2)` in host coordinates, the four-color layout), and a test that runs the JS module +under the root pixi Node and compares bank assignments cell by cell, so the two copies can't drift. +Duty scales the brightness. Pass: each of the four patterns renders in its own color, top row only. + +**Phase 2** (plan §D replay, §F): extend `nodes/foxglove_json.py` to replay `stimulus_core` over a bag's +`/arena/browser` records. Add `tools/ufmf_to_mcap.py`: read the run's `.ufmf` (write a small reader; +test it on a real recording from this PC) and write a sidecar MCAP with `/camera/image` JPEG timed by +the bag's `/bias/frames` stamps, downsampled (default 10 Hz, half resolution, both flags). Pass: today's +smoke run opens in Foxglove with the camera, the expected display and health in sync. Spot-check sync +on the reversal: the fly's turning should flip within a few hundred ms of the display's. + +## Constraints + +- **Never touch Production or Next**, never merge to `main`, open no PR. Code reaches `main` only + through a release candidate, which Michael builds (`docs/development/release-process.md`). +- **No Studio changes** (`arena_studio.html`, `js/*`) and **no bridge 3.4 changes** + (`fictrac-bridge/bridge.py`). Everything lives in `ros2-poc/`. If a Studio change turns out to be + needed, stop and write it up instead. +- Tests: `pixi run test` and `pixi run test-viz` inside `ros2-poc/`, plus `pixi run test` at the repo + root, all green before every commit. Prettier only on `*.js`, never on HTML. +- Windows habits from ww18: write text with `newline="\n"`; take durations from `perf_counter`; never + run `bias_sim` or the harness's sim while the real BIAS is open on 5010; Ctrl-C the rig to stop it + (the SIGBREAK handler closes the bag cleanly). +- Capture an error's full text before you try to fix it. Commit small fixes found in job 1 to + `claude/okey-cokey-pairs` with a message that names the step. +- American English in code, docs and commits. + +## What to send back + +The results file for job 1; per phase: what was built, test counts, the rig screenshot, and anything +that needed a manual step or went against the plan. Push `claude/okey-cokey-pairs` and +`claude/okey-cokey-dashboard`, and end with a short summary Michael can paste into the LAB-303 ticket. From 2badafb6e9dc84286419540debc75ad9ecfebdd4 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 10:07:52 -0400 Subject: [PATCH 04/13] ros2-poc: camera + G6 Foxglove dashboard for the okey-cokey rig (LAB-303) Live: bridge_node also publishes /arena/browser (the bag record is unchanged); the new stimulus_node turns it into the EXPECTED display (the trial's .pat frame x duty in the panels' LED colors, PNG) and the OBSERVED display (controller telemetry cf rows), plus the arena's lag, display rate and health, all on the controller's own clock. viz_node gains BIAS fps-from-count, drops, rtt and writer queue, and --sources. rig.py: --stimulus (included by --foxglove) and --no-fictrac. Layout: foxglove/arena-okey-cokey.json. Replay: foxglove_json.py --stimulus rebuilds the same topics from a finished bag (nodes/dashboard_replay.py, the same pure code) and /camera/image from the run's uFMF (tools/ufmf.py, BIAS's v4 layout), placed on the bridge clock by the getTimestamp replies. Panel colors are a Python port of js/panel-color.js, pinned cell by cell under Node. Tests: test-stimulus 54/54 (incl. a real 80 fps trial from the 2026-10-14 smoke run, every lag 0), test-stimulus-live 10/10, harness 54/54, root suite 71/71. No live camera (BIAS unchanged by decision). Co-Authored-By: Claude Opus 5.5 --- ros2-poc/README.md | 2 + ros2-poc/foxglove/arena-okey-cokey.json | 324 +++++++++++ ros2-poc/nodes/arena_display.py | 131 +++++ ros2-poc/nodes/bias_health.py | 35 ++ ros2-poc/nodes/bridge_node.py | 9 + ros2-poc/nodes/dashboard_replay.py | 228 ++++++++ ros2-poc/nodes/foxglove_json.py | 65 ++- ros2-poc/nodes/rig.py | 18 +- ros2-poc/nodes/stimulus_core.py | 511 ++++++++++++++++++ ros2-poc/nodes/stimulus_node.py | 195 +++++++ ros2-poc/nodes/viz_node.py | 43 +- ros2-poc/pixi.toml | 6 + .../g6_color_optomotor_smoke.yaml | 25 + ros2-poc/tests/test_stimulus.py | 419 ++++++++++++++ ros2-poc/tests/test_stimulus_live.py | 141 +++++ ros2-poc/tools/ufmf.py | 234 ++++++++ 16 files changed, 2375 insertions(+), 11 deletions(-) create mode 100644 ros2-poc/foxglove/arena-okey-cokey.json create mode 100644 ros2-poc/nodes/arena_display.py create mode 100644 ros2-poc/nodes/bias_health.py create mode 100644 ros2-poc/nodes/dashboard_replay.py create mode 100644 ros2-poc/nodes/stimulus_core.py create mode 100644 ros2-poc/nodes/stimulus_node.py create mode 100644 ros2-poc/tests/fixtures/protocols-2026-10-14/g6_color_optomotor_smoke.yaml create mode 100644 ros2-poc/tests/test_stimulus.py create mode 100644 ros2-poc/tests/test_stimulus_live.py create mode 100644 ros2-poc/tools/ufmf.py diff --git a/ros2-poc/README.md b/ros2-poc/README.md index aa147a01..b2f91ad1 100644 --- a/ros2-poc/README.md +++ b/ros2-poc/README.md @@ -41,6 +41,8 @@ Pieces, individually (each is also a `pixi run` task; `--help` on every script): | `sim` | the root `fictrac-bridge/fictrac_sim.py`, unmodified (`-- --model fly --kp 2` for a fixating fly) | | `kinefly-sim` / `bias-sim` | synthetic KF/1 producer; a fake BIAS program (HTTP, state machine, writes the video file) | | `studio-emulator` | the Studio runner's exact bridge conversation, for machines without a browser | +| `stimulus` | `nodes/stimulus_node.py`: the arena's EXPECTED display (the trial's `.pat` frame × duty in the panels' LED colors) and its OBSERVED display (controller telemetry) as PNG images, plus arena rate / lag / health, from the live `/arena/browser`; `pixi run rig -- --stimulus` (included by `--foxglove`). Layout `foxglove/arena-okey-cokey.json` (camera + G6, no FicTrac); the same layout on a finished run: `pixi run fox-json -- --stimulus --panel-layout four-color` (adds `/camera/image` from the run's uFMF). Plan + status: `docs/okey-cokey-foxglove-dashboard-plan.md` | +| `test-stimulus` / `test-stimulus-live` | the dashboard's pure parts (panel colors pinned to `js/panel-color.js`, renderer, tracker on synthetic + a real 80 fps trial, uFMF reader, replay converter); the live path through router + bridge + stimulus node (~25 s, needs ports 7447/8765 free) | Environment set by the workspace (`[activation.env]`): `RMW_IMPLEMENTATION=rmw_zenoh_cpp`, `ZENOH_ROUTER_CHECK_ATTEMPTS=0` (nodes wait for the router), `ZENOH_ROUTER_CONFIG_URI` / diff --git a/ros2-poc/foxglove/arena-okey-cokey.json b/ros2-poc/foxglove/arena-okey-cokey.json new file mode 100644 index 00000000..34474f38 --- /dev/null +++ b/ros2-poc/foxglove/arena-okey-cokey.json @@ -0,0 +1,324 @@ +{ + "configById": { + "Image!camera": { + "foxglovePanelTitle": "Camera (BIAS) \u2014 replay: from the run's uFMF", + "imageMode": { + "imageTopic": "/camera/image" + } + }, + "Image!expected": { + "foxglovePanelTitle": "Arena \u2014 EXPECTED display (trial \u00d7 frame \u00d7 duty)", + "imageMode": { + "imageTopic": "/arena/display" + } + }, + "Image!observed": { + "foxglovePanelTitle": "Arena \u2014 OBSERVED (controller telemetry, ~10 Hz)", + "imageMode": { + "imageTopic": "/arena/display_observed" + } + }, + "Plot!index": { + "title": "Arena frame index: expected vs observed", + "paths": [ + { + "value": "/viz/arena/expected_idx.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "expected" + }, + { + "value": "/viz/arena/observed_idx.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "observed" + } + ], + "showXAxisLabels": true, + "showYAxisLabels": true, + "showLegend": true, + "legendDisplay": "floating", + "showPlotValuesInLegend": true, + "isSynced": true, + "xAxisVal": "timestamp", + "followingViewWidth": 30 + }, + "Plot!rate": { + "title": "Rates (Hz): arena frames shown vs commanded; camera fps", + "paths": [ + { + "value": "/viz/arena/display_rate_hz.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "arena shown" + }, + { + "value": "/viz/arena/cmd_fps.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "arena commanded (\u00b1)" + }, + { + "value": "/viz/bias/fps.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "camera (BIAS)" + }, + { + "value": "/viz/bias/fps_from_count.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "camera (from count)" + } + ], + "showXAxisLabels": true, + "showYAxisLabels": true, + "showLegend": true, + "legendDisplay": "floating", + "showPlotValuesInLegend": true, + "isSynced": true, + "xAxisVal": "timestamp", + "followingViewWidth": 30 + }, + "Plot!lag": { + "title": "Arena lag (frames, observed \u2212 expected; 0 = on schedule)", + "paths": [ + { + "value": "/viz/arena/lag_frames.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "lag" + } + ], + "showXAxisLabels": true, + "showYAxisLabels": true, + "showLegend": true, + "legendDisplay": "floating", + "showPlotValuesInLegend": true, + "isSynced": true, + "xAxisVal": "timestamp", + "followingViewWidth": 30, + "minYValue": -5, + "maxYValue": 5 + }, + "Plot!drops": { + "title": "Camera: frames dropped (cumulative) \u00b7 writer queue", + "paths": [ + { + "value": "/viz/bias/drops.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "dropped" + }, + { + "value": "/viz/bias/log_queue.data", + "enabled": true, + "timestampMethod": "receiveTime", + "label": "writer queue" + } + ], + "showXAxisLabels": true, + "showYAxisLabels": true, + "showLegend": true, + "legendDisplay": "floating", + "showPlotValuesInLegend": true, + "isSynced": true, + "xAxisVal": "timestamp", + "followingViewWidth": 30 + }, + "StateTransitions!states": { + "paths": [ + { + "value": "/viz/arena/condition.data", + "timestampMethod": "receiveTime", + "label": "condition" + }, + { + "value": "/viz/arena/state.data", + "timestampMethod": "receiveTime", + "label": "arena" + }, + { + "value": "/viz/bias/state.data", + "timestampMethod": "receiveTime", + "label": "BIAS" + } + ], + "isSynced": true + }, + "Indicator!arena": { + "foxglovePanelTitle": "Arena", + "path": "/viz/arena/ok.data", + "style": "bulb", + "fallbackColor": "#8b949e", + "fallbackLabel": "no data", + "rules": [ + { + "operator": "=", + "rawValue": "true", + "color": "#00e676", + "label": "OK" + }, + { + "operator": "=", + "rawValue": "false", + "color": "#ffd740", + "label": "check health" + } + ] + }, + "Indicator!bias": { + "foxglovePanelTitle": "BIAS", + "path": "/viz/bias/reachable.data", + "style": "bulb", + "fallbackColor": "#8b949e", + "fallbackLabel": "no status", + "rules": [ + { + "operator": "=", + "rawValue": "true", + "color": "#00e676", + "label": "online" + }, + { + "operator": "=", + "rawValue": "false", + "color": "#ff5252", + "label": "offline" + } + ] + }, + "Indicator!video": { + "foxglovePanelTitle": "Video", + "path": "/viz/bias/logging.data", + "style": "bulb", + "fallbackColor": "#8b949e", + "fallbackLabel": "no data", + "rules": [ + { + "operator": "=", + "rawValue": "true", + "color": "#ffd740", + "label": "writing" + }, + { + "operator": "=", + "rawValue": "false", + "color": "#8b949e", + "label": "off" + } + ] + }, + "RawMessages!health": { + "foxglovePanelTitle": "Arena health", + "topicPath": "/viz/arena/health", + "diffEnabled": false, + "diffMethod": "custom", + "diffTopicPath": "", + "showFullMessageForDiff": false, + "expansion": "all" + }, + "RawMessages!stimulus": { + "foxglovePanelTitle": "Stimulus now", + "topicPath": "/arena/stimulus", + "diffEnabled": false, + "diffMethod": "custom", + "diffTopicPath": "", + "showFullMessageForDiff": false, + "expansion": "all" + }, + "DiagnosticSummary!diag": { + "foxglovePanelTitle": "Health (live)", + "minLevel": 0, + "pinnedIds": [], + "hardwareIdFilter": "", + "topicToRender": "/diagnostics", + "sortByLevel": true + }, + "RawMessages!reply": { + "foxglovePanelTitle": "BIAS replies", + "topicPath": "/bias/reply", + "diffEnabled": false, + "diffMethod": "custom", + "diffTopicPath": "", + "showFullMessageForDiff": false, + "expansion": "all" + } + }, + "globalVariables": {}, + "userNodes": {}, + "playbackConfig": { + "speed": 1 + }, + "layout": { + "direction": "row", + "splitPercentage": 66, + "first": { + "direction": "column", + "splitPercentage": 52, + "first": { + "direction": "row", + "splitPercentage": 45, + "first": "Image!camera", + "second": { + "direction": "column", + "splitPercentage": 50, + "first": "Image!expected", + "second": "Image!observed" + } + }, + "second": { + "direction": "column", + "splitPercentage": 22, + "first": "StateTransitions!states", + "second": { + "direction": "row", + "splitPercentage": 50, + "first": { + "direction": "column", + "splitPercentage": 55, + "first": "Plot!index", + "second": "Plot!lag" + }, + "second": { + "direction": "column", + "splitPercentage": 55, + "first": "Plot!rate", + "second": "Plot!drops" + } + } + } + }, + "second": { + "direction": "column", + "splitPercentage": 14, + "first": { + "direction": "row", + "splitPercentage": 33, + "first": "Indicator!arena", + "second": { + "direction": "row", + "splitPercentage": 50, + "first": "Indicator!bias", + "second": "Indicator!video" + } + }, + "second": { + "direction": "column", + "splitPercentage": 18, + "first": "RawMessages!health", + "second": { + "direction": "column", + "splitPercentage": 40, + "first": "RawMessages!stimulus", + "second": { + "direction": "column", + "splitPercentage": 45, + "first": "DiagnosticSummary!diag", + "second": "RawMessages!reply" + } + } + } + } + } +} \ No newline at end of file diff --git a/ros2-poc/nodes/arena_display.py b/ros2-poc/nodes/arena_display.py new file mode 100644 index 00000000..7764d66b --- /dev/null +++ b/ros2-poc/nodes/arena_display.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""arena_display.py — render a G6 arena frame the way the Studio and Pattern Designer draw it, as PNG. + +Pure (numpy + stdlib), shared by the live `stimulus_node.py` and the offline `foxglove_json.py +--stimulus` converter, so a live dashboard and a replayed bag show the same pixels. + +Panel color is a PANEL property (LAB-228): a four-color G6 panel is the standard 20×20 board +with its four LED banks populated with different LEDs in a repeating 2×2 mosaic. This module is +a minimal Python port of `js/panel-color.js`, the one authority, and must stay equal to it: + + bank(row, col) = 2 * ((row + ROW_PARITY_FLIP) % 2) + (col % 2), ROW_PARITY_FLIP = 1 + +in HOST coordinates (row 0 = the bottom of the arena, col 0 = left), bench-verified 2026-09-29. +`tests/test_stimulus.py` runs the JS module under Node and compares every cell of every layout, +plus the channel colors, so a change on either side fails the harness. + + render_frame(frame, gs, layout="four-color", duty=255, scale=4) -> (H*scale, W*scale, 3) uint8 + png_bytes(img) -> bytes +""" + +from __future__ import annotations + +import struct +import zlib + +import numpy as np + +ROW_PARITY_FLIP = 1 # js/panel-color.js ROW_PARITY_FLIP (bench-verified 2026-09-29) +DEFAULT_LAYOUT = "g6-green" +OFF_RGB = (0x1E, 0x23, 0x29) # js/panel-color.js OFF_CSS '#1e2329' — an LED that is off + +# js/panel-color.js CHANNELS: on-screen primary at full brightness. +CHANNEL_RGB = { + "green": (153, 255, 51), + "violet": (150, 60, 255), + "blue": (40, 120, 255), + "yellow": (255, 170, 20), + "red": (255, 50, 40), + "ir": (190, 70, 110), +} + +# js/panel-color.js LAYOUT_DEFS: banks T0..T3. +LAYOUTS = { + "g6-green": ("green", "green", "green", "green"), + "four-color": ("violet", "blue", "green", "yellow"), + "red-ir-v0.4r2": ("red", "ir", "ir", "red"), + "red-ir-v0.4r1": ("red", "ir", "red", "ir"), +} + + +def bank_at(row: int, col: int) -> int: + """Bank index 0..3 of the LED at host (row, col) — js/panel-color.js bankAt.""" + return 2 * ((row + ROW_PARITY_FLIP) & 1) + (col & 1) + + +def is_mono(layout: str) -> bool: + banks = LAYOUTS.get(layout) + return banks is None or len(set(banks)) == 1 + + +def channel_grid(layout: str, rows: int, cols: int) -> np.ndarray: + """(rows, cols, 3) float64 of each LED's full-brightness color, host row 0 first. + + A mono layout uses the legacy green-phosphor ramp at full (0.6, 1.0, 0.2) × 255, which is what + js/panel-color.js `legacyRgb` draws (it equals the `green` channel color).""" + banks = LAYOUTS.get(layout) or LAYOUTS[DEFAULT_LAYOUT] + if is_mono(layout): + rgb = np.array((0.6 * 255, 255.0, 0.2 * 255), dtype=np.float64) + return np.broadcast_to(rgb, (rows, cols, 3)).copy() + r = np.arange(rows)[:, None] + c = np.arange(cols)[None, :] + bank = 2 * ((r + ROW_PARITY_FLIP) & 1) + (c & 1) + table = np.array([CHANNEL_RGB[b] for b in banks], dtype=np.float64) + return table[bank] + + +_GRID_CACHE: dict[tuple[str, int, int], np.ndarray] = {} + + +def render_frame(frame, gs: int, layout: str = DEFAULT_LAYOUT, duty: int = 255, scale: int = 4) -> np.ndarray: + """A .pat frame (uint8 (H, W), host row 0 = bottom; values 0..1 for GS2, 0..15 for GS16) → an + RGB image with the TOP of the arena at the top, each LED `scale`×`scale` pixels. + + Brightness = value / max × duty / 255 (the per-trial duty override scales the whole panel); an + LED at 0 is drawn in the OFF color, as the Pattern Designer grid does. `frame=None` = the + arena dark (allOff).""" + if frame is None: + raise ValueError("render_frame needs a frame; use render_dark for allOff") + h, w = frame.shape + key = (layout, h, w) + grid = _GRID_CACHE.get(key) + if grid is None: + grid = _GRID_CACHE[key] = channel_grid(layout, h, w) + vmax = 15.0 if gs == 16 else 1.0 + b = np.clip(frame.astype(np.float64) / vmax, 0.0, 1.0) * (max(0, min(255, int(duty))) / 255.0) + img = grid * b[:, :, None] + off = frame == 0 + img[off] = OFF_RGB + img = img[::-1, :, :] # host row 0 (bottom) → image bottom + out = np.clip(np.floor(img + 0.5), 0, 255).astype(np.uint8) # Math.round, as js/panel-color.js pixelCss + if scale > 1: + out = np.repeat(np.repeat(out, scale, axis=0), scale, axis=1) + return out + + +def render_dark(rows_px: int = 40, cols_px: int = 200, scale: int = 4) -> np.ndarray: + out = np.empty((rows_px * scale, cols_px * scale, 3), dtype=np.uint8) + out[:] = OFF_RGB + return out + + +def render_placeholder(rows_px: int = 40, cols_px: int = 200, scale: int = 4) -> np.ndarray: + """A pattern the node cannot find: a gray hatch, so it never looks like 'dark' or 'lit'.""" + out = render_dark(rows_px, cols_px, scale) + yy, xx = np.mgrid[0:rows_px * scale, 0:cols_px * scale] + out[((xx + yy) // (4 * scale)) % 2 == 0] = (70, 74, 80) + return out + + +def png_bytes(img: np.ndarray) -> bytes: + """8-bit grayscale (H, W) or RGB (H, W, 3) → PNG (stdlib zlib; no Pillow in the env).""" + color_type = 0 if img.ndim == 2 else 2 + h, w = img.shape[:2] + rows = np.ascontiguousarray(img, dtype=np.uint8).reshape(h, -1) + raw = np.concatenate([np.zeros((h, 1), dtype=np.uint8), rows], axis=1).tobytes() + + def chunk(tag: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body) & 0xFFFFFFFF) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, color_type, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"")) diff --git a/ros2-poc/nodes/bias_health.py b/ros2-poc/nodes/bias_health.py new file mode 100644 index 00000000..0dc9df16 --- /dev/null +++ b/ros2-poc/nodes/bias_health.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""bias_health.py — BIAS rate and loss numbers from `/bias/status` polls (pure; viz_node.py live, +foxglove_json.py --stimulus on replay).""" + +from __future__ import annotations + +from collections import deque + +# Loss counters the atlas-bench Atlas10 BIAS build reports in get-status (the FLIR/Spinnaker +# build may report none: absent = not reported, never 0). +BIAS_DROP_KEYS = ("frameIdGaps", "cameraSkipped", "incompleteFrames", "loggerSkipped", "writerSkipped") + + +class CountRate: + """Frames per second from a running frame counter sampled at uneven times: the slope over the + last `window_s` seconds. A counter that goes DOWN (a new recording) restarts the window.""" + + def __init__(self, window_s: float = 2.0) -> None: + self.window_s = window_s + self.samples: deque = deque() + + def add(self, t_s: float, count: float) -> float | None: + if self.samples and (count < self.samples[-1][1] or t_s <= self.samples[-1][0] - 5.0): + self.samples.clear() + self.samples.append((t_s, count)) + while len(self.samples) > 2 and self.samples[0][0] < t_s - self.window_s: + self.samples.popleft() + (t0, c0), (t1, c1) = self.samples[0], self.samples[-1] + return (c1 - c0) / (t1 - t0) if t1 - t0 > 0.2 else None + + +def bias_drops(st: dict) -> float | None: + """Sum of the drop counters BIAS reports, None when it reports none of them.""" + vals = [st[k] for k in BIAS_DROP_KEYS if isinstance(st.get(k), (int, float))] + return float(sum(vals)) if vals else None diff --git a/ros2-poc/nodes/bridge_node.py b/ros2-poc/nodes/bridge_node.py index a48b4800..4dbaeb07 100644 --- a/ros2-poc/nodes/bridge_node.py +++ b/ros2-poc/nodes/bridge_node.py @@ -22,6 +22,11 @@ the envelope and is used only for the `lat_ms` latency statistics. * `/arena/frame` — every frame the hub publishes is also published as a ROS topic, so `ros2 topic echo/hz` introspect the closed loop and ROS-native nodes can subscribe. + * `/arena/browser` — every Studio message is also PUBLISHED live, as received + (`{"rx_ms", "raw"}`, reliable): trialParams echoes, runner status events, controller + telemetry rows. The run's bag records the same messages separately (with `seq`, only while + a log is open), so this is for live consumers such as stimulus_node.py (the expected + display + arena health); nothing that writes the log reads it. * other device topics are LOGGED, not applied: `/kinefly/raw` (KF/1) → compact tagged rows `["kf", ms, seq, src_t_s, left_rad, right_rad, head_rad, abdomen_rad]`; `/bias/reply`, `/bias/status`, `/scanimage/reply`, `/scanimage/status` → `{"type":"connector","src":,…}` events. @@ -586,6 +591,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue, on_kin self.typed = typed self.stats = {"fictrac": SourceStats("fictrac"), "kinefly": SourceStats("kinefly"), **{c: SourceStats(c) for c in CONNECTOR_TOPICS}} self.frame_pub = self.create_publisher(ArenaFrame if typed else String, FRAME_TOPIC, FRAME_QOS) + self.browser_pub = self.create_publisher(String, BROWSER_TOPIC, RAW_QOS) # Studio → bridge connector_command relayed to the device node (LAB-303 connectors). self.cmd_pubs = {c: self.create_publisher(String, t["cmd"], RAW_QOS) for c, t in CONNECTOR_TOPICS.items()} if typed: @@ -771,6 +777,9 @@ async def dispatch(raw, websocket=None) -> None: (so the Studio's bridge client knows what it may drive; a ≤ 3.4 bridge never sends it) and `connector_command` is relayed to the device node. Everything else is 3.4.""" text = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else raw + live = String() + live.data = json.dumps({"rx_ms": B.now_ms(), "raw": text}, separators=(",", ":")) + node.browser_pub.publish(live) obj = None try: obj = json.loads(text) diff --git a/ros2-poc/nodes/dashboard_replay.py b/ros2-poc/nodes/dashboard_replay.py new file mode 100644 index 00000000..1f7ee17f --- /dev/null +++ b/ros2-poc/nodes/dashboard_replay.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""dashboard_replay.py — rebuild the live dashboard's derived topics from a finished run's bag. + +Used by `foxglove_json.py --stimulus` so `foxglove/arena-okey-cokey.json` works on a replayed +run exactly as it does live. It runs the SAME pure code the live nodes run — stimulus_core +(the expected/observed display and arena health), arena_display (the pixels), bias_health +(BIAS rate and losses) — over the bag's `/arena/browser` and `/bias/*` records, and emits: + + /arena/display, /arena/display_observed foxglove.CompressedImage (PNG), as stimulus_node + /viz/arena/{expected_idx, observed_idx, display_rate_hz, lag_frames, cmd_fps, ok, state, + condition, health} {"data": …} — the same `.data` paths as the live + std_msgs topics, so one layout serves both + /arena/stimulus, /arena/health JSON objects (the live /arena/stimulus + the health values) + /viz/bias/{fps, fps_from_count, frame_count, drops, rtt_ms, log_queue, reachable, capturing, + logging, state} {"data": …}, as viz_node + /camera/image foxglove.CompressedImage (grayscale PNG) from the run's + uFMF, when one is given or found (see camera_streams) + +Time: everything is stamped on the bridge's clock (the records' rx_ms), as live. Observed frames +and their lag are placed by the controller clock mapped to the bridge clock (stimulus_core's +offset). Camera frames are placed by BIAS's frame timestamps mapped to the bridge clock with the +run's getTimestamp replies: offset = median((t_ms + rx_epoch_ms)/2 − value·1000) over replies with +value > 0 (BIAS reads 0 before its first frame). +""" + +from __future__ import annotations + +import base64 +import json +import statistics +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / "tools")) +import arena_display as AD # noqa: E402 +import stimulus_core as SC # noqa: E402 +from bias_health import CountRate, bias_drops # noqa: E402 + +IMAGE_SCHEMA = "foxglove.CompressedImage" + + +def _ts(ns: int) -> dict: + return {"sec": ns // 1_000_000_000, "nsec": ns % 1_000_000_000} + + +def image_msg(t_ns: int, png: bytes, frame_id: str) -> dict: + return {"timestamp": _ts(t_ns), "frame_id": frame_id, "format": "png", "data": base64.b64encode(png).decode()} + + +def stimulus_streams(records, patterns, protocols, layout: str = "g6-green", display_hz: float = 15.0, scale: int = 4): + """records: [(topic, log_ns, obj)] in log-time order (obj = the String record's JSON). + Yields (topic, t_ns, obj, schema_name_or_None).""" + tr = SC.StimulusTracker(SC.PatternLibrary(patterns), SC.ProtocolIndex(protocols)) + tr.frame_log = [] + step_ms = 1000.0 / max(1.0, display_hz) + bias_rate = CountRate() + last = {"disp": None, "obs_key": None, "obs_ms": -1e18, "stim": None, "health_ms": None} + + def ns(ms: float) -> int: + return int(round(ms * 1e6)) + + def f(topic, ms, v): + if isinstance(v, bool) or isinstance(v, (int, float, str)): + return (topic, ns(ms), {"data": v}, None) + return None + + def render(name, idx, duty): + pat = tr.patterns.get(name) if name else None + if pat is None: + return AD.render_placeholder(scale=scale) + return AD.render_frame(pat["frames"][int(idx) % pat["num_frames"]], pat["gs"], layout, duty, scale) + + def display_at(ms): + st = tr.stimulus() + exp = tr.expected_index(ms) + key = (st["state"], st.get("pattern"), exp, st.get("duty")) + if key == last["disp"]: + return + last["disp"] = key + img = render(st.get("pattern"), exp, st.get("duty") or 255) if (st["state"] == "trial" and exp is not None) else AD.render_dark(scale=scale) + yield ("/arena/display", ns(ms), image_msg(ns(ms), AD.png_bytes(img), "arena_display"), IMAGE_SCHEMA) + if exp is not None: + yield f("/viz/arena/expected_idx", ms, exp) + + def stim_out(ms): + st = tr.stimulus() + if st != last["stim"]: + last["stim"] = st + yield ("/arena/stimulus", ns(ms), st, None) + yield f("/viz/arena/state", ms, st["state"]) + yield f("/viz/arena/condition", ms, st.get("condition") or "") + yield f("/viz/arena/cmd_fps", ms, st.get("fps", 0) if st["state"] == "trial" else 0) + + def health_out(ms): + h = tr.health(ms) + v = h["values"] + out = [("/arena/health", ns(ms), {"level": h["level"], "level_name": h["level_name"], "message": h["message"], **v}, None), + f("/viz/arena/health", ms, f"{h['level_name']}: {h['message']}"), f("/viz/arena/ok", ms, h["level"] == SC.OK), + f("/viz/arena/display_rate_hz", ms, v["display_rate_hz"]), f("/viz/arena/lag_frames", ms, v["lag_frames"])] + return [o for o in out if o] + + t_disp = None + for topic, log_ns, obj in records: + if not isinstance(obj, dict): + continue + if topic == "/arena/browser": + rx = obj.get("rx_ms") + if not isinstance(rx, (int, float)): + rx = log_ns / 1e6 + # display ticks up to this record (only while a pattern plays; one image on each change) + if t_disp is not None and tr.state == "trial": + while t_disp < rx: + yield from (x for x in display_at(t_disp) if x) + t_disp += step_ms + if last["health_ms"] is not None: + while last["health_ms"] + 1000.0 <= rx: + last["health_ms"] += 1000.0 + yield from health_out(last["health_ms"]) + tr.feed_record(obj) + if last["health_ms"] is None and tr.seen_any: + last["health_ms"] = rx + yield from (x for x in stim_out(rx) if x) + if tr.state != "trial" or t_disp is None or t_disp < rx: + t_disp = rx + yield from (x for x in display_at(rx) if x) + if tr.frame_log: + off = tr.clock_offset_us() + for t_us, idx, pid, lag in tr.frame_log: + t_ms = (t_us + off) / 1000.0 if off is not None else rx + o = f("/viz/arena/observed_idx", t_ms, idx) + if o: + yield o + if lag is not None: + yield f("/viz/arena/lag_frames", t_ms, lag) + okey = (idx, pid, tr.state) + if okey != last["obs_key"] and t_ms - last["obs_ms"] >= step_ms: + last["obs_key"], last["obs_ms"] = okey, t_ms + st = tr.stimulus() + img = render(tr.pid_names.get(pid), idx, st.get("duty") or 255) if st["state"] == "trial" else AD.render_dark(scale=scale) + yield ("/arena/display_observed", ns(t_ms), image_msg(ns(t_ms), AD.png_bytes(img), "arena_display_observed"), IMAGE_SCHEMA) + tr.frame_log.clear() + elif topic == "/bias/status": + ms = obj.get("t_ms") or obj.get("rx_epoch_ms") or log_ns / 1e6 + reach, cap, logg = bool(obj.get("reachable")), bool(obj.get("capturing")), bool(obj.get("logging")) + state = "unreachable" if not reach else ("recording" if cap and logg else "preview" if cap else "idle") + outs = [f("/viz/bias/reachable", ms, reach), f("/viz/bias/capturing", ms, cap), f("/viz/bias/logging", ms, logg), + f("/viz/bias/state", ms, state), f("/viz/bias/fps", ms, obj.get("framesPerSec")), + f("/viz/bias/frame_count", ms, obj.get("frameCount")), f("/viz/bias/rtt_ms", ms, obj.get("rtt_ms")), + f("/viz/bias/log_queue", ms, obj.get("logQueueSize")), f("/viz/bias/drops", ms, bias_drops(obj))] + if isinstance(obj.get("frameCount"), (int, float)): + r = bias_rate.add(ms / 1000.0, obj["frameCount"]) + outs.append(f("/viz/bias/fps_from_count", ms, r) if r is not None else None) + yield from (o for o in outs if o) + + +def bias_clock_offset_ms(records) -> float | None: + """bridge-clock ms − BIAS seconds·1000, from the run's getTimestamp replies (median).""" + offs = [] + for topic, _, obj in records: + if topic == "/bias/reply" and isinstance(obj, dict) and obj.get("command") == "getTimestamp" and obj.get("ok"): + v = obj.get("value") + t0, t1 = obj.get("t_ms"), obj.get("rx_epoch_ms") + if isinstance(v, (int, float)) and v > 0 and isinstance(t0, (int, float)) and isinstance(t1, (int, float)): + offs.append((t0 + t1) / 2.0 - v * 1000.0) + return statistics.median(offs) if offs else None + + +def recorded_video(records) -> str | None: + """The file BIAS wrote, from the run's stopRecording reply (bias_node resolves it).""" + path = None + for topic, _, obj in records: + if topic == "/bias/reply" and isinstance(obj, dict) and obj.get("command") == "stopRecording": + art = obj.get("artifact") or {} + if art.get("path"): + path = art["path"] + return path + + +def camera_streams(records, ufmf_path, camera_hz: float = 10.0, factor: int = 2): + """Yield /camera/image messages from the run's uFMF, aligned to the bridge clock. Returns + (via StopIteration.value) a report dict.""" + from ufmf import UfmfReader, downsample + + off = bias_clock_offset_ms(records) + report = {"ufmf": str(ufmf_path), "offset_ms": off, "images": 0} + if off is None: + report["error"] = "no getTimestamp reply with a non-zero value: cannot place camera frames on the run clock" + return report + with UfmfReader(ufmf_path) as u: + report.update({"frames": u.n_frames, "width": u.width, "height": u.height, "indexed": u.indexed}) + step = 1.0 / max(0.1, camera_hz) + nxt = None + for i in range(u.n_frames): + t = float(u.timestamps[i]) + if nxt is not None and t < nxt: + continue + nxt = t + step if nxt is None else max(nxt + step, t) + ts, img = u.frame(i) + t_ns = int(round((ts * 1000.0 + off) * 1e6)) + yield ("/camera/image", t_ns, image_msg(t_ns, AD.png_bytes(downsample(img, factor)), "bias_camera"), IMAGE_SCHEMA) + report["images"] += 1 + return report + + +def run_streams(records, stimulus: dict | None, camera: dict | None) -> tuple[list, dict]: + """All derived (topic, t_ns, obj, schema) rows + a report.""" + out, report = [], {} + if stimulus is not None: + out.extend(x for x in stimulus_streams(records, **stimulus) if x) + report["stimulus_messages"] = len(out) + if camera is not None: + path = camera.get("ufmf") + if path == "auto": + path = recorded_video(records) + if path and not Path(path).is_file(): + report["camera"] = {"error": f"the run's video is not on this machine: {path}"} + path = None + if path: + gen = camera_streams(records, path, camera.get("hz", 10.0), camera.get("factor", 2)) + try: + while True: + out.append(next(gen)) + except StopIteration as stop: + report["camera"] = stop.value + elif "camera" not in report: + report["camera"] = {"error": "no uFMF given and none recorded by this run"} + return out, report diff --git a/ros2-poc/nodes/foxglove_json.py b/ros2-poc/nodes/foxglove_json.py index f973338f..73ecaaeb 100644 --- a/ros2-poc/nodes/foxglove_json.py +++ b/ros2-poc/nodes/foxglove_json.py @@ -17,6 +17,14 @@ - /fictrac/raw: `cols`, FicTrac's 25 output columns by name, parsed from `raw`; - /arena/browser: `msg`, the browser message parsed from `raw` (when it is JSON). +With `--stimulus` the live dashboard's derived topics are rebuilt too (nodes/dashboard_replay.py, +the same code the live stimulus_node / viz_node run): the expected and observed arena display as +images, the arena's rate / lag / health, BIAS's rate and losses, and — from the run's uFMF movie +(`--ufmf `, or found through the run's stopRecording reply) — `/camera/image`. Then +`foxglove/arena-okey-cokey.json` shows the run as it looked live: + + pixi run fox-json -- runlogs/.bag .foxglove.mcap --stimulus --panel-layout four-color + Slim by default: where `raw` was parsed into `cols` / `msg`, the `raw` text is dropped (it is the same data twice); `--keep-raw` keeps it. The original bag stays the record of truth. @@ -185,7 +193,14 @@ def read_messages(path: Path): return out, torn -def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False) -> dict: +IMAGE_JSON_SCHEMA = { + "title": "foxglove.CompressedImage", "type": "object", "properties": { + "timestamp": {"type": "object", "properties": {"sec": {"type": "integer"}, "nsec": {"type": "integer"}}}, + "frame_id": {"type": "string"}, "data": {"type": "string", "contentEncoding": "base64"}, "format": {"type": "string"}}} + + +def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False, stimulus: dict | None = None, + camera: dict | None = None) -> dict: from mcap.writer import CompressionType, Writer if src.is_dir(): @@ -193,6 +208,8 @@ def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False) -> dic msgs, torn = read_messages(src) deriver = FrameDeriver(ball_mm) rows, schemas, skipped = [], {}, 0 + fixed: dict[str, str] = {} # topic → well-known schema name (images) + records = [] # (topic, log_t, obj) for the dashboard replay for topic, sname, log_t, pub_t, seq, data in msgs: if sname != "std_msgs/msg/String": skipped += 1 @@ -227,6 +244,22 @@ def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False) -> dic obj = finite(obj) schemas[topic] = merge_schema(schemas.get(topic, {}), obj) rows.append((topic, log_t, pub_t, seq, json.dumps(obj, separators=(",", ":")).encode())) + if stimulus is not None or camera is not None: + records.append((topic, log_t, obj)) + + report = {} + if stimulus is not None or camera is not None: + from dashboard_replay import run_streams + + records.sort(key=lambda r: r[1]) + derived, report = run_streams(records, stimulus, camera) + for topic, t_ns, obj, schema_name in derived: + obj = finite(obj) + if schema_name: + fixed[topic] = schema_name + else: + schemas[topic] = merge_schema(schemas.get(topic, {}), obj) + rows.append((topic, t_ns, t_ns, 0, json.dumps(obj, separators=(",", ":")).encode())) sha = hashlib.sha256(src.read_bytes()).hexdigest() dst.parent.mkdir(parents=True, exist_ok=True) @@ -234,6 +267,9 @@ def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False) -> dic w = Writer(f, compression=CompressionType.ZSTD) w.start(profile="", library="ros2-poc foxglove_json.py") chan = {} + for topic in sorted(fixed): + sid = w.register_schema(name=fixed[topic], encoding="jsonschema", data=json.dumps(IMAGE_JSON_SCHEMA).encode()) + chan[topic] = w.register_channel(topic=topic, message_encoding="json", schema_id=sid) for topic in sorted(schemas): sc = finish_schema(schemas[topic]) or {"type": "object"} sc = {"$schema": "https://json-schema.org/draft/2020-12/schema", "title": topic, **sc} @@ -248,8 +284,11 @@ def convert(src: Path, dst: Path, ball_mm: float, keep_raw: bool = False) -> dic counts = {} for r in rows: counts[r[0]] = counts.get(r[0], 0) + 1 - return {"source": src.name, "messages": len(rows), "skipped": skipped, "torn": torn, "topics": counts, - "bytes": dst.stat().st_size} + out = {"source": src.name, "messages": len(rows), "skipped": skipped, "torn": torn, "topics": counts, + "bytes": dst.stat().st_size} + if report: + out["dashboard"] = report + return out def main(argv=None) -> int: @@ -258,8 +297,26 @@ def main(argv=None) -> int: ap.add_argument("dst", type=Path) ap.add_argument("--ball-mm", type=float, default=9.0, help="ball diameter for forward_mms (default 9, as the bridge)") ap.add_argument("--keep-raw", action="store_true", help="keep the `raw` text beside the parsed `cols` / `msg`") + repo = Path(__file__).resolve().parents[2] + g = ap.add_argument_group("dashboard replay (foxglove/arena-okey-cokey.json)") + g.add_argument("--stimulus", action="store_true", help="rebuild the expected/observed display, arena + BIAS health, and the camera") + g.add_argument("--panel-layout", default="g6-green", help="panel LED colors (g6-green, four-color, red-ir-v0.4r2, red-ir-v0.4r1)") + g.add_argument("--pattern-dir", action="append", default=None, help="where .pat files are (repeatable; default the repo's protocols/ + patterns/)") + g.add_argument("--protocol-dir", action="append", default=None, help="where the run's protocol YAML is (repeatable; default the repo's protocols/)") + g.add_argument("--display-hz", type=float, default=10.0, help="expected-display image rate while a pattern plays (default 10)") + g.add_argument("--scale", type=int, default=4, help="screen pixels per LED (default 4)") + g.add_argument("--ufmf", default="auto", help="the run's uFMF movie (default auto: the file its stopRecording reply names); 'none' to skip") + g.add_argument("--camera-hz", type=float, default=10.0, help="camera images per second (default 10)") + g.add_argument("--camera-factor", type=int, default=2, help="camera downsampling, N×N mean (default 2)") a = ap.parse_args(argv) - print(json.dumps(convert(a.src, a.dst, a.ball_mm, a.keep_raw))) + stim = cam = None + if a.stimulus: + stim = {"patterns": a.pattern_dir or [str(repo / "protocols"), str(repo / "patterns")], + "protocols": a.protocol_dir or [str(repo / "protocols")], "layout": a.panel_layout, + "display_hz": a.display_hz, "scale": a.scale} + if a.ufmf != "none": + cam = {"ufmf": a.ufmf, "hz": a.camera_hz, "factor": a.camera_factor} + print(json.dumps(convert(a.src, a.dst, a.ball_mm, a.keep_raw, stim, cam))) return 0 diff --git a/ros2-poc/nodes/rig.py b/ros2-poc/nodes/rig.py index 4e617ea5..ae96f168 100644 --- a/ros2-poc/nodes/rig.py +++ b/ros2-poc/nodes/rig.py @@ -7,6 +7,9 @@ pixi run rig -- --bias --bias-sim # + a simulated BIAS program (no camera software) pixi run rig -- --scanimage # + the ScanImage node (JSON server on this PC); --si-mock for a stand-in pixi run rig -- --foxglove # + foxglove_bridge (:8766) from the `viz` environment + pixi run rig -- --no-fictrac --bias --foxglove --viz-args "--sources bias" --stimulus-args "--panel-layout four-color" + # the okey-cokey camera rig: BIAS + G6, no FicTrac; + # layout foxglove/arena-okey-cokey.json (--foxglove includes --stimulus) Order: the router first (nodes retry until it answers on 127.0.0.1:7447), then every adapter/node. Each child's stderr is prefixed with its name. Ctrl-C stops everything in @@ -140,6 +143,11 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--scanimage", action="store_true", help="also run the ScanImage node (talks to ScanImage's JSON server, default 127.0.0.1:5555)") p.add_argument("--si-mock", action="store_true", help="also run tests/si_json_mock.py (a stand-in for ScanImage's JSON server, port 5555)") p.add_argument("--sim", action="store_true", help="also run the FicTrac simulator (random walk, 50 Hz)") + p.add_argument("--no-fictrac", action="store_true", + help="do not run the FicTrac UDP adapter (a rig without FicTrac, e.g. the okey-cokey camera rig)") + p.add_argument("--stimulus", action="store_true", + help="also run stimulus_node.py: the expected/observed arena display + arena health (included by --foxglove)") + p.add_argument("--stimulus-args", default="", help="extra args for stimulus_node.py (e.g. --panel-layout four-color)") p.add_argument("--viz-args", default="", help="extra args for viz_node.py (--deg-per-frame, --frame-dir, --feature-az0, --frames)") p.add_argument("--foxglove", action="store_true", help=f"also run foxglove_bridge on ws://127.0.0.1:{FOXGLOVE_PORT} (the `viz` environment; skipped with a note when it is not installed)") @@ -171,7 +179,7 @@ def main(argv: list[str] | None = None) -> int: for proto, port, who in ( ("tcp", 7447, "a Zenoh router (another `pixi run rig` / `pixi run router`)") if not args.no_router else (None, 0, ""), ("tcp", 8765, "a bridge on the Studio WebSocket port — bridge 3.4 (`pixi run bridge` at the repo root) or another rig"), - ("udp", 60000, "a FicTrac receiver — bridge 3.4 or another fictrac adapter"), + ("udp", 60000, "a FicTrac receiver — bridge 3.4 or another fictrac adapter") if not args.no_fictrac else (None, 0, ""), ("udp", 60001, "another Kinefly adapter") if args.kinefly else (None, 0, ""), ("tcp", 5010, "a BIAS program or bias_sim") if args.bias_sim else (None, 0, ""), ("tcp", 5555, "ScanImage's JSON server or si_json_mock") if args.si_mock else (None, 0, ""), @@ -193,9 +201,9 @@ def main(argv: list[str] | None = None) -> int: children: list[Child] = [] if not args.no_router: children.append(Child("router", ["ros2", "run", "rmw_zenoh_cpp", "rmw_zenohd"])) - nodes: list[Child] = [ - Child("fictrac", [py, str(NODES / "fictrac_udp_adapter.py"), *split_args(args.fictrac_args)]), - ] + nodes: list[Child] = [] + if not args.no_fictrac: + nodes.append(Child("fictrac", [py, str(NODES / "fictrac_udp_adapter.py"), *split_args(args.fictrac_args)])) if args.kinefly: nodes.append(Child("kinefly", [py, str(NODES / "kinefly_udp_adapter.py"), *split_args(args.kinefly_args)])) if args.bias_sim: @@ -208,6 +216,8 @@ def main(argv: list[str] | None = None) -> int: nodes.append(Child("scanimage", [py, str(NODES / "scanimage_node.py"), *split_args(args.scanimage_args)])) bridge = Child("bridge", [py, str(NODES / "bridge_node.py"), *split_args(args.bridge_args)]) nodes.append(bridge) + if args.stimulus or foxglove: + nodes.append(Child("stimulus", [py, str(NODES / "stimulus_node.py"), *split_args(args.stimulus_args)])) if foxglove: # The typed republisher (ros-base messages Foxglove plots natively) runs in THIS # environment; only foxglove_bridge itself needs the `viz` one. diff --git a/ros2-poc/nodes/stimulus_core.py b/ros2-poc/nodes/stimulus_core.py new file mode 100644 index 00000000..effd18dd --- /dev/null +++ b/ros2-poc/nodes/stimulus_core.py @@ -0,0 +1,511 @@ +#!/usr/bin/env python3 +"""stimulus_core.py — what the G6 arena should be showing, what it did show, and whether it keeps up. + +Pure (no rclpy): the live `stimulus_node.py` and the offline `foxglove_json.py --stimulus` feed it +the same `/arena/browser` records, so a live dashboard and a replayed bag agree. + +Inputs (every Studio → bridge message, as the bridge records it; formats: +docs/development/telemetry-logging-reference.md): + * runner status events {"type":"log","event":"runner","phase":…} + run_metadata a new run: protocol_filename (→ pattern names), telemetry level + step-start condition name; resets the per-step trialParams counter + trial-running params {mode, patternId, frameRate, initPos, duty}, durationSec — + the host's view of a trial start (stamped with the record's rx_ms) + command op=allOff the arena goes dark + sequence-start / -complete / error + * {"type":"log","event":"trial_quality"|"display_gap"} the Studio's own verdicts + * {"type":"rows","rows":[…]} controller telemetry (ring firmware, drained ~10 Hz): + ["cc", rx, t_us, seq, cmd, status, "reqhex"] cmd 8 = trialParams (req = mode u8 · + patternId u16 · frameRate i16 · initPos u16), + cmd 0 = allOff + ["cf", rx, t_us, seq, idx, pattern, sd_load_us, spi_us(, req_age_us, superseded, flags)] + ["cs", rx, t_us, seq, kind, code, arg] kind 3 = error glyph, 4 = slow SD read + +The controller rows are the ground truth. Each displayed frame (`cf`) carries the controller's +own microsecond clock, and so does the trialParams command that started the trial (`cc` cmd 8). +So the expected index, the lag and the display rate are all computed on THAT clock, not affected +by the ~100 ms batching of the drain: + + expected(t) = (init + sign(fps) · floor(|fps| · (t − t_trialParams))) mod n_frames + lag = observed − expected, wrapped into ±n/2 (0 = on schedule) + rate = cf rows in the last controller second + +For the LIVE picture (the expected display at host time "now"), the controller clock is mapped to +host epoch ms with offset = min over the last 30 s of (rx_ms − t_us/1000): `rx` is when the +browser drained the row, never before it happened, so the minimum is the drain's latency floor +(a few ms: every batch's newest row was recorded just before the read), and the live picture runs +that much behind the arena. +Without telemetry (a firmware without the ring, or before the first rows) the host trial start +(the trial-running record's rx_ms) is the anchor and observed/lag/rate are unknown — never "OK" +without evidence. + +Pattern names: the wire carries only the SD slot (`patternId`), so a trial's pattern NAME comes +from the protocol: the run's `protocol_filename` is looked up under the protocol dirs, and the +k-th trial-running of a step takes the k-th trialParams `pattern:` of that condition. Learned +slot → name pairs serve Console trials and protocols that cannot be found. +""" + +from __future__ import annotations + +import json +import math +import sys +import time +from collections import deque +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / "tools")) +from patfile import read_pat # noqa: E402 + +OK, WARN, ERROR, STALE = 0, 1, 2, 3 # diagnostic_msgs/DiagnosticStatus levels +LEVEL_NAMES = {OK: "OK", WARN: "WARN", ERROR: "ERROR", STALE: "STALE"} + +CMD_ALL_OFF = 0x00 +CMD_TRIAL_PARAMS = 0x08 +CMD_SET_FRAME_POSITION = 0x70 +# A non-ok reply to one of THESE is an arena error; other commands (SD listing, health, info +# queries) are refused routinely while a pattern plays (status 10 = display active) and are only +# counted. +DISPLAY_CMDS = (CMD_ALL_OFF, CMD_TRIAL_PARAMS, CMD_SET_FRAME_POSITION) +CS_ERROR_GLYPH = 3 +CS_SD_SLOW = 4 + +U32 = 1 << 32 + + +def wrap_diff(d: int, n: int) -> int: + """d mod n, as the nearest value in [−n/2, n/2).""" + return (d + n // 2) % n - n // 2 if n > 1 else 0 + + +def parse_trial_params_req(req_hex: str): + """The first 8 request bytes of a 0x08 command → (mode, pattern_id, fps, init) or None.""" + try: + b = bytes.fromhex(req_hex) + except (ValueError, TypeError): + return None + if len(b) < 7: + return None + mode = b[0] + pid = b[1] | (b[2] << 8) + fps = int.from_bytes(b[3:5], "little", signed=True) + init = b[5] | (b[6] << 8) + return mode, pid, fps, init + + +class PatternLibrary: + """`.pat` files by name (the file stem), searched recursively under `dirs`, first dir wins.""" + + def __init__(self, dirs) -> None: + self.dirs = [Path(d) for d in dirs if d] + self._index: dict[str, Path] | None = None + self._indexed_at = 0.0 + self._cache: dict[str, dict | None] = {} + + def _scan(self) -> None: + idx: dict[str, Path] = {} + for d in self.dirs: + if d.is_dir(): + for p in sorted(d.rglob("*.pat")): + idx.setdefault(p.stem, p) + self._index = idx + self._indexed_at = time.monotonic() + + def path(self, name: str | None) -> Path | None: + if not name: + return None + stem = Path(str(name)).name + stem = stem[:-4] if stem.lower().endswith(".pat") else stem + if self._index is None or (stem not in self._index and time.monotonic() - self._indexed_at > 10.0): + self._scan() + return self._index.get(stem) + + def get(self, name: str | None) -> dict | None: + if not name: + return None + if name in self._cache and self._cache[name] is not None: + return self._cache[name] + p = self.path(name) + pat = None + if p is not None: + try: + pat = read_pat(p) + except (OSError, ValueError): + pat = None + self._cache[name] = pat + return pat + + +class ProtocolIndex: + """condition name → [trialParams pattern names, in command order], from a v3 protocol YAML.""" + + def __init__(self, dirs) -> None: + self.dirs = [Path(d) for d in dirs if d] + self._cache: dict[str, dict] = {} + + def find(self, filename: str | None) -> Path | None: + if not filename: + return None + base = Path(str(filename)).name + for d in self.dirs: + if (d / base).is_file(): + return d / base + if d.is_dir(): + for p in d.rglob(base): + return p + return None + + def conditions(self, filename: str | None) -> dict: + if not filename: + return {} + if filename in self._cache: + return self._cache[filename] + out: dict[str, list[str]] = {} + p = self.find(filename) + if p is not None: + try: + import yaml + + doc = yaml.safe_load(p.read_text(encoding="utf-8")) or {} + for c in doc.get("conditions") or []: + if not isinstance(c, dict) or not c.get("name"): + continue + names = [str(cmd.get("pattern")) for cmd in (c.get("commands") or []) + if isinstance(cmd, dict) and cmd.get("command_name") == "trialParams" and cmd.get("pattern")] + out[str(c["name"])] = names + except Exception: # a protocol we cannot read only costs us the names + out = {} + self._cache[filename] = out + return out + + +class StimulusTracker: + def __init__(self, patterns: PatternLibrary | None = None, protocols: ProtocolIndex | None = None, + stale_s: float = 2.0, rate_tol: float = 0.05, lag_tol: int = 2) -> None: + self.patterns = patterns or PatternLibrary([]) + self.protocols = protocols or ProtocolIndex([]) + self.stale_s, self.rate_tol, self.lag_tol = stale_s, rate_tol, lag_tol + self.pid_names: dict[int, str] = {} + self.frame_log: list | None = None # set to [] to collect (t_us, idx, pid, lag) per cf row (replay) + self.reset_run() + self.seen_any = False + + # ── state ────────────────────────────────────────────────────────────────── + def reset_run(self) -> None: + self.run: dict = {} + self.run_open = False + self.telemetry_level = None + self.condition = None + self.step_tp = 0 + self.proto: dict = {} + self.state = "idle" # idle | dark | trial + self.trial: dict | None = None # host view (runner) + self.ctrl: dict | None = None # controller view (cc cmd 8): {mode, pid, fps, init, t0} + self.ctrl_dark_t = None + self.obs = None # last cf: {idx, pid, t} + self.cf_times: deque = deque() + self.lag = None + self.rate_hz = None + self.rows = 0 + self.last_rows_ms = None + self._t_last_raw = None + self._t_base = 0 + self._offsets: deque = deque() # (rx_ms, rx_ms*1000 − t_us) + self.max_idx: dict[int, int] = {} + self.refused_other = 0 + self.last_cf_ms = None + self.errors: list[tuple[float, str]] = [] + self.warnings: list[tuple[float, str]] = [] + self.flagged = 0 + self.trials_judged = 0 + self.last_quality = None + self.changed = True + + # ── feeding ──────────────────────────────────────────────────────────────── + def feed_record(self, rec: dict) -> None: + """A bridge `/arena/browser` record: {rx_ms, raw, …} (raw = the browser message text).""" + raw = rec.get("raw") + try: + msg = json.loads(raw) if isinstance(raw, str) else rec.get("msg") + except json.JSONDecodeError: + return + if isinstance(msg, dict): + self.feed(msg, rec.get("rx_ms")) + + def feed(self, msg: dict, rx_ms) -> None: + self.seen_any = True + rx_ms = float(rx_ms) if isinstance(rx_ms, (int, float)) else time.time() * 1000.0 + kind = msg.get("type") + if kind == "rows": + self._rows(msg.get("rows") or [], rx_ms) + return + if kind != "log": + return + ev = msg.get("event") + if ev == "run_metadata": + pid_names = self.pid_names + self.reset_run() + self.pid_names = pid_names + self.run = {k: msg.get(k) for k in ("run_id", "protocol_filename", "rig", "arena_config", "telemetry", "firmware")} + self.telemetry_level = msg.get("telemetry") + self.proto = self.protocols.conditions(msg.get("protocol_filename")) + self.run_open = True + self.state = "dark" + self.changed = True + elif ev == "runner": + self._runner(msg, rx_ms) + elif ev == "trial_quality": + trials = msg.get("trials") or [] + self.trials_judged = len(trials) + self.flagged = sum(1 for t in trials if t.get("status") == "flagged") + self.last_quality = trials[-1].get("status") if trials else None + if self.flagged: + self.warnings.append((rx_ms, f"{self.flagged} of {len(trials)} trials flagged by trial_quality")) + elif ev == "display_gap": + self.warnings.append((rx_ms, f"display gap ({msg.get('kind') or 'see the run log'}) in trial {msg.get('trial')}")) + + def _runner(self, m: dict, rx_ms: float) -> None: + phase = m.get("phase") + cond = m.get("condition") or m.get("conditionName") + if phase == "sequence-start": + self.run_open = True + if self.state == "idle": + self.state = "dark" + elif phase in ("sequence-complete", "sequence-aborted", "aborted", "stopped"): + self.run_open = False + summ = m.get("summary") or {} + if summ.get("fault") or summ.get("errors"): + self.errors.append((rx_ms, f"sequence ended: errors {summ.get('errors')}, fault {summ.get('fault')}")) + elif phase == "step-start": + self.condition = cond + self.step_tp = 0 + elif phase == "trial-running": + p = m.get("params") or {} + k = self.step_tp + self.step_tp += 1 + names = self.proto.get(cond or self.condition) or [] + pid = p.get("patternId") + name = names[k] if k < len(names) else (names[-1] if names else None) + if name is None and isinstance(pid, int): + name = self.pid_names.get(pid) + if name is not None and isinstance(pid, int): + self.pid_names[pid] = name + duty = p.get("duty") + self.trial = { + "condition": cond or self.condition, "pattern": name, "pattern_id": pid, + "mode": p.get("mode"), "fps": p.get("frameRate") or 0, "init": p.get("initPos") or 0, + "duty": 255 if not duty else int(duty), "duration_s": m.get("durationSec"), "t_host_ms": rx_ms, + } + self.state = "trial" + self.lag = None + self.changed = True + if m.get("ok") is False or (m.get("status") not in (None, 0)): + self.errors.append((rx_ms, f"trialParams status {m.get('status')} ({cond})")) + elif phase == "command": + if m.get("op") == "allOff": + self.state = "dark" + self.trial = None + self.changed = True + v = m.get("value") + if isinstance(v, dict) and v.get("ok") is False and m.get("op") != "connectorCommand": + self.errors.append((rx_ms, f"{m.get('op')} failed ({cond})")) + elif phase == "error": + self.errors.append((rx_ms, str(m.get("message") or "runner error"))) + + def _unwrap(self, t_us: int) -> int | None: + if not isinstance(t_us, (int, float)): + return None + t = int(t_us) + last = self._t_last_raw + if last is not None: + d = t - last + if d < -(U32 // 2): + self._t_base += U32 + elif -(U32 // 2) <= d < -2_000_000: # backwards by > 2 s: the controller rebooted + self._t_base = 0 + self._offsets.clear() + self.cf_times.clear() + self.ctrl = None + self._t_last_raw = t + return self._t_base + t + + def _rows(self, rows: list, rx_ms: float) -> None: + self.last_rows_ms = rx_ms + batch_min = None + for r in rows: + if not isinstance(r, list) or len(r) < 7 or not isinstance(r[0], str): + continue + tag = r[0] + t = self._unwrap(r[2]) + if t is None: + continue + self.rows += 1 + row_rx = r[1] if isinstance(r[1], (int, float)) else rx_ms + off = row_rx * 1000.0 - t + batch_min = off if batch_min is None else min(batch_min, off) + if tag == "cc": + cmd, status = r[4], r[5] + if cmd == CMD_TRIAL_PARAMS and status == 0: + tp = parse_trial_params_req(r[6]) + if tp: + mode, pid, fps, init = tp + self.ctrl = {"mode": mode, "pid": pid, "fps": fps, "init": init, "t0": t} + self.ctrl_dark_t = None + self.lag = None + elif cmd == CMD_ALL_OFF and status == 0: + self.ctrl = None + self.ctrl_dark_t = t + elif status not in (0, None): + if cmd in DISPLAY_CMDS: + self.errors.append((rx_ms, f"controller refused cmd 0x{int(cmd):02X} (status {status})")) + else: + self.refused_other += 1 + elif tag == "cf": + idx, pid = r[4], r[5] + self.obs = {"idx": idx, "pid": pid, "t": t} + self.last_cf_ms = rx_ms + if isinstance(pid, int) and isinstance(idx, int): + self.max_idx[pid] = max(self.max_idx.get(pid, 0), idx) + self.cf_times.append(t) + while self.cf_times and self.cf_times[0] <= t - 1_000_000: + self.cf_times.popleft() + self._judge_frame(idx, pid, t) + if self.frame_log is not None: + self.frame_log.append((t, idx, pid, self.lag)) + elif tag == "cs": + kind = r[4] + if kind == CS_ERROR_GLYPH: + self.errors.append((rx_ms, f"controller error glyph (code {r[5]}, arg {r[6]})")) + elif kind == CS_SD_SLOW: + us = r[6] * 100 if isinstance(r[6], int) else None + self.warnings.append((rx_ms, f"slow SD read: {us / 1000:.1f} ms" if us is not None else "slow SD read")) + if batch_min is not None: + self._offsets.append((rx_ms, batch_min)) + while self._offsets and self._offsets[0][0] < rx_ms - 30_000: + self._offsets.popleft() + if self.cf_times: + self.rate_hz = (len(self.cf_times) - 1) / max(1e-6, (self.cf_times[-1] - self.cf_times[0]) / 1e6) if len(self.cf_times) > 1 else 0.0 + + # ── controller-clock judgments ───────────────────────────────────────────── + def n_frames(self, pid=None, name=None) -> int | None: + pat = self.patterns.get(name) if name else None + if pat: + return int(pat["num_frames"]) + if pid is not None and pid in self.max_idx: + return self.max_idx[pid] + 1 + return None + + def _ctrl_expected(self, t: int) -> int | None: + c = self.ctrl + if not c or c["mode"] != 2 or t < c["t0"]: + return None + name = self.pid_names.get(c["pid"]) + n = self.n_frames(c["pid"], name) + steps = math.floor(abs(c["fps"]) * (t - c["t0"]) / 1e6 + 1e-9) + e = c["init"] + (steps if c["fps"] >= 0 else -steps) + return e % n if n else e + + def _judge_frame(self, idx, pid, t: int) -> None: + c = self.ctrl + if not c or pid != c["pid"] or c["mode"] != 2: + return + exp = self._ctrl_expected(t) + n = self.n_frames(pid, self.pid_names.get(pid)) + if exp is None or not n or not isinstance(idx, int): + return + self.lag = wrap_diff(idx - exp, n) + + def clock_offset_us(self) -> float | None: + return min(o for _, o in self._offsets) if self._offsets else None + + # ── outputs ──────────────────────────────────────────────────────────────── + def stimulus(self) -> dict: + t = self.trial + out = {"state": self.state, "run_id": self.run.get("run_id"), "protocol": self.run.get("protocol_filename"), + "condition": self.condition} + if self.state == "trial" and t: + n = self.n_frames(t["pattern_id"], t["pattern"]) + out.update({k: t[k] for k in ("pattern", "pattern_id", "mode", "fps", "init", "duty", "duration_s", "t_host_ms")}) + out["condition"] = t["condition"] + out["n_frames"] = n + out["pattern_found"] = self.patterns.path(t["pattern"]) is not None + return out + + def expected_index(self, now_ms: float) -> int | None: + """The frame the arena should show at host epoch ms `now_ms` (None when dark/unknown).""" + t = self.trial + if self.state != "trial" or not t: + return None + n = self.n_frames(t["pattern_id"], t["pattern"]) + if t["mode"] != 2: + return self.obs["idx"] if self.obs and self.obs.get("pid") == t["pattern_id"] else (t["init"] % n if n else t["init"]) + off = self.clock_offset_us() + c = self.ctrl + if c and off is not None and c["pid"] == t["pattern_id"] and c["fps"] == t["fps"]: + e = self._ctrl_expected(int(now_ms * 1000.0 - off)) + if e is not None: + return e + dt = max(0.0, (now_ms - t["t_host_ms"]) / 1000.0) + steps = math.floor(abs(t["fps"]) * dt + 1e-9) + e = t["init"] + (steps if t["fps"] >= 0 else -steps) + return e % n if n else e + + def observed_index(self): + return self.obs["idx"] if self.obs else None + + def has_telemetry(self) -> bool: + return self.rows > 0 + + def health(self, now_ms: float | None = None) -> dict: + now_ms = time.time() * 1000.0 if now_ms is None else now_ms + age_s = None if self.last_rows_ms is None else max(0.0, (now_ms - self.last_rows_ms) / 1000.0) + frame_age_s = None + if self.last_cf_ms is not None or self.trial: + since = max(self.last_cf_ms or 0.0, (self.trial or {}).get("t_host_ms") or 0.0) + frame_age_s = max(0.0, (now_ms - since) / 1000.0) + recent = lambda lst, s=10.0: [txt for (t, txt) in lst if now_ms - t < s * 1000.0] + t = self.trial + rotating = self.state == "trial" and t and t.get("mode") == 2 and (t.get("fps") or 0) != 0 + expected_rate = abs(t["fps"]) if rotating else 0.0 + c = self.ctrl + since_start_s = None + if c and self.obs and c["pid"] == (t or {}).get("pattern_id"): + since_start_s = (self.obs["t"] - c["t0"]) / 1e6 + values = { + "state": self.state, "condition": (t or {}).get("condition") or self.condition or "", + "pattern": (t or {}).get("pattern") or "", "fps": (t or {}).get("fps", 0), "duty": (t or {}).get("duty", ""), + "expected_rate_hz": expected_rate, "display_rate_hz": self.rate_hz, "lag_frames": self.lag, + "observed_idx": self.observed_index(), "telemetry_rows": self.rows, + "telemetry_age_s": None if age_s is None else round(age_s, 2), + "other_cmds_refused": self.refused_other, "trials_flagged": self.flagged, "trials_judged": self.trials_judged, + "clock_offset_ms": None if self.clock_offset_us() is None else round(self.clock_offset_us() / 1000.0, 1), + } + errs, warns = recent(self.errors), recent(self.warnings) + if not self.seen_any: + level, msg = WARN, "no Studio messages yet" + elif errs: + level, msg = ERROR, errs[-1] + elif rotating and self.rows and frame_age_s is not None and frame_age_s > self.stale_s: + # only while a pattern plays: in the dark the ring has nothing to send + level, msg = STALE, f"no displayed-frame telemetry for {frame_age_s:.0f} s" + elif self.run_open and not self.rows: + level, msg = WARN, ("no controller telemetry yet" if str(self.telemetry_level or "").startswith("ring") + else "no controller telemetry on this run (expected display only)") + elif rotating and since_start_s is not None and since_start_s > 1.2 and self.rate_hz is not None \ + and abs(self.rate_hz - expected_rate) > max(1.0, self.rate_tol * expected_rate): + level, msg = WARN, f"display rate {self.rate_hz:.1f} Hz, expected {expected_rate:.0f} Hz" + elif rotating and self.lag is not None and abs(self.lag) > self.lag_tol: + level, msg = WARN, f"display {self.lag:+d} frames off schedule" + elif warns: + level, msg = WARN, warns[-1] + elif self.state == "trial" and t: + msg = f"{t.get('pattern') or 'pattern ' + str(t.get('pattern_id'))} @ {t.get('fps')} fps" + if self.rate_hz is not None and rotating: + msg += f" · shown {self.rate_hz:.0f} Hz" + if self.lag is not None: + msg += f" · lag {self.lag:+d}" + level = OK + else: + level, msg = OK, self.state + return {"level": level, "level_name": LEVEL_NAMES[level], "message": msg, "values": values} diff --git a/ros2-poc/nodes/stimulus_node.py b/ros2-poc/nodes/stimulus_node.py new file mode 100644 index 00000000..1f1d2631 --- /dev/null +++ b/ros2-poc/nodes/stimulus_node.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""stimulus_node.py — the arena's EXPECTED display, its OBSERVED display and its health, live. + +Subscribes to `/arena/browser` (every Studio message, published live by bridge_node) and runs +`stimulus_core.StimulusTracker` over it; publishes standard ROS messages Foxglove shows natively: + + sensor_msgs/CompressedImage (PNG) + /arena/display what the arena SHOULD show now: the trial's .pat frame at the + expected index × duty, in the panels' LED colors (--panel-layout), + top of the arena at the top; dark between trials + /arena/display_observed the frame the controller reported it put up (telemetry `cf` rows, + drained ~10 Hz, so it steps and trails by ~0.1 s) + std_msgs/Float64 /viz/arena/expected_idx (at --display-hz), /viz/arena/observed_idx + /viz/arena/lag_frames + (one per displayed frame the controller reports; lag = observed − expected on the + controller clock, 0 = on schedule), /viz/arena/display_rate_hz (frame changes per + controller second, 1 Hz), /viz/arena/cmd_fps + std_msgs/Bool /viz/arena/ok (health level OK) — for an Indicator panel + std_msgs/String /viz/arena/state ("idle|dark|trial"), /viz/arena/condition, /viz/arena/health + (": "), /arena/stimulus (JSON: condition, pattern, mode, fps, + duty, n_frames, … — on change and every second) + diagnostic_msgs/DiagnosticArray /diagnostics, status "arena/display" (1 Hz): OK / WARN (rate off + by > 5 %, lag > 2 frames, a flagged trial, a slow SD read) / ERROR (a refused + display command, an error glyph) / STALE (no frame telemetry while a pattern plays) + +Nothing here is logged or fed back. Patterns are found by name under --pattern-dir (recursive); +pattern names come from the run's protocol under --protocol-dir (see stimulus_core). + + pixi run stimulus -- --panel-layout four-color +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import rclpy +from diagnostic_msgs.msg import DiagnosticArray, DiagnosticStatus, KeyValue +from rclpy.node import Node +from sensor_msgs.msg import CompressedImage +from std_msgs.msg import Bool, Float64, String + +from _common import RELIABLE_QOS, spin, wall_ms +import arena_display as AD +import stimulus_core as SC + +REPO = Path(__file__).resolve().parents[2] + + +class StimulusNode(Node): + def __init__(self, a: argparse.Namespace) -> None: + super().__init__("arena_stimulus") + self.a = a + self.tr = SC.StimulusTracker(SC.PatternLibrary(a.pattern_dir), SC.ProtocolIndex(a.protocol_dir), stale_s=a.stale_s) + self.tr.frame_log = [] # every displayed frame → /viz/arena/observed_idx + lag_frames, as the replay + self.p_img = self.create_publisher(CompressedImage, "/arena/display", 10) + self.p_obs = self.create_publisher(CompressedImage, "/arena/display_observed", 10) + f64 = lambda t: self.create_publisher(Float64, t, 10) + self.p = {k: f64(f"/viz/arena/{k}") for k in ("expected_idx", "observed_idx", "display_rate_hz", "lag_frames", "cmd_fps")} + self.p_ok = self.create_publisher(Bool, "/viz/arena/ok", 10) + self.p_state = self.create_publisher(String, "/viz/arena/state", 10) + self.p_cond = self.create_publisher(String, "/viz/arena/condition", 10) + self.p_health = self.create_publisher(String, "/viz/arena/health", 10) + self.p_stim = self.create_publisher(String, "/arena/stimulus", 10) + self.p_diag = self.create_publisher(DiagnosticArray, "/diagnostics", 10) + self.create_subscription(String, "/arena/browser", self._browser, RELIABLE_QOS) + self._last_disp = None + self._last_disp_t = 0.0 + self._last_obs = None + self._last_stim = None + self.create_timer(1.0 / max(1.0, a.display_hz), self._display) + self.create_timer(1.0, self._slow) + self.get_logger().info(f"stimulus up: /arena/display (+_observed), /viz/arena/*, /diagnostics; layout {a.panel_layout}; " + f"patterns {', '.join(str(d) for d in a.pattern_dir)}") + + # ── helpers ───────────────────────────────────────────────────────────────── + def _f(self, key: str, v) -> None: + if isinstance(v, (int, float)): + m = Float64() + m.data = float(v) + self.p[key].publish(m) + + def _s(self, pub, text: str) -> None: + m = String() + m.data = text + pub.publish(m) + + def _image(self, pub, img, frame_id: str) -> None: + m = CompressedImage() + m.header.stamp = self.get_clock().now().to_msg() + m.header.frame_id = frame_id + m.format = "png" + m.data = AD.png_bytes(img) + pub.publish(m) + + def _render(self, pattern: str | None, idx, duty) -> "AD.np.ndarray": + pat = self.tr.patterns.get(pattern) if pattern else None + if pat is None: + return AD.render_placeholder(scale=self.a.scale) + n = pat["num_frames"] + return AD.render_frame(pat["frames"][int(idx) % n], pat["gs"], self.a.panel_layout, duty, self.a.scale) + + # ── callbacks ─────────────────────────────────────────────────────────────── + def _browser(self, msg: String) -> None: + try: + rec = json.loads(msg.data) + except json.JSONDecodeError: + return + if isinstance(rec, dict): + self.tr.feed_record(rec) + for _, idx, _, lag in self.tr.frame_log: + self._f("observed_idx", idx) + if lag is not None: + self._f("lag_frames", lag) + self.tr.frame_log.clear() + + def _display(self) -> None: + tr = self.tr + now = float(wall_ms()) + st = tr.stimulus() + exp = tr.expected_index(now) + key = (st["state"], st.get("pattern"), exp, st.get("duty")) + if key != self._last_disp or time.monotonic() - self._last_disp_t > 1.0: + if st["state"] == "trial" and exp is not None: + img = self._render(st.get("pattern"), exp, st.get("duty") or 255) + else: + img = AD.render_dark(scale=self.a.scale) + self._image(self.p_img, img, "arena_display") + self._last_disp, self._last_disp_t = key, time.monotonic() + self._f("expected_idx", exp if exp is not None else -1) + obs = tr.obs + okey = (obs or {}).get("idx"), (obs or {}).get("pid"), st["state"] + if obs and okey != self._last_obs: + name = tr.pid_names.get(obs["pid"]) + duty = (st.get("duty") or 255) if st["state"] == "trial" else 255 + img = self._render(name, obs["idx"], duty) if st["state"] == "trial" else AD.render_dark(scale=self.a.scale) + self._image(self.p_obs, img, "arena_display_observed") + self._last_obs = okey + if tr.changed: + tr.changed = False + self._stimulus(st) + + def _stimulus(self, st: dict) -> None: + self._s(self.p_stim, json.dumps(st, separators=(",", ":"))) + self._s(self.p_state, st["state"]) + self._s(self.p_cond, st.get("condition") or "") + self._f("cmd_fps", st.get("fps", 0) if st["state"] == "trial" else 0) + self._last_stim = st + + def _slow(self) -> None: + tr = self.tr + h = tr.health(float(wall_ms())) + v = h["values"] + self._f("display_rate_hz", v["display_rate_hz"]) + ok = Bool() + ok.data = h["level"] == SC.OK + self.p_ok.publish(ok) + self._s(self.p_health, f"{h['level_name']}: {h['message']}") + self._stimulus(tr.stimulus()) # once a second for late joiners (StateTransitions dedups) + arr = DiagnosticArray() + arr.header.stamp = self.get_clock().now().to_msg() + s = DiagnosticStatus() + s.name, s.hardware_id = "arena/display", "g6_controller" + s.level = bytes([h["level"]]) + s.message = h["message"] + s.values = [KeyValue(key=k, value="" if x is None else str(x)) for k, x in v.items()] + arr.status.append(s) + self.p_diag.publish(arr) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--pattern-dir", action="append", default=None, + help="where to find .pat by name, searched recursively; repeatable (default: the repo's protocols/ and patterns/)") + p.add_argument("--protocol-dir", action="append", default=None, + help="where to find the run's protocol YAML (for pattern names); repeatable (default: the repo's protocols/)") + p.add_argument("--panel-layout", default="g6-green", choices=sorted(AD.LAYOUTS), + help="LED colors of the panels (js/panel-color.js layouts; default g6-green; the okey-cokey arena is four-color)") + p.add_argument("--display-hz", type=float, default=15.0, help="expected-display update rate (default 15)") + p.add_argument("--scale", type=int, default=4, help="screen pixels per LED (default 4 → 800×160 for a 2×10)") + p.add_argument("--stale-s", type=float, default=2.0, help="no frame telemetry this long while a pattern plays → STALE (default 2)") + args, ros_args = p.parse_known_args(argv) + args.pattern_dir = args.pattern_dir or [str(REPO / "protocols"), str(REPO / "patterns")] + args.protocol_dir = args.protocol_dir or [str(REPO / "protocols")] + rclpy.init(args=ros_args) + node = StimulusNode(args) + spin(node) + print("[stimulus] stopped", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ros2-poc/nodes/viz_node.py b/ros2-poc/nodes/viz_node.py index b1a10ea4..48ef82c3 100644 --- a/ros2-poc/nodes/viz_node.py +++ b/ros2-poc/nodes/viz_node.py @@ -20,13 +20,20 @@ /viz/arena/bias_deg live bias angle when a waveform runs (else 0) /viz/kinefly/left_deg, right_deg, lmr_deg, wbf_hz /viz/bias/frame_count, /viz/bias/fps + /viz/bias/fps_from_count frameCount slope over the last 2 s of status polls — an estimate + independent of BIAS's own framesPerSec + /viz/bias/drops frames BIAS reports lost so far: frameIdGaps + cameraSkipped + + incompleteFrames + loggerSkipped + writerSkipped (the atlas-bench + Atlas10 build; absent keys = not reported, never 0) + /viz/bias/rtt_ms, /viz/bias/log_queue status-poll round trip; frames queued for the writer std_msgs/Bool /viz/bias/reachable, /viz/bias/capturing, /viz/bias/logging std_msgs/String /viz/bias/state ("unreachable|idle|preview|recording") — StateTransitions panel /viz/last_error (" : ") diagnostic_msgs/DiagnosticArray /diagnostics (1 Hz): one status per source — fictrac, kinefly, - bias, arena — OK (data flowing), STALE (seen, then silent > --stale-s), - WARN (never seen), ERROR (BIAS unreachable / last reply failed); values carry - rate_hz, messages, last_seq, adapter→viz latency, last_error. + bias, arena (`--sources` picks which) — OK (data flowing), STALE (seen, then + silent > --stale-s), WARN (never seen; for BIAS also: frames dropped while + recording in the last 10 s), ERROR (BIAS unreachable / last reply failed); + values carry rate_hz, messages, last_seq, adapter→viz latency, last_error. visualization_msgs/MarkerArray /viz/markers (--markers-hz, default 20): the ball (sphere + markings rotated by FicTrac's lab-frame orientation, cols 12–14), the fly (arrow, fixed, facing +X), the arena (translucent cylinder) and the pattern @@ -56,6 +63,7 @@ from visualization_msgs.msg import Marker, MarkerArray from _common import RELIABLE_QOS, spin +from bias_health import CountRate, bias_drops BALL_R = 1.0 ARENA_R = 3.2 @@ -189,6 +197,7 @@ def __init__(self, a: argparse.Namespace) -> None: super().__init__("arena_viz") self.a = a self.src = {k: Source(k) for k in ("fictrac", "kinefly", "bias", "arena")} + self.diag_sources = [k for k in (a.sources or "").split(",") if k in self.src] or list(self.src) f64 = lambda t: self.create_publisher(Float64, t, 10) self.p = { "heading": f64("/viz/fictrac/heading_deg"), @@ -209,6 +218,10 @@ def __init__(self, a: argparse.Namespace) -> None: "wbf": f64("/viz/kinefly/wbf_hz"), "frames": f64("/viz/bias/frame_count"), "fps": f64("/viz/bias/fps"), + "fps_count": f64("/viz/bias/fps_from_count"), + "drops": f64("/viz/bias/drops"), + "rtt": f64("/viz/bias/rtt_ms"), + "log_queue": f64("/viz/bias/log_queue"), } self.p_bool = { "reachable": self.create_publisher(Bool, "/viz/bias/reachable", 10), @@ -241,6 +254,9 @@ def __init__(self, a: argparse.Namespace) -> None: self.bias_deg = 0.0 self.bias_active = False self.bias_state = "unreachable" + self.bias_rate = CountRate() + self.bias_drops = None + self.bias_drop_t = None # time.time() of the last increase while recording self.create_timer(1.0, self._diagnostics) self.create_timer(1.0 / max(1.0, a.markers_hz), self._markers) self.get_logger().info("typed republisher up: /viz/*, /diagnostics, /viz/markers (Foxglove via foxglove_bridge :8766)") @@ -355,6 +371,21 @@ def _bias_status(self, msg: String) -> None: self._pub("frames", st["frameCount"]) if isinstance(st.get("framesPerSec"), (int, float)): self._pub("fps", st["framesPerSec"]) + if isinstance(st.get("frameCount"), (int, float)) and isinstance(st.get("t_ms"), (int, float)): + r = self.bias_rate.add(st["t_ms"] / 1000.0, st["frameCount"]) + if r is not None: + self._pub("fps_count", r) + drops = bias_drops(st) + if drops is not None: + if self.bias_drops is not None and drops > self.bias_drops and logging_: + self.bias_drop_t = time.time() + self._note_error("bias", f"{drops - self.bias_drops:.0f} frame(s) dropped (total {drops:.0f})") + self.bias_drops = drops + self._pub("drops", drops) + if isinstance(st.get("rtt_ms"), (int, float)): + self._pub("rtt", st["rtt_ms"]) + if isinstance(st.get("logQueueSize"), (int, float)): + self._pub("log_queue", st["logQueueSize"]) state = "unreachable" if not reachable else ("recording" if capturing and logging_ else "preview" if capturing else "idle") self.bias_state = state # Published on EVERY status tick (1 Hz), not only on change, so a dashboard that @@ -397,6 +428,8 @@ def _diagnostics(self) -> None: for s in self.src.values(): rate = s.tick_rate() self._pub("ft_rate", rate) if s.name == "fictrac" else None + if s.name not in self.diag_sources: + continue st = DiagnosticStatus() st.name = f"arena/{s.name}" st.hardware_id = s.name @@ -406,6 +439,8 @@ def _diagnostics(self) -> None: st.level, st.message = DiagnosticStatus.STALE, f"silent {now - s.last_mono:.0f} s" elif s.name == "bias" and self.bias_state == "unreachable": st.level, st.message = DiagnosticStatus.ERROR, "BIAS unreachable" + elif s.name == "bias" and self.bias_drop_t and time.time() - self.bias_drop_t < 10: + st.level, st.message = DiagnosticStatus.WARN, s.last_error or "frames dropped" elif s.name == "bias" and s.last_error_t and time.time() - s.last_error_t < 10: st.level, st.message = DiagnosticStatus.ERROR, s.last_error or "error" else: @@ -498,6 +533,8 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--markers-hz", type=float, default=20.0, help="3D marker update rate (default: 20)") p.add_argument("--stale-s", type=float, default=2.0, help="seconds of silence before a source reads STALE (default: 2)") p.add_argument("--frame-id", default="arena", help="fixed frame for the markers (default: arena)") + p.add_argument("--sources", default="fictrac,kinefly,bias,arena", + help="sources that get a /diagnostics status (default: all four; the okey-cokey rig: bias)") args, ros_args = p.parse_known_args(argv) rclpy.init(args=ros_args) node = VizNode(args) diff --git a/ros2-poc/pixi.toml b/ros2-poc/pixi.toml index fbec7aeb..3cb29b8f 100644 --- a/ros2-poc/pixi.toml +++ b/ros2-poc/pixi.toml @@ -164,6 +164,12 @@ graph = "python nodes/graph_dump.py" viz = "python nodes/viz_node.py" test-viz = "python tests/test_viz_node.py" fox-json = "python nodes/foxglove_json.py" +# The camera + G6 dashboard (foxglove/arena-okey-cokey.json): the live expected/observed display + +# arena health node (`pixi run rig -- --stimulus`), and its unit test (pure; also runs the JS +# panel-color module under the root env's Node to pin the Python port) +stimulus = "python nodes/stimulus_node.py" +test-stimulus = "python tests/test_stimulus.py" +test-stimulus-live = "python tests/test_stimulus_live.py" # after-the-fact Foxglove MCAP of a 2P color-sweep run from its ScanImage TIFFs + the arena .pat files (tools/tiff_run_to_mcap.py) tiff2mcap = "python tools/tiff_run_to_mcap.py" # `viz` environment only (`pixi run -e viz …`): the interactive views. diff --git a/ros2-poc/tests/fixtures/protocols-2026-10-14/g6_color_optomotor_smoke.yaml b/ros2-poc/tests/fixtures/protocols-2026-10-14/g6_color_optomotor_smoke.yaml new file mode 100644 index 00000000..836d6c2d --- /dev/null +++ b/ros2-poc/tests/fixtures/protocols-2026-10-14/g6_color_optomotor_smoke.yaml @@ -0,0 +1,25 @@ +version: 3 +# The two conditions of the 2026-10-14 smoke run that tests/fixtures/okey_smoke_violet_cw.jsonl.gz +# covers (the protocol had one direction per condition then), cut to what the pattern-name lookup +# in nodes/stimulus_core.py reads. +conditions: + - name: "violet_cw_d100" + commands: + - type: "controller" + command_name: "trialParams" + pattern: "G6_2x10_toprow_grat20_violet" + pattern_ID: 1 + duration: 10 + mode: 2 + frame_index: 0 + frame_rate: 80 + gain: 0 + duty: 255 + - type: "wait" + duration: 10 + - name: "dark_iti" + commands: + - type: "controller" + command_name: "allOff" + - type: "wait" + duration: 2 diff --git a/ros2-poc/tests/test_stimulus.py b/ros2-poc/tests/test_stimulus.py new file mode 100644 index 00000000..58a50fd7 --- /dev/null +++ b/ros2-poc/tests/test_stimulus.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""test_stimulus.py — the camera + G6 dashboard's pure parts, no ROS graph needed. + + pixi run test-stimulus + + * arena_display: the Python port of js/panel-color.js, pinned to the JS module cell by cell + (run under the root env's Node; SKIP without it), the renderer (orientation, duty, off LEDs), + the PNG encoder; + * stimulus_core: the expected/observed frame, lag, display rate and health on synthetic rows + (CW → CCW pair, u32 clock wrap, a skipped frame, refusals, an error glyph, staleness) and on + a REAL trial (tests/fixtures/okey_smoke_violet_cw.jsonl.gz: the 2026-10-14 smoke run's violet + CW trial on the 2×10 four-color bench arena, 80 fps, ring telemetry); + * bias_health; tools/ufmf.py on a synthetic BIAS-layout movie (indexed, unindexed, torn); + * foxglove_json.py --stimulus end to end on a small MCAP bag built from the fixture. +""" + +from __future__ import annotations + +import gzip +import json +import os +import shutil +import struct +import subprocess +import sys +import tempfile +import zlib +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +REPO = ROOT.parent +sys.path.insert(0, str(ROOT / "nodes")) +sys.path.insert(0, str(ROOT / "tools")) + +import arena_display as AD # noqa: E402 +import stimulus_core as SC # noqa: E402 +from bias_health import CountRate, bias_drops # noqa: E402 +from ufmf import UfmfReader # noqa: E402 + +failures = 0 +PAT_DIR = REPO / "protocols" / "g6_color_optomotor_test_patterns" +VIOLET = "G6_2x10_toprow_grat20_violet" +FIXTURE = HERE / "fixtures" / "okey_smoke_violet_cw.jsonl.gz" +FIXTURE_PROTOCOLS = HERE / "fixtures" / "protocols-2026-10-14" + + +def check(name: str, ok: bool, detail: str = "") -> None: + global failures + failures += not ok + print(f"{'PASS' if ok else 'FAIL'} {name}" + (f" ({detail})" if detail and not ok else "")) + + +def skip(name: str, why: str) -> None: + print(f"SKIP {name} ({why})") + + +def find_node() -> str | None: + n = shutil.which("node") + if n: + return n + for root in (REPO, *REPO.parents[:3]): + env = root / ".pixi" / "envs" / "default" + for cand in ((env / "node.exe", env / "bin" / "node.exe") if os.name == "nt" else (env / "bin" / "node",)): + if cand.exists(): + return str(cand) + return None + + +# ── 1. panel colors: the Python port == js/panel-color.js ───────────────────── +node = find_node() +if node is None: + skip("panel colors match js/panel-color.js", "no node: run `pixi install` at the repo root") +else: + js = f""" +const P = require({json.dumps(str(REPO / 'js' / 'panel-color.js'))}); +const out = {{flip: P.ROW_PARITY_FLIP, off: P.OFF_CSS, layouts: {{}}, channels: {{}}}}; +for (const k of P.layoutKeys()) {{ + const cells = []; + for (let r = 0; r < 4; r++) for (let c = 0; c < 4; c++) + cells.push([P.bankAt(r, c), P.channelAt(k, r, c).id, P.pixelCss(k, r, c, 1), P.pixelCss(k, r, c, 7 / 15)]); + out.layouts[k] = {{banks: P.getLayout(k).banks, cells}}; +}} +for (const [id, ch] of Object.entries(P.CHANNELS)) out.channels[id] = ch.rgb; +console.log(JSON.stringify(out)); +""" + res = subprocess.run([node, "-e", js], capture_output=True, text=True, timeout=30) + if res.returncode != 0: + check("js/panel-color.js runs under node", False, res.stderr.strip()[-300:]) + else: + ref = json.loads(res.stdout) + check("ROW_PARITY_FLIP equals the JS module's", ref["flip"] == AD.ROW_PARITY_FLIP, f"js {ref['flip']}") + check("the OFF color equals the JS OFF_CSS", ref["off"].lower() == "#%02x%02x%02x" % AD.OFF_RGB, ref["off"]) + check("same layout keys", sorted(ref["layouts"]) == sorted(AD.LAYOUTS), f"js {sorted(ref['layouts'])}") + check("same channel colors", all(tuple(ref["channels"][k]) == AD.CHANNEL_RGB.get(k) for k in ref["channels"]), + json.dumps(ref["channels"])) + bad = [] + for key, L in ref["layouts"].items(): + if tuple(L["banks"]) != AD.LAYOUTS.get(key): + bad.append(f"{key} banks") + continue + for i, (bank, ch, full, seven) in enumerate(L["cells"]): + r, c = divmod(i, 4) + if bank != AD.bank_at(r, c): + bad.append(f"{key} bank({r},{c})") + for value, gs, want in ((15, 16, full), (1, 2, full), (7, 16, seven)): + fr = np.zeros((4, 4), dtype=np.uint8) + fr[r, c] = value + img = AD.render_frame(fr, gs, key, 255, 1)[::-1] # back to host rows + got = "rgb(%d,%d,%d)" % tuple(int(x) for x in img[r, c]) + if got != want: + bad.append(f"{key} ({r},{c}) v{value}: {got} vs js {want}") + check("every cell of every layout: bank, color at full and at 7/15, GS2 and GS16", not bad, "; ".join(bad[:6])) + +# ── 2. the renderer ────────────────────────────────────────────────────────── +fr = np.zeros((40, 200), dtype=np.uint8) +fr[0, 0] = 15 # host row 0 = the BOTTOM, col 0 = left +fr[39, 199] = 15 # the top right +img = AD.render_frame(fr, 16, "four-color", 255, 2) +check("image is (rows·scale, cols·scale, 3)", img.shape == (80, 400, 3), str(img.shape)) +check("host row 0 is drawn at the image bottom", tuple(img[79, 0]) == AD.CHANNEL_RGB["green"], f"{tuple(img[79, 0])}") +check("the top-right LED at the image top right (bank T1 blue)", tuple(img[0, 399]) == AD.CHANNEL_RGB["blue"], f"{tuple(img[0, 399])}") +check("an LED at 0 is drawn in the OFF color", tuple(img[40, 200]) == AD.OFF_RGB) +half = AD.render_frame(fr, 16, "four-color", 128, 1) +check("duty scales brightness (128/255)", tuple(half[39, 0]) == tuple(int(np.floor(v * 128 / 255 + 0.5)) for v in AD.CHANNEL_RGB["green"]), + f"{tuple(half[39, 0])}") +mono = AD.render_frame(fr, 16, "g6-green", 255, 1) +check("mono layout = the legacy green ramp (153, 255, 51)", tuple(mono[39, 0]) == (153, 255, 51), f"{tuple(mono[39, 0])}") +check("dark + placeholder images have the arena's size", AD.render_dark(scale=4).shape == (160, 800, 3) + and AD.render_placeholder(scale=4).shape == (160, 800, 3)) + +png = AD.png_bytes(img) +w, h = struct.unpack(">II", png[16:24]) +idat = png[png.index(b"IDAT") + 4: png.index(b"IEND") - 8] +raw = zlib.decompress(idat) +rows = np.frombuffer(raw, dtype=np.uint8).reshape(h, 1 + w * 3) +check("PNG: signature, IHDR size, filter bytes 0, pixels round-trip", + png[:8] == b"\x89PNG\r\n\x1a\n" and (w, h) == (400, 80) and not rows[:, 0].any() + and np.array_equal(rows[:, 1:].reshape(h, w, 3), img)) + +pat = SC.PatternLibrary([REPO / "protocols"]).get(VIOLET) +check("the violet pattern is found by name and has 20 frames", bool(pat) and pat["num_frames"] == 20) +if pat: + lit = AD.render_frame(pat["frames"][0], pat["gs"], "four-color", 255, 1) + on = (lit != np.array(AD.OFF_RGB, dtype=np.uint8)).any(axis=2) + colors = {tuple(x) for x in lit[on]} + check("violet pattern: only violet LEDs, top panel row only", colors == {AD.CHANNEL_RGB["violet"]} and not on[20:].any(), + f"colors {colors}, bottom lit {on[20:].sum()}") + +# ── 3. stimulus_core: synthetic rows ───────────────────────────────────────── +check("trialParams request decode (+80)", SC.parse_trial_params_req("0204005000000000") == (2, 4, 80, 0)) +check("trialParams request decode (−80, init 7)", SC.parse_trial_params_req("020400b0ff070000") == (2, 4, -80, 7)) +check("wrap_diff", SC.wrap_diff(19, 20) == -1 and SC.wrap_diff(-19, 20) == 1 and SC.wrap_diff(3, 20) == 3) + +tmp = Path(tempfile.mkdtemp(prefix="stim-test-")) +(tmp / "pair.yaml").write_text(f"""version: 3 +conditions: + - name: "violet_pair" + commands: + - type: "plugin" + plugin_name: "camera" + command_name: "getTimestamp" + - {{type: "controller", command_name: "trialParams", pattern: "{VIOLET}", pattern_ID: 1, duration: 2, mode: 2, frame_index: 0, frame_rate: 80, gain: 0, duty: 128}} + - {{type: "wait", duration: 2}} + - {{type: "controller", command_name: "trialParams", pattern: "{VIOLET}", pattern_ID: 1, duration: 2, mode: 2, frame_index: 0, frame_rate: -80, gain: 0, duty: 128}} + - {{type: "wait", duration: 2}} +""", encoding="utf-8") + + +def run_trial(tr, t0_us, host0_ms, fps, n_cf, latency_ms=1.0, batch=8, skip_at=None, req=None, pid=4): + """Controller rows for one mode-2 trial: the cc trialParams at t0, then n_cf frame changes, + drained in batches whose LAST row is `latency_ms` old when it arrives (as a real drain: the + newest row of a batch was recorded just before the read).""" + req = req or ("0204" + "00" + (fps & 0xFFFF).to_bytes(2, "little").hex() + "000000") + tr.feed({"type": "rows", "rows": [["cc", host0_ms + latency_ms, t0_us % SC.U32, 1, 8, 0, req]]}, host0_ms + latency_ms) + host_of = [] + rows = [] + for k in range(n_cf): + t = t0_us + 3000 + round(k * 1e6 / abs(fps)) + idx = (k if fps > 0 else -k) % 20 + if skip_at is not None and k >= skip_at: + idx = (idx + 3) % 20 + h = host0_ms + (t - t0_us) / 1000.0 + rows.append(["cf", 0, t % SC.U32, 100 + k, idx, pid, 160, 232, 600, 0, 3]) + host_of.append(h) + if len(rows) == batch or k == n_cf - 1: + rx = h + latency_ms + for r in rows: + r[1] = rx + tr.feed({"type": "rows", "rows": rows}, rx) + rows = [] + return host_of + + +tr = SC.StimulusTracker(SC.PatternLibrary([REPO / "protocols"]), SC.ProtocolIndex([tmp])) +check("never seen → WARN", tr.health(0)["level"] == SC.WARN) +tr.feed({"type": "log", "event": "run_metadata", "protocol_filename": "pair.yaml", "telemetry": "ring-10hz", "run_id": "t1"}, 1000) +tr.feed({"type": "log", "event": "runner", "phase": "sequence-start", "total": 3}, 1000) +tr.feed({"type": "log", "event": "runner", "phase": "step-start", "index": 0, "condition": "violet_pair"}, 1001) +h = tr.health(1500) +check("run open, no telemetry yet → WARN", h["level"] == SC.WARN and "telemetry" in h["message"], h["message"]) +tr.feed({"type": "log", "event": "runner", "phase": "trial-running", "params": {"mode": 2, "patternId": 4, "frameRate": 80, "initPos": 0, "duty": 128}, + "durationSec": 2, "condition": "violet_pair", "status": 0, "ok": True}, 2000) +st = tr.stimulus() +check("trial-running → pattern NAME from the protocol (1st trialParams), 20 frames, duty 128", + st["state"] == "trial" and st["pattern"] == VIOLET and st["n_frames"] == 20 and st["duty"] == 128 and st["pattern_found"], json.dumps(st)) +lags = [] +T0 = 5_000_000 +host = run_trial(tr, T0, 2000, 80, 160) +check("CW: lag 0 on every displayed frame", tr.lag == 0, f"lag {tr.lag}") +check("CW: display rate ≈ 80 Hz", abs(tr.rate_hz - 80) < 1.0, f"{tr.rate_hz}") +e = [tr.expected_index(hh) for hh in host[-20:]] +check("expected index at host time = the controller's frame (via the clock offset)", e == [k % 20 for k in range(140, 160)], f"{e[:6]}") +h = tr.health(host[-1] + 60) +check("CW: health OK, names the pattern, rate and lag", h["level"] == SC.OK and VIOLET in h["message"] and "lag +0" in h["message"], h["message"]) +# CCW straight after (no allOff in between: Grace's pair) +tr.feed({"type": "log", "event": "runner", "phase": "trial-running", "params": {"mode": 2, "patternId": 4, "frameRate": -80, "initPos": 0, "duty": 128}, + "durationSec": 2, "condition": "violet_pair", "status": 0, "ok": True}, 4000) +check("CCW: the 2nd trialParams of the step → its pattern name", tr.stimulus()["pattern"] == VIOLET and tr.stimulus()["fps"] == -80) +run_trial(tr, T0 + 2_000_000, 4000, -80, 160) +check("CCW: lag 0 counting down", tr.lag == 0 and abs(tr.rate_hz - 80) < 1.0, f"lag {tr.lag} rate {tr.rate_hz}") +# a frame stream 3 frames ahead of schedule +run_trial(tr, T0 + 4_000_000, 6000, -80, 160, skip_at=100) +h = tr.health(6000 + 2100) +check("frames off schedule → lag −/+3 and WARN", abs(tr.lag) == 3 and h["level"] == SC.WARN and "off schedule" in h["message"], f"{tr.lag} {h['message']}") +# u32 wrap of the controller clock mid-trial +tr2 = SC.StimulusTracker(SC.PatternLibrary([REPO / "protocols"]), SC.ProtocolIndex([tmp])) +tr2.feed({"type": "log", "event": "run_metadata", "protocol_filename": "pair.yaml", "telemetry": "ring-10hz"}, 1000) +tr2.feed({"type": "log", "event": "runner", "phase": "step-start", "condition": "violet_pair"}, 1001) +tr2.feed({"type": "log", "event": "runner", "phase": "trial-running", "params": {"mode": 2, "patternId": 4, "frameRate": 80, "initPos": 0, "duty": 255}, + "condition": "violet_pair", "status": 0, "ok": True}, 2000) +run_trial(tr2, SC.U32 - 1_000_000, 2000, 80, 200) +check("controller clock wraps (u32 µs) mid-trial: lag stays 0, rate 80", tr2.lag == 0 and abs(tr2.rate_hz - 80) < 1.0, f"{tr2.lag} {tr2.rate_hz}") +# routine refusal of a non-display command is NOT an error; a refused trialParams is +tr2.feed({"type": "rows", "rows": [["cc", 4600, 1_700_000, 9, 0x84, 10, ""]]}, 4600) +check("a refused SD/info command (0x84 status 10) is counted, not an ERROR", tr2.health(4600)["level"] != SC.ERROR and tr2.refused_other == 1) +check("stale: no frames for > 2 s while a pattern plays → STALE", tr2.health(2000 + 2600 + 3000)["level"] == SC.STALE, + tr2.health(7600)["message"]) +tr2.feed({"type": "log", "event": "runner", "phase": "command", "op": "allOff", "condition": "dark"}, 8000) +h = tr2.health(8001) +check("allOff → dark, OK (no STALE in the dark)", tr2.state == "dark" and h["level"] == SC.OK and tr2.expected_index(8001) is None, h["message"]) +tr2.feed({"type": "rows", "rows": [["cs", 8100, 1_900_000, 10, 3, 2, 0]]}, 8100) +check("an error glyph → ERROR", tr2.health(8101)["level"] == SC.ERROR) +check("…for 10 s, then back to OK", tr2.health(8100 + 10_500)["level"] == SC.OK) +tr2.feed({"type": "rows", "rows": [["cc", 20000, 2_000_000, 11, 8, 3, "0204005000000000"]]}, 20000) +check("a refused trialParams → ERROR", tr2.health(20001)["level"] == SC.ERROR) + +# ── 4. stimulus_core on the REAL trial ─────────────────────────────────────── +if not FIXTURE.exists(): + check("fixture present", False, str(FIXTURE)) +else: + recs = [json.loads(line) for line in gzip.open(FIXTURE, "rt")] + tr = SC.StimulusTracker(SC.PatternLibrary([REPO / "protocols"]), SC.ProtocolIndex([FIXTURE_PROTOCOLS])) + tr.frame_log = [] + lags, levels, names, rate_end = [], set(), set(), None + for r in recs: + tr.feed_record(r) + if tr.state == "trial": + names.add(tr.stimulus()["pattern"]) + levels.add(tr.health(r["rx_ms"])["level_name"]) + if tr.rate_hz: + rate_end = tr.rate_hz + for t, idx, pid, lag in tr.frame_log: + if pid == 4 and lag is not None: + lags.append(lag) + tr.frame_log.clear() + check("real trial: pattern name resolved from the protocol", names == {VIOLET}, str(names)) + check("real trial: ≥ 790 frames judged, every lag 0", len(lags) >= 790 and set(lags) == {0}, f"{len(lags)} frames, lags {sorted(set(lags))}") + check("real trial: display rate 80 Hz", rate_end is not None and abs(rate_end - 80) < 1.0, str(rate_end)) + check("real trial: health never ERROR/STALE during the trial", not ({"ERROR", "STALE"} & levels), str(levels)) + check("real trial: dark after the ITI's allOff", tr.state == "dark") + +# ── 5. bias_health ─────────────────────────────────────────────────────────── +cr = CountRate(2.0) +vals = [cr.add(t / 5.0, 30 * t / 5.0) for t in range(20)] +check("CountRate: 30 fps from a counter polled at 5 Hz", abs(vals[-1] - 30.0) < 1e-6, str(vals[-1])) +check("CountRate: a new recording (counter back to 0) restarts the window", + cr.add(4.2, 0) is None and abs(cr.add(4.6, 12) - 30.0) < 1e-6) +check("bias_drops: None when BIAS reports no counters", bias_drops({"frameCount": 5}) is None) +check("bias_drops: the sum of the counters it reports", bias_drops({"frameIdGaps": 2, "writerSkipped": 1, "cameraSkipped": 0}) == 3.0) + + +# ── 6. tools/ufmf.py on a synthetic BIAS-layout movie ──────────────────────── +def write_ufmf(path: Path, frames, bg, indexed=True, torn=False) -> None: + """BIAS's VideoWriter_ufmf layout (version 4, MONO8, index_loc past the chunk id).""" + H, W = bg.shape + b = bytearray(b"ufmf" + struct.pack(" bytes: + body = text.encode() + b"\x00" + data = b"\x00\x01\x00\x00" + struct.pack("= 90, json.dumps({k: v for k, v in topics.items() if "display" in k})) + lagv = [o["data"] for _, o in got.get("/viz/arena/lag_frames", [])] + check("replay: lag samples present, all 0", len(lagv) > 700 and set(lagv) == {0}, f"{len(lagv)} {sorted(set(lagv))[:5]}") + states = [o["data"] for _, o in got.get("/viz/arena/state", [])] + check("replay: state transitions dark → trial → dark", "trial" in states and states[-1] == "dark", str(states)) + cam = got.get("/camera/image", []) + cam_rep = rep.get("dashboard", {}).get("camera", {}) + first_ns = cam[0][0] if cam else None + check("replay: camera images from the run's uFMF (found via stopRecording), ~10 Hz", + schema_of.get("/camera/image") == "foxglove.CompressedImage" and 120 <= len(cam) <= 131, f"{len(cam)} {json.dumps(cam_rep)}") + check("replay: camera frame 0 placed on the bridge clock by the getTimestamp replies", + first_ns is not None and abs(first_ns / 1e6 - (t0 + 1000)) < 1.0, f"{first_ns and first_ns / 1e6 - t0}") + fpsc = [o["data"] for _, o in got.get("/viz/bias/fps_from_count", [])] + drops = [o["data"] for _, o in got.get("/viz/bias/drops", [])] + check("replay: BIAS fps_from_count ≈ 25 and drops 0 → 2", fpsc and abs(fpsc[-1] - 25.0) < 0.01 and drops[0] == 0 and drops[-1] == 2, + f"{fpsc[-1:] } {drops[:1]} {drops[-1:]}") +except ImportError as exc: + skip("foxglove_json --stimulus end to end", f"{exc}") + +# ── 8. the layout references only topics something publishes ───────────────── +layout = json.loads((ROOT / "foxglove" / "arena-okey-cokey.json").read_text(encoding="utf-8")) +paths = [] +for cfg in layout["configById"].values(): + for k in ("path", "topicPath", "topicToRender"): + if cfg.get(k): + paths.append(cfg[k]) + for p in cfg.get("paths", []): + paths.append(p["value"]) + if cfg.get("imageMode"): + paths.append(cfg["imageMode"]["imageTopic"]) +topics = {p.split(".")[0] for p in paths} +known = {"/camera/image", "/arena/display", "/arena/display_observed", "/arena/stimulus", "/diagnostics", "/bias/reply"} \ + | {f"/viz/arena/{k}" for k in ("expected_idx", "observed_idx", "display_rate_hz", "lag_frames", "cmd_fps", "ok", "state", "condition", "health")} \ + | {f"/viz/bias/{k}" for k in ("fps", "fps_from_count", "frame_count", "drops", "rtt_ms", "log_queue", "reachable", "capturing", "logging", "state")} +check("layout: every panel topic is one the nodes / the replay publish", topics <= known, str(sorted(topics - known))) + +shutil.rmtree(tmp, ignore_errors=True) +print(f"\n{'PASS' if not failures else 'FAIL'}: {failures} failure(s)") +sys.exit(1 if failures else 0) diff --git a/ros2-poc/tests/test_stimulus_live.py b/ros2-poc/tests/test_stimulus_live.py new file mode 100644 index 00000000..9b0fe160 --- /dev/null +++ b/ros2-poc/tests/test_stimulus_live.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""test_stimulus_live.py — the live dashboard path end to end: Studio WebSocket → bridge node → +/arena/browser → stimulus_node → the topics foxglove/arena-okey-cokey.json shows. + + pixi run test-stimulus-live # ~25 s; needs ports 7447 + 8765 free (no rig running) + +Starts a Zenoh router, bridge_node.py and stimulus_node.py (four-color), then plays the REAL +2026-10-14 smoke-run trial (tests/fixtures/okey_smoke_violet_cw.jsonl.gz: run metadata, runner +events, ~800 controller telemetry rows) into the bridge's WebSocket at its recorded pace, as the +Studio would, and listens with rclpy. Pass: the expected display is published as PNG images while +the grating turns and dark after the allOff, the observed index follows, the lag is 0, the display +rate reads 80 Hz, and the arena diagnostic is OK during the trial. +""" + +from __future__ import annotations + +import asyncio +import gzip +import json +import os +import signal +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT / "nodes")) +from _common import popen_group, signal_group, wait_port # noqa: E402 + +FIXTURE = HERE / "fixtures" / "okey_smoke_violet_cw.jsonl.gz" +failures = 0 + + +def check(name: str, ok: bool, detail: str = "") -> None: + global failures + failures += not ok + print(f"{'PASS' if ok else 'FAIL'} {name}" + (f" ({detail})" if detail and not ok else ""), flush=True) + + +async def play(records, url: str) -> None: + import websockets + + async with websockets.connect(url, max_size=None) as ws: + await ws.send(json.dumps({"type": "hello", "client": "test_stimulus_live", "v": 1})) + t_first = records[0]["rx_ms"] + t_start = time.monotonic() + for r in records: + due = (r["rx_ms"] - t_first) / 1000.0 + wait = due - (time.monotonic() - t_start) + if wait > 0: + await asyncio.sleep(wait) + await ws.send(r["raw"]) + await asyncio.sleep(1.5) + + +def main() -> int: + cfg = ROOT / "config" + env = dict(os.environ, ZENOH_ROUTER_CONFIG_URI=str(cfg / "zenoh-router.json5"), ZENOH_SESSION_CONFIG_URI=str(cfg / "zenoh-session.json5"), + RMW_IMPLEMENTATION=os.environ.get("RMW_IMPLEMENTATION", "rmw_zenoh_cpp"), ZENOH_ROUTER_CHECK_ATTEMPTS="0") + os.environ.update(env) + if wait_port("127.0.0.1", 8765, 0.2) or wait_port("127.0.0.1", 7447, 0.2): + print("FAIL ports 7447/8765 are in use — stop the rig (or bridge 3.4) first") + return 2 + logdir = Path(tempfile.gettempdir()) / f"stimulus-live-{os.getpid()}" + logdir.mkdir(parents=True, exist_ok=True) + procs = [] + + def start(name, argv): + fh = open(logdir / f"{name}.log", "w") + p = popen_group(argv, stdout=fh, stderr=subprocess.STDOUT) + procs.append((name, p, fh)) + return p + + py = sys.executable + records = [json.loads(line) for line in gzip.open(FIXTURE, "rt")] + got = {"display": [], "observed": [], "obs_idx": [], "lag": [], "rate": [], "diag": [], "state": []} + try: + start("router", ["ros2", "run", "rmw_zenoh_cpp", "rmw_zenohd"]) + check("zenoh router up", wait_port("127.0.0.1", 7447, 20.0)) + start("bridge", [py, str(ROOT / "nodes" / "bridge_node.py"), "--log-dir", str(logdir / "runlogs")]) + start("stimulus", [py, str(ROOT / "nodes" / "stimulus_node.py"), "--panel-layout", "four-color", + "--protocol-dir", str(HERE / "fixtures" / "protocols-2026-10-14")]) + check("bridge node serving ws://127.0.0.1:8765", wait_port("127.0.0.1", 8765, 20.0)) + + import rclpy + from diagnostic_msgs.msg import DiagnosticArray + from rclpy.node import Node + from sensor_msgs.msg import CompressedImage + from std_msgs.msg import Float64, String + + rclpy.init() + n = Node("stimulus_live_test") + n.create_subscription(CompressedImage, "/arena/display", lambda m: got["display"].append((time.monotonic(), m.format, bytes(m.data[:8]), len(m.data))), 50) + n.create_subscription(CompressedImage, "/arena/display_observed", lambda m: got["observed"].append(time.monotonic()), 50) + n.create_subscription(Float64, "/viz/arena/observed_idx", lambda m: got["obs_idx"].append(m.data), 50) + n.create_subscription(Float64, "/viz/arena/lag_frames", lambda m: got["lag"].append(m.data), 50) + n.create_subscription(Float64, "/viz/arena/display_rate_hz", lambda m: got["rate"].append(m.data), 50) + n.create_subscription(String, "/viz/arena/state", lambda m: got["state"].append(m.data), 50) + n.create_subscription(DiagnosticArray, "/diagnostics", + lambda m: got["diag"].extend((s.name, s.level[0] if isinstance(s.level, bytes) else s.level, s.message) for s in m.status), 50) + stop = threading.Event() + th = threading.Thread(target=lambda: [rclpy.spin_once(n, timeout_sec=0.05) for _ in iter(stop.is_set, True)], daemon=True) + th.start() + time.sleep(3.0) # discovery through the router + t_play = time.monotonic() + asyncio.run(play(records, "ws://127.0.0.1:8765")) + stop.set() + th.join(timeout=2.0) + + disp = [d for d in got["display"] if d[0] >= t_play] + check("expected display published as PNG while the grating turns (≥ 100 images)", + len(disp) >= 100 and all(f == "png" and head == b"\x89PNG\r\n\x1a\n" for _, f, head, _ in disp), f"{len(disp)} images") + check("observed display published (≥ 30 images)", len(got["observed"]) >= 30, str(len(got["observed"]))) + check("observed index follows the 20-frame grating", len(set(int(x) for x in got["obs_idx"])) == 20, str(sorted(set(got["obs_idx"]))[:25])) + check("lag 0 frames throughout", got["lag"] and set(got["lag"]) == {0.0}, str(sorted(set(got["lag"])))) + check("display rate reads 80 Hz during the trial", any(abs(r - 80) < 1.0 for r in got["rate"]), str(got["rate"][-12:])) + arena = [(lvl, msg) for name, lvl, msg in got["diag"] if name == "arena/display"] + trial_ok = [m for lvl, m in arena if lvl == 0 and "G6_2x10_toprow_grat20_violet" in m] + check("arena diagnostic OK naming the pattern during the trial", bool(trial_ok), str(arena[-6:])) + check("no ERROR/STALE arena diagnostic", all(lvl not in (2, 3) for lvl, _ in arena), str([a for a in arena if a[0] in (2, 3)][:3])) + check("state goes trial → dark", "trial" in got["state"] and got["state"][-1] == "dark", str(got["state"][-5:])) + n.destroy_node() + rclpy.shutdown() + finally: + for name, p, fh in reversed(procs): + try: + signal_group(p, signal.SIGINT) + p.wait(timeout=5) + except Exception: + p.kill() + fh.close() + print(f"\n{'PASS' if not failures else 'FAIL'}: {failures} failure(s) (logs: {logdir})") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ros2-poc/tools/ufmf.py b/ros2-poc/tools/ufmf.py new file mode 100644 index 00000000..944d2294 --- /dev/null +++ b/ros2-poc/tools/ufmf.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""ufmf.py — a reader for the uFMF movies BIAS writes (version 4), numpy only. + + from ufmf import UfmfReader + with UfmfReader("run_cam_0_date_…_v001.ufmf") as u: + u.n_frames, u.width, u.height + ts, img = u.frame(i) # ts = BIAS's frame timestamp (s); img uint8 (H, W) + u.timestamps # every frame's timestamp, from the index + +The layout is BIAS's writer (bias/src/gui/video_writer_ufmf.cpp), little-endian: + header "ufmf" · version u32 · index_loc u64 · max_w u16 · max_h u16 · is_fixed_size u8 · + coding_len u8 · coding ("MONO8") + chunk 0 keyframe: type_len u8 · type ("mean") · dtype char ('B') · w u16 · h u16 · ts f64 · w·h bytes + chunk 1 frame: ts f64 · n_boxes u32 · n × (col u16 · row u16 · w u16 · h u16 · w·h bytes) + chunk 2 index (BIAS's index_loc points just past the chunk id byte): a dict — 'd' · n_keys u8 · n × (key_len u16 · key · value), + value = 'd' (a dict) | 'a' · dtype char ('q' u64, 'd' f64) · n_bytes u32 · data; + {"frame": {"loc", "timestamp"}, "keyframe": {"mean": {"loc", "timestamp"}}} +A frame is the latest "mean" keyframe at or before it, with the frame's boxes pasted on top. +A file whose index was never written (BIAS killed mid-recording) is scanned chunk by chunk. +""" + +from __future__ import annotations + +import struct +from bisect import bisect_right +from pathlib import Path + +import numpy as np + +KEYFRAME, FRAME, INDEX = 0, 1, 2 +DTYPES = {"q": (" None: + self.path = Path(path) + self.f = open(self.path, "rb") + self._read_header() + self._bg_cache: dict[int, np.ndarray] = {} + self.indexed = False + try: + self._read_index() + self.indexed = True + except (UfmfError, struct.error, OSError): + self._scan() + + def __enter__(self): + return self + + def __exit__(self, *exc) -> None: + self.close() + + def close(self) -> None: + self.f.close() + + # ── header + index ───────────────────────────────────────────────────────── + def _read(self, n: int) -> bytes: + b = self.f.read(n) + if len(b) != n: + raise UfmfError(f"{self.path.name}: truncated at offset {self.f.tell()}") + return b + + def _read_header(self) -> None: + magic = self._read(4) + if magic != b"ufmf": + raise UfmfError(f"{self.path.name}: not a uFMF file (magic {magic!r})") + self.version, self.index_loc = struct.unpack("= 4 else False + n = self._read(1)[0] + self.coding = self._read(n).decode("ascii", errors="replace") + self.data_start = self.f.tell() + if self.coding.upper() not in ("MONO8", "RAW8"): + raise UfmfError(f"{self.path.name}: only 8-bit mono uFMF is supported (coding {self.coding!r})") + + def _read_value(self): + kind = self._read(1) + if kind == b"d": + return self._read_dict_body() + if kind == b"a": + dt = self._read(1).decode() + (nbytes,) = struct.unpack(" dict: + n = self._read(1)[0] + out = {} + for _ in range(n): + (klen,) = struct.unpack(" None: + if not self.index_loc: + raise UfmfError("no index") + self.f.seek(self.index_loc) + b = self._read(1) + if b[0] == INDEX: # motmot's writers point at the chunk id; BIAS points just past it + b = self._read(1) + if b != b"d": + raise UfmfError("index is not a dict") + idx = self._read_dict_body() + fr = idx.get("frame") or {} + kf = (idx.get("keyframe") or {}).get("mean") or {} + self.frame_locs = np.asarray(fr.get("loc", []), dtype=np.uint64) + self.timestamps = np.asarray(fr.get("timestamp", []), dtype=np.float64) + self.kf_locs = np.asarray(kf.get("loc", []), dtype=np.uint64) + self.kf_ts = np.asarray(kf.get("timestamp", []), dtype=np.float64) + if len(self.frame_locs) != len(self.timestamps) or not len(self.kf_locs): + raise UfmfError("index incomplete") + + def _scan(self) -> None: + """No usable index: walk the chunks from the header to the end (or the first torn chunk).""" + locs, ts, kf_locs, kf_ts = [], [], [], [] + size = self.path.stat().st_size + + def skip(n: int) -> None: # seek() past the end does not raise: a torn chunk would count + if self.f.tell() + n > size: + raise UfmfError("torn chunk") + self.f.seek(n, 1) + + self.f.seek(self.data_start) + while True: + pos = self.f.tell() + head = self.f.read(1) + if not head: + break + try: + cid = head[0] + if cid == KEYFRAME: + t_len = self._read(1)[0] + self._read(t_len + 1) + w, h = struct.unpack(" int: + return len(self.frame_locs) + + def _keyframe(self, k: int) -> np.ndarray: + if k in self._bg_cache: + return self._bg_cache[k] + self.f.seek(int(self.kf_locs[k])) + if self._read(1)[0] != KEYFRAME: + raise UfmfError("keyframe chunk id mismatch") + t_len = self._read(1)[0] + self._read(t_len) + dt = self._read(1).decode() + if dt != "B": + raise UfmfError(f"keyframe dtype {dt!r} not supported") + w, h = struct.unpack(" 4: + self._bg_cache.clear() + self._bg_cache[k] = img + return img + + @property + def width(self) -> int: + return self._keyframe(0).shape[1] + + @property + def height(self) -> int: + return self._keyframe(0).shape[0] + + def frame(self, i: int) -> tuple[float, np.ndarray]: + """(timestamp s, uint8 (H, W)) of frame i: its background with its boxes pasted on.""" + t = float(self.timestamps[i]) + k = max(0, bisect_right(self.kf_ts.tolist(), t) - 1) if len(self.kf_ts) > 1 else 0 + img = self._keyframe(k).copy() + self.f.seek(int(self.frame_locs[i])) + if self._read(1)[0] != FRAME: + raise UfmfError(f"frame {i}: chunk id mismatch") + self._read(8) + (n,) = struct.unpack(" np.ndarray: + """Mean over factor × factor blocks (edges cropped).""" + if factor <= 1: + return img + h, w = (img.shape[0] // factor) * factor, (img.shape[1] // factor) * factor + return img[:h, :w].reshape(h // factor, factor, w // factor, factor).mean(axis=(1, 3)).astype(np.uint8) + + +if __name__ == "__main__": + import sys + + for p in sys.argv[1:]: + with UfmfReader(p) as u: + ts = u.timestamps + fps = (len(ts) - 1) / (ts[-1] - ts[0]) if len(ts) > 1 and ts[-1] > ts[0] else 0.0 + print(f"{Path(p).name}: v{u.version} {u.coding} {u.width}×{u.height}, {u.n_frames} frames, " + f"{len(u.kf_locs)} keyframes, {ts[0] if len(ts) else 0:.3f}–{ts[-1] if len(ts) else 0:.3f} s ({fps:.1f} fps), " + f"index {'read' if u.indexed else 'MISSING (scanned)'}") From 40d89a2efe56a80e1c40151d1b6ea7b193df7ce4 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 10:07:52 -0400 Subject: [PATCH 05/13] =?UTF-8?q?ros2-poc:=20okey-cokey=20docs=20=E2=80=94?= =?UTF-8?q?=20dashboard=20BUILT=20status,=20run=20sheet=20with=20the=20das?= =?UTF-8?q?hboard=20steps,=20handover=20for=20the=20lab-PC=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ros2-poc/HANDOVER-okey-cokey-2026-10-21.md | 149 ++++++++---------- .../okey-cokey-foxglove-dashboard-plan.md | 40 ++++- ros2-poc/okey-cokey-pc-2026-10-21.md | 35 ++-- 3 files changed, 124 insertions(+), 100 deletions(-) diff --git a/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md index ce0756b7..215e630e 100644 --- a/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md +++ b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md @@ -5,113 +5,92 @@ four-color arena, an overhead BIAS camera, freely walking flies, no FicTrac). Mi remote path into the rig PCs, so start the session there: open Claude Code in a checkout of `reiserlab/webDisplayTools` and paste the kickoff prompt below as the first message. -There are two jobs, in this order: +There are two jobs, in this order, both on ONE run sheet: +[`okey-cokey-pc-2026-10-21.md`](okey-cokey-pc-2026-10-21.md). -1. **Test** today's protocol and BIAS on this PC: [`okey-cokey-pc-2026-10-21.md`](okey-cokey-pc-2026-10-21.md). -2. **Implement and test** the dashboard, phase 1 then 1b then 2: - [`docs/okey-cokey-foxglove-dashboard-plan.md`](docs/okey-cokey-foxglove-dashboard-plan.md). +1. **Test** today's protocol and BIAS on this PC. +2. **Test the camera + G6 Foxglove dashboard** on the real rig, live and on the run's replay. It was + BUILT and tested off-rig on Michael's Mac the same day + ([`docs/okey-cokey-foxglove-dashboard-plan.md`](docs/okey-cokey-foxglove-dashboard-plan.md), the + status block at the top); nothing of it has met this rig, its camera or a real `.ufmf` from it. ## Kickoff prompt (paste on the lab PC) ``` Okey-cokey rig work, LAB-303. Read ros2-poc/HANDOVER-okey-cokey-2026-10-21.md first and follow it. -Branch: git fetch && git switch claude/okey-cokey-pairs (clone to a short path such as C:\w if this -PC has no checkout yet; install pixi if missing, asking me before any admin/UAC step). -Job 1: run ros2-poc/okey-cokey-pc-2026-10-21.md in order, keeping +Branch: git fetch && git switch claude/okey-cokey-dashboard (clone to a short path such as C:\w if +this PC has no checkout yet; install pixi if missing, asking me before any admin/UAC step). +Run ros2-poc/okey-cokey-pc-2026-10-21.md in order, keeping ros2-poc/okey-cokey-pc-results-2026-10-21.md as you go (one row per step: pass/fail, timings, -sizes, every error verbatim). Stop and ask me (Michael) before anything that needs a click: the -BIAS GUI, the Chrome serial-port picker, the IR backlight, or putting flies in. -Job 2: on a new branch claude/okey-cokey-dashboard cut from claude/okey-cokey-pairs, implement -ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md phases 1, 1b, 2 in that order, each tested -(harness first, then on this rig) and committed separately. Rules are in the handover's -"Constraints" section. Push both branches when done; open no PR. +sizes, every error verbatim). First, without hardware: in ros2-poc, `pixi install`, +`pixi install -e viz`, `pixi run test-stimulus` and `pixi run test-stimulus-live` (both must pass +on Windows; record the counts). Stop and ask me (Michael) before anything that needs a click: the +BIAS GUI, the Chrome serial-port picker, the IR backlight, Foxglove, or putting flies in. Rules +are in the handover's "Constraints" section. Commit fixes and the results file to +claude/okey-cokey-dashboard and push; open no PR. ``` ## Where things stand -- **Branch `claude/okey-cokey-pairs`** = release candidate 2026-10-20 rc1 (Studio v0.97, the ROS 2 - graph, the BIAS connector; on Next) plus three commits: - - `protocols/g6_color_optomotor_test.yaml` + `_smoke.yaml` now use **Grace's trial shape - (2026-10-21)**: each trial is a CW/CCW pair, 15 s CW then 15 s CCW with no gap, then 10 s dark (also - across block boundaries). The full run is 24 pairs, 16.1 min; the smoke run is 4 pairs, 10 s per - direction, 5 s dark, 1.7 min. Parsed and flattened with the real modules (no blocking errors, no - warnings); `pixi run test -- protocol` 106/106. - - The run sheet and the dashboard plan (linked above). +- **Branch `claude/okey-cokey-dashboard`** = release candidate 2026-10-20 rc1 (Studio v0.97, the ROS 2 + graph, the BIAS connector; on Next) plus: + - `claude/okey-cokey-pairs`: `protocols/g6_color_optomotor_test.yaml` + `_smoke.yaml` in **Grace's + trial shape (2026-10-21)** — each trial a CW/CCW pair, 15 s CW then 15 s CCW with no gap, then 10 s + dark (also across block boundaries). Full run 24 pairs, 16.1 min; smoke 4 pairs (10 s per + direction, 5 s dark), 1.7 min. Parsed and flattened with the real modules; `pixi run test -- + protocol` 106/106. + - the dashboard (plan doc status block): `nodes/stimulus_node.py` + `stimulus_core.py` + + `arena_display.py` (expected/observed display in the four-color LEDs, arena rate/lag/health from + controller telemetry), BIAS rate/drops in `viz_node.py`, `rig.py --no-fictrac --stimulus`, the + replay (`fox-json --stimulus`, `tools/ufmf.py`), `foxglove/arena-okey-cokey.json`. Mac: 54/54 + + 10/10 + harness 54/54. - **Not on any Pages tier**: Next serves the candidate *without* the new protocol shape. Serve this checkout locally (`python .claude\nocache-server.py 8092`) or open the YAML via Protocol ▾ → Open…. -- **Background reading for job 1**: the ww18 results, `windows-results-ww18-2026-10-15.md` (every - Windows finding is in its table; most are fixed on this branch). For job 2: the plan's "What exists - today" table, which has file:line refs. -- **Not settled yet; the defaults below apply unless Michael says otherwise**: - 1. Live camera in Foxglove: **no** for now (no phase 3; the BIAS window sits beside Foxglove). Replay - camera (phase 2): yes. - 2. Controller telemetry: check it in job 1. Look for the firmware line at Connect and whether the - Studio starts the telemetry poller (GET_FIRMWARE_VERSION flag bit 2). Record the answer; phase 1 - must work either way, falling back to expected display only, with health = command status. - 3. BIAS build: record which one in job 1 (the Atlas10 fork reports drop counters, the Spinnaker - build may not). Missing keys mean "not reported", never 0. - -## Job 2: implementation notes - -Build in this order; each phase ends green before the next starts. - -**Phase 1** (plan §A, §B without color, §C, §E): -- `nodes/bridge_node.py`: also *publish* `/arena/browser` (String, reliable, depth 50), the same record - it already writes to the bag. Keep the bag write exactly as it is. -- `nodes/stimulus_core.py` (pure, no rclpy): stimulus state from `/arena/browser` records (runner - status events + trialParams `a` echoes + `cf` telemetry rows); expected index for Mode 2 - `(init + round(fps·(t − t_start))) mod n`; observed index, display rate and lag from `cf` rows; - diagnostic level. Read the row/event formats from `docs/development/telemetry-logging-reference.md` - (the format authority) and `js/arena-telemetry.js`'s header comment (the `cf` schema). Remember that - `cf` rows are 8 or 11 elements long. -- `nodes/stimulus_node.py`: a thin rclpy wrapper; `--pattern-dir`; publishes the topics in the plan's - table; `/arena/display` as `foxglove.CompressedImage` PNG reusing `tools/patfile.py` and the rendering - in `tools/tiff_run_to_mcap.py` (`display_png`), moved into a shared helper rather than copied. -- `nodes/viz_node.py`: feed the `arena` diagnostic from the new topics; add the BIAS - `fps_from_count` / `drops` / `rtt_ms` topics. -- `nodes/rig.py`: `--stimulus` (and include it with `--foxglove`), passing `--pattern-dir`. -- `foxglove/arena-okey-cokey.json`: the layout in plan §E (the camera panel can stay empty until - phase 2). -- Tests: extend `tests/test_ros2_poc.py` / `tests/test_viz_node.py`. `tests/studio_emulator.py` already - drives runs: make it send a Mode-2 trialParams + a stream of `cf` rows and assert expected vs - observed index, the rate (≈ |fps|) and the diagnostic levels (including the no-telemetry - fallback). Put unit tests for `stimulus_core` beside it. -- On the rig: `pixi run rig -- --bias --foxglove --stimulus …`, Foxglove (desktop app) → - `ws://localhost:8766` → import the layout; run the smoke protocol. Pass if the expected display - turns, reverses with no gap, goes dark in the pauses, and shows the condition name; the arena rate - reads ≈ 80 Hz during rotation and 0 in the dark (with telemetry); the BIAS fps and indicators are - live. Take a screenshot for the results file. - -**Phase 1b**: a Python port of the minimum of `js/panel-color.js` (layout table, `bankAt(row, col) = -2·((row+1)%2)+(col%2)` in host coordinates, the four-color layout), and a test that runs the JS module -under the root pixi Node and compares bank assignments cell by cell, so the two copies can't drift. -Duty scales the brightness. Pass: each of the four patterns renders in its own color, top row only. - -**Phase 2** (plan §D replay, §F): extend `nodes/foxglove_json.py` to replay `stimulus_core` over a bag's -`/arena/browser` records. Add `tools/ufmf_to_mcap.py`: read the run's `.ufmf` (write a small reader; -test it on a real recording from this PC) and write a sidecar MCAP with `/camera/image` JPEG timed by -the bag's `/bias/frames` stamps, downsampled (default 10 Hz, half resolution, both flags). Pass: today's -smoke run opens in Foxglove with the camera, the expected display and health in sync. Spot-check sync -on the reversal: the fly's turning should flip within a few hundred ms of the display's. +- **Background reading**: the ww18 results, `windows-results-ww18-2026-10-15.md` (every Windows finding + is in its table; most are fixed on this branch). +- **Settled**: the okey-cokey controller runs telemetry firmware; BIAS is NOT changed now (no live + camera in Foxglove — the BIAS window sits beside it; the replay has the camera). +- **Record in job 1**: which BIAS build (the Atlas10 fork reports drop counters, the Spinnaker build + may not: those panels then read "no data", never 0). + +## What could go wrong on this rig (look here first) + +- **Windows paths and the pattern/protocol lookup**: the stimulus node finds `.pat` under + the repo's `protocols/` + `patterns/` and the run's protocol by its `protocol_filename` under + `protocols/`. A protocol opened from elsewhere (Downloads) still runs, but its patterns' NAMES may + not resolve → a gray hatched placeholder and "pattern N" in the health line. Fix with + `--stimulus-args "--protocol-dir --pattern-dir "`, not code. +- **The camera's clock**: the replay places camera frames with the run's `getTimestamp` replies. + `camera.offset_ms` in the converter's report should be stable; if `camera.error` says no reply + had a value > 0, the protocol's first getTimestamp came too early (BIAS reads 0 before its first + frame). +- **uFMF from this camera**: the reader follows BIAS's writer (v4, MONO8, index or scan). A different + coding (e.g. RGB) is refused with a clear error: record it. +- **foxglove_bridge on Windows**: never run on Windows yet (ww18 did not install `viz`). If it fails + to start, the rig prints it and the rest runs; record the error verbatim. +- **Load**: the stimulus node renders 800×160 PNGs at 15 Hz (a few ms each on the Mac). If the + bridge's latency numbers (`connector_status`) get worse with it on, record both. ## Constraints - **Never touch Production or Next**, never merge to `main`, open no PR. Code reaches `main` only through a release candidate, which Michael builds (`docs/development/release-process.md`). -- **No Studio changes** (`arena_studio.html`, `js/*`) and **no bridge 3.4 changes** - (`fictrac-bridge/bridge.py`). Everything lives in `ros2-poc/`. If a Studio change turns out to be - needed, stop and write it up instead. -- Tests: `pixi run test` and `pixi run test-viz` inside `ros2-poc/`, plus `pixi run test` at the repo - root, all green before every commit. Prettier only on `*.js`, never on HTML. +- **No Studio changes** (`arena_studio.html`, `js/*`), **no bridge 3.4 changes** + (`fictrac-bridge/bridge.py`), **no BIAS changes**. Everything lives in `ros2-poc/`. If a Studio + change turns out to be needed, stop and write it up instead. +- Tests: `pixi run test`, `pixi run test-viz`, `pixi run test-stimulus` and `pixi run test-stimulus-live` + inside `ros2-poc/` (the last needs the rig stopped), plus `pixi run test` at the repo root, all green + before every commit. Prettier only on `*.js`, never on HTML. - Windows habits from ww18: write text with `newline="\n"`; take durations from `perf_counter`; never run `bias_sim` or the harness's sim while the real BIAS is open on 5010; Ctrl-C the rig to stop it (the SIGBREAK handler closes the bag cleanly). -- Capture an error's full text before you try to fix it. Commit small fixes found in job 1 to - `claude/okey-cokey-pairs` with a message that names the step. +- Capture an error's full text before you try to fix it. Commit small fixes to + `claude/okey-cokey-dashboard` with a message that names the step. - American English in code, docs and commits. ## What to send back -The results file for job 1; per phase: what was built, test counts, the rig screenshot, and anything -that needed a manual step or went against the plan. Push `claude/okey-cokey-pairs` and -`claude/okey-cokey-dashboard`, and end with a short summary Michael can paste into the LAB-303 ticket. +The results file (both jobs): the PC facts, BIAS build, both runs' checks, the two test counts on +Windows, a screenshot of the live dashboard mid-rotation, the replay converter's printed report, +and anything that needed a manual step. Push `claude/okey-cokey-dashboard`, and end with a short +summary Michael can paste into the LAB-303 ticket. diff --git a/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md b/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md index df28d1ff..350e1b36 100644 --- a/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md +++ b/ros2-poc/docs/okey-cokey-foxglove-dashboard-plan.md @@ -1,6 +1,37 @@ # Plan: a purpose-built Foxglove dashboard for the okey-cokey rig (camera + G6, no FicTrac) -Status: PLAN, 2026-10-21. Nothing built. Part of LAB-303 (ROS 2 graph); ticket to be filed. +> **Status 2026-10-21: phases 1, 1b and 2 BUILT** on branch `claude/okey-cokey-dashboard` (off +> `claude/okey-cokey-pairs`, itself the 2026-10-20 candidate + Grace's protocol shape). Phase 3 +> (live camera) NOT built, by decision: BIAS stays unchanged for now. The okey-cokey controller +> runs telemetry firmware (Michael), so the observed side is live. Part of LAB-303. +> +> | built | where | +> |---|---| +> | `/arena/browser` published live | `nodes/bridge_node.py` (one publisher; the bag record is unchanged) | +> | tracker: expected/observed index, lag, display rate, health | `nodes/stimulus_core.py` (pure) | +> | renderer + Python port of the panel colors | `nodes/arena_display.py` (pinned cell by cell to `js/panel-color.js` under Node) | +> | live node | `nodes/stimulus_node.py`; `pixi run rig -- --stimulus` (`--foxglove` includes it), `--no-fictrac` for this rig | +> | BIAS rate / drops / rtt / writer queue | `nodes/bias_health.py`, `viz_node.py` (`--sources bias` hides the unused FicTrac/Kinefly/arena-frame statuses) | +> | replay: same topics from a finished bag + `/camera/image` from the uFMF | `nodes/dashboard_replay.py` via `pixi run fox-json -- --stimulus`; `tools/ufmf.py` (BIAS v4 layout, index or scan) | +> | layout | `foxglove/arena-okey-cokey.json` (15 panels) | +> +> Tested: `pixi run test-stimulus` 54/54 (incl. a REAL 80 fps trial from the 2026-10-14 smoke run: +> 800+ frames, every lag 0, 80 Hz); `pixi run test-stimulus-live` 10/10 (router + bridge + +> stimulus node, the real trial played into the Studio WebSocket); ROS harness `pixi run test` +> 54/54 and `test-viz` unchanged; `rig.py --no-fictrac --bias-sim --bias --foxglove` brings every +> dashboard topic up with foxglove_bridge on :8766; uFMF reader on three real BIAS movies +> (1920×1080, up to 2.1 GB, indexed). On the real 2026-10-14 bag the replay MCAP is 612 kB for +> 2.5 min. **Not verified:** the layout opened in the Foxglove app (written to Foxglove's layout +> format by hand; its topics are checked by the test), a live run on the okey-cokey rig, a real +> uFMF from that rig's camera through the replay. +> +> Deviations from the plan below: (1) the arena status is the stimulus node's own +> `arena/display` diagnostic instead of re-feeding viz_node's FicTrac-frame `arena` entry, which +> keeps meaning closed-loop frames; (2) an OBSERVED display image (`/arena/display_observed`) +> was added beside the expected one; (3) `/viz/arena/observed_idx` and `lag_frames` are one sample +> per displayed frame (live and replay alike); (4) the expected display's stall check is STALE only +> while a pattern plays, since the telemetry ring has nothing to send in the dark; (5) the tiff +> tool's PNG writer was not refactored into the shared module (left as it was). ## Goal @@ -137,10 +168,9 @@ converter, the same split as `studio-replay.js`. ## Open questions -1. **Is a live camera in Foxglove needed, or is replay enough?** This decides whether phase 3 exists. -2. **Does the okey-cokey controller run telemetry-capable firmware?** (0xCB flag bit 2.) Without it, - the observed index and display rate are unavailable and arena health falls back to command status. +1. ~~Is a live camera in Foxglove needed?~~ Not now (2026-10-21): BIAS stays unchanged; replay has the camera. +2. ~~Telemetry firmware on the okey-cokey controller?~~ Yes (Michael, 2026-10-21). 3. Which BIAS build is on the okey-cokey PC: the Atlas10 fork has the drop counters, the Spinnaker - build may not. + build may not (the panels then read "no data", never 0). 4. Typed messages: `viz_node` subscribes to `/arena/frame` as String, which breaks in the `typed` env. That doesn't affect this rig (no `/arena/frame`), but fix it when touching viz_node. diff --git a/ros2-poc/okey-cokey-pc-2026-10-21.md b/ros2-poc/okey-cokey-pc-2026-10-21.md index c2b9eda4..c93e2872 100644 --- a/ros2-poc/okey-cokey-pc-2026-10-21.md +++ b/ros2-poc/okey-cokey-pc-2026-10-21.md @@ -5,9 +5,10 @@ BIAS camera, G6 2×10 four-color arena, **no FicTrac**). It is `WINDOWS-TEST.md` rig needs, with the ww18 findings of 2026-10-15 (`windows-results-ww18-2026-10-15.md`) applied up front. Record results in a sibling `okey-cokey-pc-results-2026-10-21.md` in the same shape as the ww18 file. -Branch: `claude/okey-cokey-pairs` = the 2026-10-20 release candidate (Studio v0.97, ROS 2 graph, BIAS -connector) + the CW/CCW-pair protocols Grace asked for. The protocol change is on no Pages tier yet, -so serve the checkout locally (step 4). +Branch: `claude/okey-cokey-dashboard` = the 2026-10-20 release candidate (Studio v0.97, ROS 2 graph, +BIAS connector) + the CW/CCW-pair protocols Grace asked for (`claude/okey-cokey-pairs`) + the camera + +G6 Foxglove dashboard (`docs/okey-cokey-foxglove-dashboard-plan.md`). Neither is on a Pages tier, so +serve the checkout locally (step 4). ## 0. Machine (5 min, over Remote Desktop) @@ -16,7 +17,7 @@ so serve the checkout locally (step 4). `powershell -ExecutionPolicy ByPass -c "irm -useb https://pixi.sh/install.ps1 | iex"`. - [ ] Short checkout path (ww18 W2: long paths break the optional typed build; the default env does not care, but keep it short anyway): `git clone https://github.com/reiserlab/webDisplayTools C:\w`, - then `cd C:\w` and `git switch claude/okey-cokey-pairs`. + then `cd C:\w` and `git switch claude/okey-cokey-dashboard`. - [ ] No bridge 3.4 running; `netstat -ano | findstr ":7447 :8765 :8092 :60000 :5010"` shows only BIAS on 5010 (once BIAS is up). - [ ] `cd ros2-poc` then `pixi install --locked` (default env only; skip `-e typed`). ww18: 25 s cold. @@ -63,15 +64,19 @@ dir D:\rec\okey-cokey Stop the node and the router (Ctrl-C). -## 3. The graph, no FicTrac (1 min) +## 3. The graph, no FicTrac, with the dashboard (2 min) + +Once: `pixi install -e viz` (foxglove_bridge, ~0.3 GB) and the Foxglove desktop app. ```powershell -pixi run rig -- --bias --bias-args "--video-dir D:\rec\okey-cokey" +pixi run rig -- --no-fictrac --bias --bias-args "--video-dir D:\rec\okey-cokey" --foxglove --viz-args "--sources bias" --stimulus-args "--panel-layout four-color" ``` -Expected: `[rig] up: router, fictrac, bias, bridge — Studio → ws://localhost:8765`. The FicTrac -adapter starts but stays idle (nothing sends to UDP 60000), which is fine. Add `--foxglove` only if -`pixi install -e viz` was done (see the dashboard plan). +Expected: `[rig] up: router, bias, bridge, stimulus, viz, foxglove — Studio → ws://localhost:8765 +Foxglove → ws://localhost:8766`. In Foxglove: Open connection → Foxglove WebSocket → +`ws://localhost:8766`; Layout → Import from file → `ros2-poc\foxglove\arena-okey-cokey.json`. +Before a run: BIAS indicator green, camera fps plotted, "Arena" grey/OK. (Without the viz env, drop +`--foxglove` and add `--stimulus`: the graph runs, there is just nothing to look at.) ## 4. Studio, served from this checkout @@ -101,6 +106,11 @@ Protocol ▾ → Open… on `C:\w\protocols\g6_color_optomotor_smoke.yaml`. 100 % (10 s CW → 10 s CCW, 5 s dark), camera connect / startRecording / getTimestamp / stopRecording all `ok` in the step strip. - [ ] Watch: top row only, one color per pair, no dark gap at the CW→CCW reversal, bottom row dark. +- [ ] **Dashboard, live:** the EXPECTED display turns with the arena in the right color and reverses + with no gap; OBSERVED follows within ~0.1 s; "Arena frame index" shows the two traces on top of + each other; lag stays 0; arena rate reads 80 Hz in a rotation and 0 in the pauses; the arena + health line names the pattern; condition / arena / BIAS state bars step with the protocol. + Record any WARN/ERROR text verbatim (`/viz/arena/health`) and a screenshot. - [ ] Then `?p=g6_color_optomotor_test` (Grace's shape): 24 pairs of 15 s CW + 15 s CCW, 10 s dark between pairs and across blocks, duty 25 → 50 → 100 %. Is the 25 % vs 100 % step visible? @@ -117,7 +127,12 @@ pixi run ros2 bag info runlogs\.bag - [ ] `/bias/reply`: smoke 1 + 4 + 1 `getTimestamp` (start, each pair, stop); full 1 + 24 + 1. - [ ] `/bias/frames` count = the stamp log's lines; video length ≈ run length × camera fps (full ≈ 965 s × fps). -- [ ] Copy the bag, the export and the `.ufmf` somewhere Michael can reach (group share). +- [ ] **Dashboard, replay:** `pixi run fox-json -- runlogs\.bag runlogs\.foxglove.mcap --stimulus --panel-layout four-color` + (finds the `.ufmf` through the run's stopRecording reply; `--ufmf ` otherwise). The JSON + it prints has a `dashboard` block: `camera.images`, `camera.offset_ms`, `stimulus_messages`. Open + the `.foxglove.mcap` in Foxglove with the same layout: camera, expected and observed display in + sync; check one reversal (the fly's turning should flip within a few hundred ms of the display). +- [ ] Copy the bag, the export, the `.foxglove.mcap` and the `.ufmf` somewhere Michael can reach (group share). ## What to send back From 21d90bddf64695064136c0dbf7a833ae6aea54c3 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Wed, 21 Oct 2026 10:14:38 -0400 Subject: [PATCH 06/13] ros2-poc: okey-cokey bare-PC setup checklist; handover + run sheet start from it (no Python/git assumed) Co-Authored-By: Claude Opus 5.5 --- ros2-poc/HANDOVER-okey-cokey-2026-10-21.md | 36 ++++---- ros2-poc/okey-cokey-pc-2026-10-21.md | 20 ++--- ros2-poc/okey-cokey-pc-setup.md | 98 ++++++++++++++++++++++ 3 files changed, 127 insertions(+), 27 deletions(-) create mode 100644 ros2-poc/okey-cokey-pc-setup.md diff --git a/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md index 215e630e..deadb27f 100644 --- a/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md +++ b/ros2-poc/HANDOVER-okey-cokey-2026-10-21.md @@ -2,10 +2,16 @@ For a Claude Code session running **on the okey-cokey lab PC** (Windows; the room with the G6 2×10 four-color arena, an overhead BIAS camera, freely walking flies, no FicTrac). Michael's Mac has no -remote path into the rig PCs, so start the session there: open Claude Code in a checkout of -`reiserlab/webDisplayTools` and paste the kickoff prompt below as the first message. +remote path into the rig PCs, so start the session there: open Claude Code in any folder (the +checkout does not exist yet) and paste the kickoff prompt below as the first message. -There are two jobs, in this order, both on ONE run sheet: +**This PC starts bare** (no Foxglove, no webDisplayTools checkout, maybe no git or pixi). Setup is +its own checklist, [`okey-cokey-pc-setup.md`](okey-cokey-pc-setup.md), done first. Claude Code +cannot install itself: a person installs the Claude desktop app (or Git for Windows, then the CLI) +over Remote Desktop; from there a Claude session can do most of the checklist and asks for the rest. +Without Claude on the PC, the checklist and the run sheet work by hand too. + +Then two jobs, in this order, both on ONE run sheet: [`okey-cokey-pc-2026-10-21.md`](okey-cokey-pc-2026-10-21.md). 1. **Test** today's protocol and BIAS on this PC. @@ -17,17 +23,17 @@ There are two jobs, in this order, both on ONE run sheet: ## Kickoff prompt (paste on the lab PC) ``` -Okey-cokey rig work, LAB-303. Read ros2-poc/HANDOVER-okey-cokey-2026-10-21.md first and follow it. -Branch: git fetch && git switch claude/okey-cokey-dashboard (clone to a short path such as C:\w if -this PC has no checkout yet; install pixi if missing, asking me before any admin/UAC step). -Run ros2-poc/okey-cokey-pc-2026-10-21.md in order, keeping -ros2-poc/okey-cokey-pc-results-2026-10-21.md as you go (one row per step: pass/fail, timings, -sizes, every error verbatim). First, without hardware: in ros2-poc, `pixi install`, -`pixi install -e viz`, `pixi run test-stimulus` and `pixi run test-stimulus-live` (both must pass -on Windows; record the counts). Stop and ask me (Michael) before anything that needs a click: the -BIAS GUI, the Chrome serial-port picker, the IR backlight, Foxglove, or putting flies in. Rules -are in the handover's "Constraints" section. Commit fixes and the results file to -claude/okey-cokey-dashboard and push; open no PR. +Okey-cokey rig work, LAB-303. This PC may be bare (no webDisplayTools checkout, no Foxglove, maybe +no git or pixi). Start with the setup checklist; read it on GitHub until the repo is cloned: +https://github.com/reiserlab/webDisplayTools/blob/claude/okey-cokey-dashboard/ros2-poc/okey-cokey-pc-setup.md +It clones the repo to C:\w on branch claude/okey-cokey-dashboard. Then read +C:\w\ros2-poc\HANDOVER-okey-cokey-2026-10-21.md and run C:\w\ros2-poc\okey-cokey-pc-2026-10-21.md +in order, keeping C:\w\ros2-poc\okey-cokey-pc-results-2026-10-21.md as you go (one row per step: +pass/fail, timings, sizes, every error verbatim; the setup facts first). +Ask me (Michael) before every download or install, any admin/UAC step, and anything that needs a +click: the BIAS GUI, the Chrome serial-port picker, the IR backlight, Foxglove, putting flies in. +If BIAS is not on this PC, stop and tell me. Rules are in the handover's "Constraints" section. +Commit fixes and the results file to claude/okey-cokey-dashboard and push; open no PR. ``` ## Where things stand @@ -45,7 +51,7 @@ claude/okey-cokey-dashboard and push; open no PR. replay (`fox-json --stimulus`, `tools/ufmf.py`), `foxglove/arena-okey-cokey.json`. Mac: 54/54 + 10/10 + harness 54/54. - **Not on any Pages tier**: Next serves the candidate *without* the new protocol shape. Serve this - checkout locally (`python .claude\nocache-server.py 8092`) or open the YAML via Protocol ▾ → Open…. + checkout locally (`pixi run python .claude\nocache-server.py 8092` from `C:\w`) or open the YAML via Protocol ▾ → Open…. - **Background reading**: the ww18 results, `windows-results-ww18-2026-10-15.md` (every Windows finding is in its table; most are fixed on this branch). - **Settled**: the okey-cokey controller runs telemetry firmware; BIAS is NOT changed now (no live diff --git a/ros2-poc/okey-cokey-pc-2026-10-21.md b/ros2-poc/okey-cokey-pc-2026-10-21.md index c93e2872..8402c63d 100644 --- a/ros2-poc/okey-cokey-pc-2026-10-21.md +++ b/ros2-poc/okey-cokey-pc-2026-10-21.md @@ -10,17 +10,13 @@ BIAS connector) + the CW/CCW-pair protocols Grace asked for (`claude/okey-cokey- G6 Foxglove dashboard (`docs/okey-cokey-foxglove-dashboard-plan.md`). Neither is on a Pages tier, so serve the checkout locally (step 4). -## 0. Machine (5 min, over Remote Desktop) - -- [ ] PC name, Windows version, free disk on the video drive. -- [ ] `git --version`, `pixi --version` (≥ 0.69), Chrome version. Install pixi if missing: - `powershell -ExecutionPolicy ByPass -c "irm -useb https://pixi.sh/install.ps1 | iex"`. -- [ ] Short checkout path (ww18 W2: long paths break the optional typed build; the default env does not - care, but keep it short anyway): `git clone https://github.com/reiserlab/webDisplayTools C:\w`, - then `cd C:\w` and `git switch claude/okey-cokey-dashboard`. -- [ ] No bridge 3.4 running; `netstat -ano | findstr ":7447 :8765 :8092 :60000 :5010"` shows only BIAS - on 5010 (once BIAS is up). -- [ ] `cd ros2-poc` then `pixi install --locked` (default env only; skip `-e typed`). ww18: 25 s cold. +## 0. Setup (bare PC → ready) + +Do [`okey-cokey-pc-setup.md`](okey-cokey-pc-setup.md) first: PC access, git / pixi / Chrome / +Foxglove, BIAS + camera SDK, the controller, the clone at `C:\w` on `claude/okey-cokey-dashboard`, +the three `pixi install`s, both dashboard tests and a no-flies smoke of the whole stack. The steps +below assume it is done. + - [ ] IR backlight on by hand (Arduino). The web runner does not drive the backlight yet. ## 1. BIAS itself (10 min) @@ -82,7 +78,7 @@ Before a run: BIAS indicator green, camera fps plotted, "Arena" grey/OK. (Withou ```powershell cd C:\w -python .claude\nocache-server.py 8092 +pixi run python .claude\nocache-server.py 8092 # the root env's Python ``` Chrome: `http://localhost:8092/arena_studio.html?p=g6_color_optomotor_smoke&rig=cshl_g6_2x10&mode=run` diff --git a/ros2-poc/okey-cokey-pc-setup.md b/ros2-poc/okey-cokey-pc-setup.md new file mode 100644 index 00000000..e5ae0b82 --- /dev/null +++ b/ros2-poc/okey-cokey-pc-setup.md @@ -0,0 +1,98 @@ +# Okey-cokey rig PC — setup checklist (bare Windows PC → ready for the test) + +The PC in the okey-cokey room starts without Foxglove, without the webDisplayTools repo and possibly +without git or pixi. Work through this list **before** the run sheet +([`okey-cokey-pc-2026-10-21.md`](okey-cokey-pc-2026-10-21.md)); the run sheet assumes every box here +is ticked. Items marked **(Michael/Grace)** need a person at the PC or an admin/UAC prompt; the rest +a Claude Code session on the PC can do and check itself. + +Write down what you find in each section, it goes into the results file. + +## A. Access and the PC itself + +- [ ] **(Grace)** Remote Desktop enabled; send Michael the PC name (Settings → System → Remote Desktop). +- [ ] PC name, Windows version (`winver`), user account; is the account a local admin? (installs below + are per-user where possible, so admin is only needed for C.4 and maybe B.5). +- [ ] Free disk: ≥ 10 GB on C: for the tools (pixi envs ~2 GB + package cache ~8 GB), plus the video + drive: the 16-min run at the camera's resolution × fps in uFMF (ww18's Atlas10 wrote ~24 MB/s at + 8 MP × 135 fps; a smaller camera is far less). Note which drive gets videos (`D:\rec\okey-cokey`?). +- [ ] Power plan: High performance, sleep **off** while plugged in (a sleeping PC drops the USB serial + link and stops the camera mid-run). +- [ ] Windows Update not about to reboot (Settings → Windows Update → pause 1 week for the test). + +## B. Tools (each one: check it, install if missing, note the version) + +| # | tool | check | install if missing | why | +|---|---|---|---|---| +| B.1 | **Git for Windows** | `git --version` | `winget install --id Git.Git -e` | the clone; Claude Code's CLI also runs through Git's bash | +| B.2 | **pixi** ≥ 0.69 | `pixi --version` | `powershell -ExecutionPolicy ByPass -c "irm -useb https://pixi.sh/install.ps1 \| iex"`, then open a NEW terminal | provides Python, Node, ROS 2, foxglove_bridge — nothing else to install for the software side | +| B.3 | **Chrome** (current) | Settings → About | `winget install --id Google.Chrome -e` | the Studio (Web Serial needs Chrome/Edge) | +| B.4 | **Foxglove** desktop app | Start menu "Foxglove" | download from foxglove.dev (a free account sign-in on first start) | the dashboard; per-user install | +| B.5 | **Claude Code** (only if a Claude session will drive the PC) | `claude --version`, or the desktop app | from claude.com/claude-code | runs the handover; the CLI needs B.1 first | +| B.6 | GitHub CLI (optional) | `gh --version` | `winget install --id GitHub.cli -e` | only for pushing results from the PC; HTTPS + a token also works | + +Not needed for this test: Visual Studio Build Tools (only the optional `typed` ROS environment wants a +compiler), MATLAB, FicTrac. + +## C. Lab hardware software (not provided by pixi — must already be on the PC or copied over) + +- [ ] **C.1 BIAS**: which build is installed, where (`…\test_gui.exe` or a `.bat` launcher), which camera. + The FLIR/Spinnaker build for FLIR cameras; the atlas-bench Atlas10 build (LUCID Arena backend) for + LUCID cameras, as on ww18. **If BIAS is not on this PC, stop and tell Michael** — it cannot be + built in the test window without its toolchain; copy a known-good build (e.g. from ww18 or the + lab share) instead. +- [ ] **C.2 Camera SDK/driver** for that build (Spinnaker or LUCID Arena SDK) installed; the camera + shows a live preview in BIAS. +- [ ] **C.3 BIAS external-control server** on, port 5010 for camera 0: + `curl.exe "http://127.0.0.1:5010/?get-status"` returns `"success" : true`. Logging format **ufmf**. +- [ ] **C.4 (Michael/Grace)** Firewall: if Windows asks to allow BIAS / `rmw_zenohd` / `foxglove_bridge` + / Python on first start, "Private networks" is enough (everything here talks over 127.0.0.1). +- [ ] **C.5 IR backlight** switch / Arduino works by hand (the web runner does not drive it yet). +- [ ] **C.6 G6 controller** on USB: Device Manager shows a COM port appear when it is plugged in + (Teensy serial uses the built-in Windows driver). Its firmware is the 2×10 telemetry build + (`arena-2x10-local` / `sdfast`); the Studio shows the label at Connect. +- [ ] **C.7 SD card** in the controller is mounted (the Studio's Console lists files). If unsure, a + ≤ 32 GB FAT32 card (ww18's exFAT 64 GB card had one slow-read burst). + +## D. The repo and the environments + +```powershell +git clone https://github.com/reiserlab/webDisplayTools C:\w +cd C:\w +git switch claude/okey-cokey-dashboard +pixi install # root env: Node + Python (tests, the local Studio server) +cd ros2-poc +pixi install --locked # ROS 2 graph (~25 s on ww18, ~1.5 GB) +pixi install --locked -e viz # + foxglove_bridge (~0.3 GB) +``` + +- [ ] Short checkout path (`C:\w`): the default env does not care, but Windows' 260-char limit bites + the optional typed build, and short paths are easier to type over RDP. +- [ ] `pixi run test-stimulus` → `PASS: 0 failure(s)` (includes the JS panel-color check, which uses + the ROOT env's Node — so the root `pixi install` above must have run). +- [ ] `pixi run test-stimulus-live` → `PASS` (needs ports 7447 / 8765 free: no rig, no bridge 3.4). +- [ ] Optional (~1 min): `pixi run test` → 54 / 54 (the whole ROS graph + Studio emulator). +- [ ] Nothing else listening: `netstat -ano | findstr ":7447 :8765 :8766 :8092 :60000 :5010"` shows only + BIAS on 5010 (once BIAS runs). + +## E. Smoke of the stack, no flies (5 min) + +- [ ] Terminal 1 (`C:\w\ros2-poc`): + `pixi run rig -- --no-fictrac --bias --bias-args "--video-dir D:\rec\okey-cokey" --foxglove --viz-args "--sources bias" --stimulus-args "--panel-layout four-color"` + → `[rig] up: router, bias, bridge, stimulus, viz, foxglove …` +- [ ] Terminal 2 (`C:\w`): `pixi run python .claude\nocache-server.py 8092` +- [ ] Chrome: `http://localhost:8092/arena_studio.html?p=g6_color_optomotor_smoke&rig=cshl_g6_2x10&mode=run` + → Bridge: Connect → `connected`; Connect → pick the controller's COM port. +- [ ] Foxglove → Open connection → Foxglove WebSocket → `ws://localhost:8766`; Layout → Import from + file → `C:\w\ros2-poc\foxglove\arena-okey-cokey.json`. BIAS indicator green, camera fps plotted, + the expected display dark, "Arena" OK. + +All green → go to the run sheet (it repeats E in context and continues with patterns, direction, +smoke run, full run, collect, replay). + +## What to record from this checklist + +PC name and Windows version; admin yes/no; every tool version (B), installed or found; BIAS build + path ++ camera model + whether its status reply has the drop counters (`frameIdGaps`, `cameraSkipped`, …); +controller firmware label; SD card; both test counts; anything that needed a click, a UAC prompt or a +download, and how long the whole setup took. From 09ed5c9c2ac9a95a414966a9eb92a722f9707af4 Mon Sep 17 00:00:00 2001 From: reiserm Date: Thu, 22 Oct 2026 15:07:56 -0400 Subject: [PATCH 07/13] ros2-poc: okey-cokey rig PC (ww14) setup results + fixes found on it - pixi.toml: viz env on win-64 prepends Library\lib to PATH; foxglove_bridge exited 0xC0000135 (foxglove.dll / foxglove_cpp_shared.dll not on the DLL path) - tools/bias-ufmf.ps1 + config/bias/okey-cokey-bfs-u3-51s5m.json: BIAS v0.73.1 restarts in jpg and rejects load-configuration until the camera has captured once; the script does connect -> 2 s capture -> load, run after every BIAS start - setup checklist + run sheet: arena on this rig is mounted upside down (protocol CW turns CCW seen from above; protocols unchanged, no direction test needed), Foxglove install URL/silent flags, 8766 not 8765, path placeholders (no D: on ww14), runlog-check pre-run "unknown" trials, known issues - .gitignore: stop ignoring tests/fixtures/*.jsonl.gz (the okey smoke fixture was never committed; still needs adding from ww18) - okey-cokey-pc-ww14-setup-2026-10-22.md: PC facts, tool versions, BIAS/camera, two passing smoke runs Co-Authored-By: Claude Opus 5.5 --- ros2-poc/.gitignore | 2 + .../config/bias/okey-cokey-bfs-u3-51s5m.json | 253 ++++++++++++++++++ ros2-poc/okey-cokey-pc-2026-10-21.md | 27 +- ros2-poc/okey-cokey-pc-setup.md | 41 ++- .../okey-cokey-pc-ww14-setup-2026-10-22.md | 81 ++++++ ros2-poc/pixi.toml | 6 + ros2-poc/tools/bias-ufmf.ps1 | 41 +++ 7 files changed, 445 insertions(+), 6 deletions(-) create mode 100644 ros2-poc/config/bias/okey-cokey-bfs-u3-51s5m.json create mode 100644 ros2-poc/okey-cokey-pc-ww14-setup-2026-10-22.md create mode 100644 ros2-poc/tools/bias-ufmf.ps1 diff --git a/ros2-poc/.gitignore b/ros2-poc/.gitignore index 6da99fcf..048a46b3 100644 --- a/ros2-poc/.gitignore +++ b/ros2-poc/.gitignore @@ -7,6 +7,8 @@ __pycache__/ runlogs/ *.jsonl *.jsonl.gz +# ...except the recorded test fixtures (okey_smoke_violet_cw.jsonl.gz was silently ignored before 2026-10-22) +!tests/fixtures/*.jsonl.gz # colcon alternative build of interfaces/arena_interfaces (pixi-build is the primary route) interfaces/build/ interfaces/install/ diff --git a/ros2-poc/config/bias/okey-cokey-bfs-u3-51s5m.json b/ros2-poc/config/bias/okey-cokey-bfs-u3-51s5m.json new file mode 100644 index 00000000..ac2ce960 --- /dev/null +++ b/ros2-poc/config/bias/okey-cokey-bfs-u3-51s5m.json @@ -0,0 +1,253 @@ +{ + "camera": { + "format7Settings": { + "mode": "0", + "pixelFormat": "MONO8", + "roi": { + "height": 2048, + "offsetX": 0, + "offsetY": 0, + "width": 2448 + } + }, + "frameRate": "Format7", + "guid": "24203703", + "model": "Blackfly S BFS-U3-51S5M", + "properties": { + "autoExposure": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "brightness": { + "absoluteControl": true, + "absoluteValue": 0.0, + "autoActive": false, + "on": true, + "onePush": false, + "present": true, + "value": 33 + }, + "focus": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "frameRate": { + "absoluteControl": true, + "absoluteValue": 73.3232421875, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 95 + }, + "gain": { + "absoluteControl": true, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 0 + }, + "gamma": { + "absoluteControl": true, + "absoluteValue": 0.800048828125, + "autoActive": false, + "on": true, + "onePush": false, + "present": true, + "value": 14 + }, + "hue": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "iris": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "pan": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "saturation": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "sharpness": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "shutter": { + "absoluteControl": true, + "absoluteValue": 998.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 0 + }, + "temperature": { + "absoluteControl": true, + "absoluteValue": 52.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 70 + }, + "tilt": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + }, + "triggerDelay": { + "absoluteControl": true, + "absoluteValue": 13.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 0 + }, + "triggerMode": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": true, + "value": 0 + }, + "whiteBalance": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "valueBlue": 0, + "valueRed": 0 + }, + "zoom": { + "absoluteControl": false, + "absoluteValue": 0.0, + "autoActive": false, + "on": false, + "onePush": false, + "present": false, + "value": 0 + } + }, + "triggerType": "Internal", + "vendor": "Point Grey Research", + "videoMode": "Format7" + }, + "configuration": { + "directory": "C:/Users/labadmin/Documents", + "fileName": "bias_config" + }, + "display": { + "colorMap": "None", + "orientation": { + "flipHorizontal": false, + "flipVertical": false + }, + "rotation": 0, + "updateFrequency": 15.0 + }, + "logging": { + "autoNamingOptions": { + "cameraIdentifier": "CameraNumber", + "includeCameraIdentifier": true, + "includeTimeAndDate": true, + "includeVersionNumber": true, + "timeAndDateFormat": "\u0027date\u0027_yyyy_MM_dd_\u0027time\u0027_hh_mm_ss" + }, + "directory": "C:/rec/okey-cokey", + "enabled": false, + "fileName": "bias_video", + "format": "ufmf", + "settings": { + "avi": { + "codec": "MJPG", + "frameSkip": 1 + }, + "bmp": { + "frameSkip": 1 + }, + "fmf": { + "frameSkip": 1 + }, + "jpg": { + "compressionThreads": 10, + "frameSkip": 1, + "mjpg": true, + "mjpgMaxFramePerFile": 5000000, + "mjpgMaxFramePerFileFlag": false, + "quality": 90 + }, + "ufmf": { + "backgroundThreshold": 40, + "boxLength": 30, + "compressionThreads": 15, + "dilate": { + "on": true, + "windowSize": 1 + }, + "frameSkip": 1, + "medianUpdateCount": 100, + "medianUpdateInterval": 50 + } + } + }, + "server": { + "enabled": true, + "port": 5010 + }, + "timer": { + "enabled": false, + "settings": { + "duration": 40 + } + } +} diff --git a/ros2-poc/okey-cokey-pc-2026-10-21.md b/ros2-poc/okey-cokey-pc-2026-10-21.md index 8402c63d..65f309c5 100644 --- a/ros2-poc/okey-cokey-pc-2026-10-21.md +++ b/ros2-poc/okey-cokey-pc-2026-10-21.md @@ -26,7 +26,10 @@ below assume it is done. - [ ] External-control server on, port **5010** for camera 0: `curl.exe "http://127.0.0.1:5010/?get-status"` returns `"success" : true`. Note any extra keys. - [ ] Logging format = **ufmf** in BIAS's own settings (BIAS writes its configured format whatever - extension it is sent; ww18 B1). + extension it is sent; ww18 B1). BIAS v0.73.1 (Spinnaker) resets to jpg on **every** restart (ww14), + so after each BIAS start run + `powershell -ExecutionPolicy Bypass -File tools\bias-ufmf.ps1 -VideoDir