diff --git a/onair/data_handling/parser_util.py b/onair/data_handling/parser_util.py index 620b34e7..0f1af766 100644 --- a/onair/data_handling/parser_util.py +++ b/onair/data_handling/parser_util.py @@ -56,12 +56,14 @@ def floatify_input(_input, remove_str=False): try: x = convert_str_to_timestamp(i) floatified.append(x) - except: - if remove_str == False: - floatified.append(0.0) - else: + except Exception: + if remove_str: continue - continue + # OnAIR#161: never coerce unparseable telemetry to 0.0 — it is + # then read downstream as a VALID zero reading. Return the dash + # sentinel (the no-value marker OnAIR already uses in frames) + # so invalid data is explicit instead of masquerading as 0.0. + floatified.append("-") return floatified diff --git a/plugins/kalman/kalman_plugin.py b/plugins/kalman/kalman_plugin.py index 6350a623..e1ab9260 100644 --- a/plugins/kalman/kalman_plugin.py +++ b/plugins/kalman/kalman_plugin.py @@ -110,6 +110,13 @@ def update(self, low_level_data=None, _high_level_data=None): else: frame = floatify_input(low_level_data) + if "-" in frame: + # A partial/unparseable frame can't be appended: _generate_residuals + # index-aligns each attribute's window by position, so dropping just + # the bad values (rather than the whole frame) would desynchronize + # per-attribute windows. Skip the entire frame instead. + return + for i, value in enumerate(frame): self.frames[i].append(value) if len(self.frames[i]) > self.window_size: diff --git a/test/onair/data_handling/test_parser_util.py b/test/onair/data_handling/test_parser_util.py index a700cc2b..3cc4b1fa 100644 --- a/test/onair/data_handling/test_parser_util.py +++ b/test/onair/data_handling/test_parser_util.py @@ -528,7 +528,7 @@ def test_parser_util_flotify_input_returns_list_of_size_one_that_contains_0_dot_ assert parser_util.convert_str_to_timestamp.call_args_list[0].args == ( arg__input[0], ) - assert result == [0.0] + assert result == ["-"] def test_parser_util_flotify_input_default_arg_remove_str_is_False(mocker): @@ -547,7 +547,7 @@ def test_parser_util_flotify_input_default_arg_remove_str_is_False(mocker): result = parser_util.floatify_input(arg__input) # Assert - assert result == [0.0] # shows flow was correct for remove_str being False + assert result == ["-"] # invalid data is the dash sentinel, not 0.0 def test_parser_util_flotify_input_returns_empty_list_when_two_Exceptions_are_thrown_and_remove_str_is_True( @@ -689,7 +689,7 @@ def test_parser_util_flotify_input_returns_expected_values_for_given__input_that resultant_float = MagicMock() side_effects_for_float.append(ValueError) side_effects_for_convert_str.append(Exception) - expected_result.append(0.0) + expected_result.append("-") else: # other arg__input.append(MagicMock()) resultant_float = MagicMock() diff --git a/test/plugins/kalman/test_kalman_plugin.py b/test/plugins/kalman/test_kalman_plugin.py index a77a86e9..99d498f5 100644 --- a/test/plugins/kalman/test_kalman_plugin.py +++ b/test/plugins/kalman/test_kalman_plugin.py @@ -15,6 +15,7 @@ from plugins.kalman import kalman_plugin from plugins.kalman.kalman_plugin import Plugin as Kalman +from onair.data_handling.parser_util import floatify_input as real_floatify_input # test init @@ -253,6 +254,57 @@ def test_Kalman_update_with_full_window_size(mocker): assert frame[:-1] == original_frames[i][1:] # Check that data shifted correctly +def test_Kalman_update_with_unparseable_value_in_frame_skips_the_whole_frame(mocker): + # Arrange + cut = Kalman.__new__(Kalman) + num_headers = pytest.gen.randint(2, 5) + existing_data_points = pytest.gen.randint(1, 4) + cut.window_size = existing_data_points + pytest.gen.randint(1, 5) + + original_frames = [ + [pytest.gen.uniform(-10, 10) for _ in range(existing_data_points)] + for _ in range(num_headers) + ] + cut.frames = [frame.copy() for frame in original_frames] + + # One attribute failed to parse; floatify_input marks it with the dash sentinel + bad_index = pytest.gen.randint(0, num_headers - 1) + arg_low_level_data = [MagicMock() for _ in range(num_headers)] + forced_frame = [pytest.gen.uniform(-10, 10) for _ in range(num_headers)] + forced_frame[bad_index] = "-" + mocker.patch( + kalman_plugin.__name__ + ".floatify_input", return_value=forced_frame + ) + + # Act + cut.update(arg_low_level_data) + + # Assert + assert kalman_plugin.floatify_input.call_count == 1 + assert cut.frames == original_frames # no attribute's window was touched + + +def test_Kalman_update_with_real_floatify_input_and_unparseable_value_leaves_frames_unchanged( + mocker, +): + # Arrange + cut = Kalman.__new__(Kalman) + cut.window_size = 5 + original_frames = [[1.0, 2.0], [3.0, 4.0]] + cut.frames = [frame.copy() for frame in original_frames] + + mocker.patch( + kalman_plugin.__name__ + ".floatify_input", side_effect=real_floatify_input + ) + arg_low_level_data = ["1.5", "not-a-number"] + + # Act + cut.update(arg_low_level_data) + + # Assert + assert cut.frames == original_frames + + # test render reasoning def test_Kalman_render_reasoning_returns_empty_list_when_all_values_below_threshold( mocker,