fix(data_handling): stop coercing unparseable telemetry to 0.0 in floatify_input (#161)

onair/data_handling/parser_util.py

@@ -56,12 +56,14 @@ def floatify_input(_input, remove_str=False):
             try:
                 x = convert_str_to_timestamp(i)
                 floatified.append(x)
-            except:
-                if remove_str == False:
-                    floatified.append(0.0)
-                else:
+            except Exception:
+                if remove_str:
                     continue
-                continue
+                # OnAIR#161: never coerce unparseable telemetry to 0.0 — it is
+                # then read downstream as a VALID zero reading. Return the dash
+                # sentinel (the no-value marker OnAIR already uses in frames)
+                # so invalid data is explicit instead of masquerading as 0.0.
+                floatified.append("-")
     return floatified
 
 

plugins/kalman/kalman_plugin.py

@@ -110,6 +110,13 @@ def update(self, low_level_data=None, _high_level_data=None):
         else:
             frame = floatify_input(low_level_data)
 
+            if "-" in frame:
+                # A partial/unparseable frame can't be appended: _generate_residuals
+                # index-aligns each attribute's window by position, so dropping just
+                # the bad values (rather than the whole frame) would desynchronize
+                # per-attribute windows. Skip the entire frame instead.
+                return
+
             for i, value in enumerate(frame):
                 self.frames[i].append(value)
                 if len(self.frames[i]) > self.window_size:

test/onair/data_handling/test_parser_util.py

@@ -528,7 +528,7 @@ def test_parser_util_flotify_input_returns_list_of_size_one_that_contains_0_dot_
     assert parser_util.convert_str_to_timestamp.call_args_list[0].args == (
         arg__input[0],
     )
-    assert result == [0.0]
+    assert result == ["-"]
 
 
 def test_parser_util_flotify_input_default_arg_remove_str_is_False(mocker):
@@ -547,7 +547,7 @@ def test_parser_util_flotify_input_default_arg_remove_str_is_False(mocker):
     result = parser_util.floatify_input(arg__input)
 
     # Assert
-    assert result == [0.0]  # shows flow was correct for remove_str being False
+    assert result == ["-"]  # invalid data is the dash sentinel, not 0.0
 
 
 def test_parser_util_flotify_input_returns_empty_list_when_two_Exceptions_are_thrown_and_remove_str_is_True(
@@ -689,7 +689,7 @@ def test_parser_util_flotify_input_returns_expected_values_for_given__input_that
             resultant_float = MagicMock()
             side_effects_for_float.append(ValueError)
             side_effects_for_convert_str.append(Exception)
-            expected_result.append(0.0)
+            expected_result.append("-")
         else:  # other
             arg__input.append(MagicMock())
             resultant_float = MagicMock()

test/plugins/kalman/test_kalman_plugin.py

@@ -15,6 +15,7 @@
 
 from plugins.kalman import kalman_plugin
 from plugins.kalman.kalman_plugin import Plugin as Kalman
+from onair.data_handling.parser_util import floatify_input as real_floatify_input
 
 
 # test init
@@ -253,6 +254,57 @@ def test_Kalman_update_with_full_window_size(mocker):
         assert frame[:-1] == original_frames[i][1:]  # Check that data shifted correctly
 
 
+def test_Kalman_update_with_unparseable_value_in_frame_skips_the_whole_frame(mocker):
+    # Arrange
+    cut = Kalman.__new__(Kalman)
+    num_headers = pytest.gen.randint(2, 5)
+    existing_data_points = pytest.gen.randint(1, 4)
+    cut.window_size = existing_data_points + pytest.gen.randint(1, 5)
+
+    original_frames = [
+        [pytest.gen.uniform(-10, 10) for _ in range(existing_data_points)]
+        for _ in range(num_headers)
+    ]
+    cut.frames = [frame.copy() for frame in original_frames]
+
+    # One attribute failed to parse; floatify_input marks it with the dash sentinel
+    bad_index = pytest.gen.randint(0, num_headers - 1)
+    arg_low_level_data = [MagicMock() for _ in range(num_headers)]
+    forced_frame = [pytest.gen.uniform(-10, 10) for _ in range(num_headers)]
+    forced_frame[bad_index] = "-"
+    mocker.patch(
+        kalman_plugin.__name__ + ".floatify_input", return_value=forced_frame
+    )
+
+    # Act
+    cut.update(arg_low_level_data)
+
+    # Assert
+    assert kalman_plugin.floatify_input.call_count == 1
+    assert cut.frames == original_frames  # no attribute's window was touched
+
+
+def test_Kalman_update_with_real_floatify_input_and_unparseable_value_leaves_frames_unchanged(
+    mocker,
+):
+    # Arrange
+    cut = Kalman.__new__(Kalman)
+    cut.window_size = 5
+    original_frames = [[1.0, 2.0], [3.0, 4.0]]
+    cut.frames = [frame.copy() for frame in original_frames]
+
+    mocker.patch(
+        kalman_plugin.__name__ + ".floatify_input", side_effect=real_floatify_input
+    )
+    arg_low_level_data = ["1.5", "not-a-number"]
+
+    # Act
+    cut.update(arg_low_level_data)
+
+    # Assert
+    assert cut.frames == original_frames
+
+
 # test render reasoning
 def test_Kalman_render_reasoning_returns_empty_list_when_all_values_below_threshold(
     mocker,