fix: retry transient get_entities failures during observation

fle/env/gym_env/environment.py

@@ -1,4 +1,5 @@
 import logging
+import time
 
 import gymnasium as gym
 import numpy as np
@@ -27,6 +28,26 @@
 
 logger = logging.getLogger(__name__)
 
+
+def _call_with_retry(fn, what: str, delays=(0, 2, 5)):
+    """Call fn(), retrying transient failures with short backoff.
+
+    A single transient failure (engine busy, RCON hiccup, malformed
+    response) must not kill a multi-hour rollout. The underlying error is
+    included in the raised exception so real failures stay diagnosable.
+    """
+    last_error = None
+    for attempt, delay in enumerate(delays, start=1):
+        if delay:
+            time.sleep(delay)
+        try:
+            return fn()
+        except Exception as e:
+            last_error = e
+            logger.warning(f"Error {what} (attempt {attempt}/{len(delays)}): {e}")
+    raise Exception(f"Error {what}: {last_error}") from last_error
+
+
 # need to do this since gym doesn't work with numpy>=2.0 otherwise.
 np.bool8 = np.dtype(np.bool)
 
@@ -304,12 +325,9 @@ def get_observation(
         if self.enable_vision:
             map_image = namespace._render().to_base64()
 
-        # Get entity observations
-        try:
-            entities = namespace.get_entities()
-        except Exception as e:
-            logger.warning(f"Error getting entities: {e}")
-            raise Exception("Error getting entities while getting observation") from e
+        entities = _call_with_retry(
+            namespace.get_entities, "getting entities while getting observation"
+        )
 
         entity_obs = [e.__dict__ for e in entities]
 
@@ -463,7 +481,9 @@ def step(
             )
             terminated = task_success.success
 
-        production_score, automated_production_score = namespace.score()
+        production_score, automated_production_score = _call_with_retry(
+            namespace.score, "getting score after step"
+        )
         if not automated_production_score:
             automated_production_score = 0
         # Calculate reward