From ca1f5daf724fc1ddce2766b41f32ba43288ac894 Mon Sep 17 00:00:00 2001 From: Jack Hopkins Date: Mon, 7 Sep 2026 20:31:20 +0100 Subject: [PATCH 1/2] fix: retry transient get_entities failures during observation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single transient get_entities failure (engine busy, RCON hiccup) killed an entire multi-hour eval: one rollout errored ~1h in and fail-on-error cancelled the other three healthy epochs. The action sequence replayed clean, confirming the failure was transient. Retry up to 3 times with short backoff, and include the underlying error in the raised message — the previous wrapper discarded it, making the failure undiagnosable from the eval log. --- fle/env/gym_env/environment.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/fle/env/gym_env/environment.py b/fle/env/gym_env/environment.py index 4afbec4c..cb5a3820 100644 --- a/fle/env/gym_env/environment.py +++ b/fle/env/gym_env/environment.py @@ -1,4 +1,5 @@ import logging +import time import gymnasium as gym import numpy as np @@ -304,12 +305,24 @@ def get_observation( if self.enable_vision: map_image = namespace._render().to_base64() - # Get entity observations - try: - entities = namespace.get_entities() - except Exception as e: - logger.warning(f"Error getting entities: {e}") - raise Exception("Error getting entities while getting observation") from e + # Get entity observations. A single transient failure (engine busy, + # RCON hiccup) must not kill a multi-hour rollout, so retry briefly; + # include the underlying error so real failures stay diagnosable. + entities = None + last_entity_error = None + for attempt, delay in enumerate((0, 2, 5), start=1): + if delay: + time.sleep(delay) + try: + entities = namespace.get_entities() + break + except Exception as e: + last_entity_error = e + logger.warning(f"Error getting entities (attempt {attempt}/3): {e}") + if entities is None: + raise Exception( + f"Error getting entities while getting observation: {last_entity_error}" + ) from last_entity_error entity_obs = [e.__dict__ for e in entities] From 37ca19ea11271417dd7e1a38da2dad48ce84afcb Mon Sep 17 00:00:00 2001 From: Jack Hopkins Date: Tue, 8 Sep 2026 18:18:04 +0100 Subject: [PATCH 2/2] fix: retry transient score() failures; extract shared retry helper score() after a step could fail transiently (e.g. KeyError 'player' from a corrupted RCON response) and kill the epoch. Apply the same retry-with-backoff used for get_entities, via a shared _call_with_retry helper used by both sites. --- fle/env/gym_env/environment.py | 45 ++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/fle/env/gym_env/environment.py b/fle/env/gym_env/environment.py index cb5a3820..fbf1f720 100644 --- a/fle/env/gym_env/environment.py +++ b/fle/env/gym_env/environment.py @@ -28,6 +28,26 @@ logger = logging.getLogger(__name__) + +def _call_with_retry(fn, what: str, delays=(0, 2, 5)): + """Call fn(), retrying transient failures with short backoff. + + A single transient failure (engine busy, RCON hiccup, malformed + response) must not kill a multi-hour rollout. The underlying error is + included in the raised exception so real failures stay diagnosable. + """ + last_error = None + for attempt, delay in enumerate(delays, start=1): + if delay: + time.sleep(delay) + try: + return fn() + except Exception as e: + last_error = e + logger.warning(f"Error {what} (attempt {attempt}/{len(delays)}): {e}") + raise Exception(f"Error {what}: {last_error}") from last_error + + # need to do this since gym doesn't work with numpy>=2.0 otherwise. np.bool8 = np.dtype(np.bool) @@ -305,24 +325,9 @@ def get_observation( if self.enable_vision: map_image = namespace._render().to_base64() - # Get entity observations. A single transient failure (engine busy, - # RCON hiccup) must not kill a multi-hour rollout, so retry briefly; - # include the underlying error so real failures stay diagnosable. - entities = None - last_entity_error = None - for attempt, delay in enumerate((0, 2, 5), start=1): - if delay: - time.sleep(delay) - try: - entities = namespace.get_entities() - break - except Exception as e: - last_entity_error = e - logger.warning(f"Error getting entities (attempt {attempt}/3): {e}") - if entities is None: - raise Exception( - f"Error getting entities while getting observation: {last_entity_error}" - ) from last_entity_error + entities = _call_with_retry( + namespace.get_entities, "getting entities while getting observation" + ) entity_obs = [e.__dict__ for e in entities] @@ -476,7 +481,9 @@ def step( ) terminated = task_success.success - production_score, automated_production_score = namespace.score() + production_score, automated_production_score = _call_with_retry( + namespace.score, "getting score after step" + ) if not automated_production_score: automated_production_score = 0 # Calculate reward