From 948ef0365fe911d9cb90ab18821b75534a21593d Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 28 Sep 2026 11:46:09 -0400 Subject: [PATCH 1/2] Retry transient API errors and ease load on the nightly compositional shard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly shard was failing intermittently under -n 20: some failures were Playwright timeouts that exhausted their retries because the pool was genuinely overloaded, but others were failures the existing retry never covered — 502 Bad Gateway responses from the Table API, and a "record was not created" check that gave up after ~7.5s instead of the 30s budget used elsewhere for browser-side waits. Retry transient 502/503/504 and connection errors at the request layer, widen the record-creation poll to match SNOW_BROWSER_TIMEOUT, and drop the shard to -n 10 to reduce concurrent load on the shared pool. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/instance_pool_ci.yml | 4 ++- src/browsergym/workarena/api/utils.py | 46 +++++++++++++++++++------- src/browsergym/workarena/tasks/form.py | 10 ++++-- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/instance_pool_ci.yml b/.github/workflows/instance_pool_ci.yml index 77e5b2c..cb6ba31 100644 --- a/.github/workflows/instance_pool_ci.yml +++ b/.github/workflows/instance_pool_ci.yml @@ -154,5 +154,7 @@ jobs: echo "Reproduce with \`WORKARENA_TEST_SHARD=$WORKARENA_TEST_SHARD WORKARENA_TEST_SEED=$WORKARENA_TEST_SEED\` and the pytest command below." | tee -a "$GITHUB_STEP_SUMMARY" # L1 (test_task_general.py) already runs in full every night in test-l1-tasks + # -n 10 rather than 20: this shard's compositional tasks are heavier per-worker than L1, + # and 20 concurrent browsers routinely overloaded the shared pool (timeouts, 502s) - name: Run shard - run: pytest -n 20 --durations=10 --slowmo 1000 -v -m 'slow or pricy' --ignore=tests/test_task_general.py tests + run: pytest -n 10 --durations=10 --slowmo 1000 -v -m 'slow or pricy' --ignore=tests/test_task_general.py tests diff --git a/src/browsergym/workarena/api/utils.py b/src/browsergym/workarena/api/utils.py index 47a6b2a..d26b441 100644 --- a/src/browsergym/workarena/api/utils.py +++ b/src/browsergym/workarena/api/utils.py @@ -2,12 +2,39 @@ from ..instance import SNowInstance -from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError, HTTPError +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential from time import sleep # ServiceNow API configuration SNOW_API_HEADERS = {"Content-Type": "application/json", "Accept": "application/json"} +# Gateway/proxy status codes that indicate a transient blip on an overloaded instance +# rather than a real API error +_TRANSIENT_STATUS_CODES = {502, 503, 504} + + +def _is_transient_http_error(exception: BaseException) -> bool: + if isinstance(exception, ConnectionError): + return True + return ( + isinstance(exception, HTTPError) + and exception.response is not None + and exception.response.status_code in _TRANSIENT_STATUS_CODES + ) + + +@retry( + retry=retry_if_exception(_is_transient_http_error), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=1, max=10), + reraise=True, +) +def _request_with_retry(**kwargs) -> requests.Response: + response = requests.request(**kwargs) + response.raise_for_status() + return response + def table_api_call( instance: SNowInstance, @@ -52,8 +79,8 @@ def table_api_call( """ - # Query API - response = requests.request( + # Query API (retries transient gateway/connection errors from the shared, load-sensitive instance) + response = _request_with_retry( method=method, url=instance.snow_url + f"/api/now/table/{table}", auth=instance.snow_credentials, @@ -67,9 +94,6 @@ def table_api_call( data = {} params = {"sysparm_query": f"sys_id={sys_id}"} - # Check for HTTP success code (fail otherwise) - response.raise_for_status() - record_exists = False num_retries = 0 if method == "POST" or wait_for_record: @@ -120,12 +144,12 @@ def table_column_info(instance: SNowInstance, table: str) -> dict: """ # Query the Meta API to get most of the column info (e.g., valid choices) - response = requests.get( + response = _request_with_retry( + method="GET", url=instance.snow_url + f"/api/now/ui/meta/{table}", auth=instance.snow_credentials, headers=SNOW_API_HEADERS, ) - response.raise_for_status() meta_info = response.json()["result"]["columns"] # Clean column value choices @@ -164,11 +188,9 @@ def db_delete_from_table(instance: SNowInstance, sys_id: str, table: str) -> Non """ # Query API - response = requests.delete( + _request_with_retry( + method="DELETE", url=instance.snow_url + f"/api/now/table/{table}/{sys_id}", auth=instance.snow_credentials, headers=SNOW_API_HEADERS, ) - - # Check for HTTP code 200 (fail otherwise) - response.raise_for_status() diff --git a/src/browsergym/workarena/tasks/form.py b/src/browsergym/workarena/tasks/form.py index c32e83d..6a21e81 100644 --- a/src/browsergym/workarena/tasks/form.py +++ b/src/browsergym/workarena/tasks/form.py @@ -526,7 +526,11 @@ def show_field_tab(field): # Check if the record was created if self.check_record_created: # This does not work if multiple forms are created at once. The localStorage returns null after the first form - for attempt in range(5): + # Give this the same budget as other browser-side waits: under load, the record can + # take a while to become queryable via the API even though it was genuinely created. + wait_per_attempt_ms = 1500 + max_attempts = SNOW_BROWSER_TIMEOUT // wait_per_attempt_ms + for attempt in range(max_attempts): # in update tasks, the sys_id is already known as the asset is created from the start if update: sys_id = self.record_sys_id @@ -544,8 +548,8 @@ def show_field_tab(field): )["result"] if len(record) > 0: break - page.wait_for_timeout(1500) - if attempt == 4: + page.wait_for_timeout(wait_per_attempt_ms) + if attempt == max_attempts - 1: raise ValueError("The record was not created.") def _set_required_config_attributes(self, config: dict) -> None: From 75fb86f00eba68ccf6a245ecf8ad9c5f6698ca8d Mon Sep 17 00:00:00 2001 From: Alexandre Drouin Date: Mon, 28 Sep 2026 13:11:19 -0400 Subject: [PATCH 2/2] Fix strict-mode locator violation in change-request creation cheat The re-run of the nightly shard on this branch dropped from 9 failures to 1: iframe.get_by_label("All").click() in the change-request creation flow hit a Playwright strict-mode violation because "All" substring-matches both the intended category-filter link and the list view's unrelated "Select All" checkbox. Disambiguate with .first, matching the identical pattern already used one line below for "Normal". Verified against the live pool by reproducing the exact failing case (seed 271, level 3, NavigateAndCreateChangeRequestTask) before and after the fix. Co-Authored-By: Claude Sonnet 5 --- src/browsergym/workarena/tasks/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/browsergym/workarena/tasks/form.py b/src/browsergym/workarena/tasks/form.py index 6a21e81..ac3adcb 100644 --- a/src/browsergym/workarena/tasks/form.py +++ b/src/browsergym/workarena/tasks/form.py @@ -819,7 +819,8 @@ def cheat(self, page: Page, chat_messages: list[str]) -> None: # On the change request page, additional steps need to be taken to open the form if self.table_label == "change request": self._wait_for_ready(page, iframe_only=True) - iframe.get_by_label("All").click() + # "All" also substring-matches the list view's "Select All" checkbox label + iframe.get_by_label("All").first.click() iframe.get_by_text("Normal").first.click() self._fill_fields(page, iframe, self.task_fields)