diff --git a/tests/test_ai_scriptless_definitions.py b/tests/test_ai_scriptless_definitions.py index 3dd1dc8..d4b5def 100644 --- a/tests/test_ai_scriptless_definitions.py +++ b/tests/test_ai_scriptless_definitions.py @@ -34,6 +34,7 @@ CommandContract, command_contract, empty_mandatory_note, + handset_constant_note, validate_argument_names, ) @@ -89,6 +90,43 @@ def test_accepts_variable_data_source_form(self): ) is None +class TestHandsetConstantNote: + HANDSET_COMMAND = CommandContract( + "handset_ready", + mandatory=frozenset({"handsetId"}), + parameters={"handsetId": ParameterContract( + name="handsetId", data_type="HANDSET", mandatory=True, + data_sources=("CONSTANT", "VARIABLE", "DATATABLE"))}, + element_type="Action", + ) + + def test_notes_a_device_pinned_to_a_literal_id(self): + note = handset_constant_note( + "handset_ready", + {"handsetId": {"data_source": "CONSTANT", "value": "DEVICE-123"}}, + self.HANDSET_COMMAND, + ) + assert "handsetId" in note + assert "No handset is available" in note + + def test_no_note_when_bound_to_a_device_variable(self): + assert handset_constant_note( + "handset_ready", + {"handsetId": {"data_source": "VARIABLE", "value": "android"}}, + self.HANDSET_COMMAND, + ) is None + + def test_no_note_for_a_constant_on_another_parameter_type(self): + contract = CommandContract( + "type", + parameters={"text": ParameterContract(name="text", data_type="STRING")}, + element_type="Action", + ) + assert handset_constant_note( + "type", {"text": {"data_source": "CONSTANT", "value": "hello"}}, contract + ) is None + + class TestEmptyMandatoryNote: def test_notes_mandatory_left_empty_by_spec_default(self): note = empty_mandatory_note("ai_user-action", None, USER_ACTION) diff --git a/tests/test_ai_scriptless_manager.py b/tests/test_ai_scriptless_manager.py index 5874e35..2453399 100644 --- a/tests/test_ai_scriptless_manager.py +++ b/tests/test_ai_scriptless_manager.py @@ -114,7 +114,245 @@ def _assert_step_path_notes(result: BaseResult) -> None: assert note in result.result["notes"] +def script_with_device(name, value, referenced=True): + """A script declaring one device parameter, optionally used by a step.""" + script = new_empty_script() + script["parameters"].append({ + "@type": "Parameter", + "data": { + "@type": "HandsetData", + "key": "IMEI" if value else None, + "value": value, + "secured": False, + "description": None, + "displayName": None, + "name": name, + }, + }) + if referenced: + script["flowElements"].append({ + "@type": "Action", + "command": "handset", + "subcommand": "ready", + "arguments": [{ + "@type": "FunctionArgument", + "name": "handsetId", + "data": {"@type": "VariableArgumentData", "dataSource": "VARIABLE", "value": name}, + }], + }) + return script + + +class TestExecuteTestDevicePreflight: + """A device that cannot be reserved answers 'No handset is available', which names + neither the parameter nor the device. The preflight says both before the run starts.""" + + INVENTORY = { + "DEVICE-OK": {"deviceId": "DEVICE-OK", "available": "true", "inUse": "false", + "status": "Connected"}, + "DEVICE-BUSY": {"deviceId": "DEVICE-BUSY", "available": "true", "inUse": "true", + "status": "Connected"}, + "DEVICE-OFFLINE": {"deviceId": "DEVICE-OFFLINE", "available": "false", "inUse": None, + "status": "Not Connected"}, + } + + @pytest.fixture + def cloud(self, monkeypatch): + state = {"script": new_empty_script(), "inventory": dict(self.INVENTORY), "launched": False} + + async def fake_fetch_script_payload(_token, _test_id): + return BaseResult(result={"script": state["script"]}) + + async def fake_inventory(_self): + return state["inventory"] + + async def fake_api_request(_token, method, endpoint=None, **kwargs): + state["launched"] = True + return BaseResult(result={"executionId": "exec-1"}) + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch_script_payload) + monkeypatch.setattr(AiScriptlessManager, "_real_device_inventory", fake_inventory) + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + return state + + def run(self, perfecto_token, devices=None, dut="DEVICE-OK"): + manager = AiScriptlessManager(make_ctx(perfecto_token)) + entries = {"DUT": {"device_id": dut}} if dut else {} + entries.update(devices or {}) + return asyncio.run(manager.execute_test("PRIVATE:Folder/Test.xml", entries)) + + def test_names_the_parameter_left_without_a_device(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", None) + result = self.run(perfecto_token) + + assert "android" in result.error + assert "No handset is available" in result.error + assert cloud["launched"] is False + + def test_ignores_an_empty_declaration_no_step_uses(self, perfecto_token, cloud): + # Empty and unused reserves nothing, so it cannot fail the run. + cloud["script"] = script_with_device("android", None, referenced=False) + result = self.run(perfecto_token) + + assert result.error is None + assert cloud["launched"] is True + + def test_checks_a_device_held_by_a_declaration_no_step_uses(self, perfecto_token, cloud): + # A device parameter holding a device is reserved whether or not a step names it, + # so a device retired since it was configured fails the run from an unused variable. + cloud["script"] = script_with_device("leftover", "DEVICE-GONE", referenced=False) + result = self.run(perfecto_token) + + assert "leftover" in result.error + assert cloud["launched"] is False + + def test_names_a_device_the_cloud_no_longer_lists(self, perfecto_token, cloud): + # The device was configured once and the administrators retired it since. + cloud["script"] = script_with_device("android", "DEVICE-GONE") + result = self.run(perfecto_token) + + assert "android" in result.error + assert "DEVICE-GONE" in result.error + assert "may have been retired" in result.error + assert cloud["launched"] is False + + def test_names_a_device_that_is_disconnected(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-OFFLINE") + result = self.run(perfecto_token) + + assert "Not Connected" in result.error + assert cloud["launched"] is False + + def test_checks_the_device_the_caller_passes_too(self, perfecto_token, cloud): + result = self.run(perfecto_token, dut="DEVICE-GONE") + + assert "device parameter 'DUT'" in result.error + assert "DEVICE-GONE" in result.error + assert cloud["launched"] is False + + def test_a_device_passed_in_devices_replaces_the_stored_one(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-GONE") + result = self.run(perfecto_token, devices={"android": {"device_id": "DEVICE-OK"}}) + + assert result.error is None + assert cloud["launched"] is True + + def test_without_an_override_it_runs_on_the_stored_device(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-OK") + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test("PRIVATE:Folder/Test.xml")) + + assert result.error is None + assert cloud["launched"] is True + assert any("android=DEVICE-OK (stored in the test)" in note for note in result.info) + + def test_an_override_is_reported_as_coming_from_the_call(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-OFFLINE") + result = self.run(perfecto_token, devices={"android": {"device_id": "DEVICE-OK"}}) + + assert result.error is None + assert any("android=DEVICE-OK (passed in this call)" in note for note in result.info) + + def test_a_dut_left_empty_asks_for_its_entry(self, perfecto_token, cloud): + script = new_empty_script() + script["flowElements"].append({ + "@type": "Action", + "command": "handset", + "subcommand": "ready", + "arguments": [{ + "@type": "FunctionArgument", + "name": "handsetId", + "data": {"@type": "VariableArgumentData", "dataSource": "VARIABLE", "value": "DUT"}, + }], + }) + cloud["script"] = script + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test("PRIVATE:Folder/Test.xml")) + + assert "DUT holds no device and devices_under_test has no entry for it" in result.error + assert cloud["launched"] is False + + def test_sees_a_device_used_only_inside_a_condition(self, perfecto_token, cloud): + # Perfecto can answer with branches[] empty and the children under thenClause. + script = new_empty_script() + script["parameters"].append({ + "@type": "Parameter", + "data": {"@type": "HandsetData", "key": None, "value": None, "secured": False, + "description": None, "displayName": None, "name": "inner"}, + }) + step = { + "@type": "Action", + "command": "handset", + "subcommand": "ready", + "arguments": [{ + "@type": "FunctionArgument", + "name": "handsetId", + "data": {"@type": "VariableArgumentData", "dataSource": "VARIABLE", "value": "inner"}, + }], + } + script["flowElements"].append({ + "@type": "IfStatement", + "branches": [{"@type": "Branch", "flowElements": []}, + {"@type": "Branch", "flowElements": []}], + "thenClause": {"@type": "Branch", "flowElements": [step]}, + "elseClause": {"@type": "Branch", "flowElements": []}, + }) + cloud["script"] = script + result = self.run(perfecto_token) + + assert "inner" in result.error + assert cloud["launched"] is False + + def test_a_busy_device_warns_and_still_runs(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-BUSY") + result = self.run(perfecto_token) + + assert result.error is None + assert any("in use right now" in w.lower() for w in result.warning) + assert cloud["launched"] is True + + def test_capabilities_values_are_left_alone(self, perfecto_token, cloud): + # A virtual or desktop device is stored as capabilities, not as a handset id. + cloud["script"] = script_with_device("web", '{"platformName":"Windows"}') + result = self.run(perfecto_token) + + assert result.error is None + assert cloud["launched"] is True + + def test_an_unreadable_inventory_lets_the_run_start(self, perfecto_token, cloud): + cloud["script"] = script_with_device("android", "DEVICE-GONE") + cloud["inventory"] = None + result = self.run(perfecto_token) + + assert result.error is None + assert cloud["launched"] is True + + class TestExecuteTestDeviceMapping: + @pytest.fixture(autouse=True) + def preflight_finds_nothing_to_check(self, monkeypatch): + """These tests are about the params the call builds, not about the preflight. + + The script declares every parameter these tests name, and the device inventory reads + as unavailable, which is the preflight's "cannot establish anything" path. + """ + script = new_empty_script() + for name in ("android", "ios", "web"): + script["parameters"].append({ + "@type": "Parameter", + "data": {"@type": "HandsetData", "key": None, "value": None, "secured": False, + "description": None, "displayName": None, "name": name}, + }) + + async def fake_fetch_script_payload(_token, _test_id): + return BaseResult(result={"script": script}) + + async def no_inventory(_self): + return None + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch_script_payload) + monkeypatch.setattr(AiScriptlessManager, "_real_device_inventory", no_inventory) + def test_real_device_accepts_snake_case_device_id(self, perfecto_token, monkeypatch): captured: dict = {} @@ -127,8 +365,7 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "real", - {"device_id": "DEVICE-123"}, + {"DUT": {"device_id": "DEVICE-123"}}, )) assert result.error is None @@ -147,22 +384,150 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "real", - {"deviceId": "DEVICE-456"}, + {"DUT": {"deviceId": "DEVICE-456"}}, )) assert result.error is None assert captured["json"]["params"]["DUT"] == "DEVICE-456" - def test_real_device_requires_device_id(self, perfecto_token): + def test_every_entry_becomes_its_own_param(self, perfecto_token, monkeypatch): + captured: dict = {} + + async def fake_api_request(_token, method, endpoint=None, **kwargs): + captured["json"] = kwargs.get("json") + return BaseResult(result={"executionId": "exec-1"}) + + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "real", - {}, + { + "DUT": {"device_id": "DEVICE-DUT"}, + "android": {"device_id": "DEVICE-A"}, + "ios": {"device_id": "DEVICE-B"}, + }, + )) + + assert result.error is None + # DUT is one parameter name among the others, not a separate kind of argument. + assert captured["json"]["params"] == { + "DUT": "DEVICE-DUT", + "android": "DEVICE-A", + "ios": "DEVICE-B", + } + + def test_entries_may_mix_device_types(self, perfecto_token, monkeypatch): + captured: dict = {} + + async def fake_api_request(_token, method, endpoint=None, **kwargs): + captured["json"] = kwargs.get("json") + return BaseResult(result={"executionId": "exec-1"}) + + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test( + "PRIVATE:Folder/Test.xml", + { + "DUT": {"device_id": "DEVICE-DUT"}, + "web": { + "device_type": "desktop", + "platform_name": "Windows", + "platform_version": "11", + "browser_name": "Chrome", + "browser_version": "143", + "resolution": "1280x1024", + "location": "US East", + }, + }, + )) + + assert result.error is None + assert captured["json"]["params"]["DUT"] == "DEVICE-DUT" + assert json.loads(captured["json"]["params"]["web"])["browserName"] == "Chrome" + + def test_the_retired_arguments_are_answered_by_name(self, perfecto_token, monkeypatch): + # device_type/device_under_test/devices were one concept split in three. An agent working + # from an older description deserves the new shape, not a run on the wrong devices. + tool = _register_tool(perfecto_token) + result = asyncio.run(_call_tool(tool, "execute_test", { + "test_id": "PRIVATE:Folder/Test.xml", + "device_type": "real", + "device_under_test": {"device_id": "DEVICE-123"}, + })) + + assert "device_type, device_under_test" in result.error + assert "devices_under_test maps a device parameter name" in result.error + + def test_an_entry_the_test_does_not_declare_is_refused(self, perfecto_token, monkeypatch): + captured: dict = {} + + async def fake_api_request(_token, method, endpoint=None, **kwargs): + captured["json"] = kwargs.get("json") + return BaseResult(result={"executionId": "exec-1"}) + + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test( + "PRIVATE:Folder/Test.xml", + {"DUT": {"device_id": "DEVICE-DUT"}, "andriod": {"device_id": "DEVICE-A"}}, + )) + + # Perfecto drops an unknown params entry silently and runs on the stored device, so a + # misspelled name would otherwise come back as a green run on the wrong phone. + assert "'andriod'" in result.error + assert "android" in result.error + assert "json" not in captured + + def test_reports_the_entry_that_is_missing_its_device_id(self, perfecto_token): + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test( + "PRIVATE:Folder/Test.xml", + {"DUT": {"device_id": "DEVICE-DUT"}, "android": {}}, + )) + + assert "devices_under_test['android']" in result.error + + def test_reports_an_entry_it_cannot_read(self, perfecto_token): + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test( + "PRIVATE:Folder/Test.xml", + {"DUT": {"model": "Galaxy S10"}}, )) assert "device_id could not be found" in result.error + def test_an_entry_that_is_not_an_object_is_reported(self, perfecto_token): + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test( + "PRIVATE:Folder/Test.xml", + {"DUT": "DEVICE-123"}, + )) + assert "devices_under_test maps a device parameter name" in result.error + + def test_a_devices_under_test_that_is_not_a_map_is_reported(self, perfecto_token): + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test("PRIVATE:Folder/Test.xml", "DEVICE-123")) + assert "Invalid devices_under_test" in result.error + + def test_no_devices_under_test_sends_no_params(self, perfecto_token, monkeypatch): + captured: dict = {} + + async def fake_api_request(_token, method, endpoint=None, **kwargs): + captured["json"] = kwargs.get("json") + return BaseResult(result={"executionId": "exec-stored"}) + + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + + manager = AiScriptlessManager(make_ctx(perfecto_token)) + result = asyncio.run(manager.execute_test("PRIVATE:Folder/Test.xml")) + + # Perfecto only overrides a device parameter that the run supplies, so empty params + # is what "run on the devices the test stores" looks like on the wire. + assert result.error is None + assert captured["json"]["params"] == {} + def test_virtual_device_serializes_capabilities_json(self, perfecto_token, monkeypatch): captured: dict = {} @@ -175,13 +540,13 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "virtual", - { + {"DUT": { + "device_type": "virtual", "platform_name": "Android", "manufacturer": "Google", "model": "Pixel 8", "platform_version": "14", - }, + }}, )) assert result.error is None @@ -201,8 +566,7 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "virtual", - {"platform_name": "Android"}, + {"DUT": {"device_type": "virtual", "platform_name": "Android"}}, )) assert result.error is None @@ -215,10 +579,10 @@ def test_invalid_device_type_returns_error(self, perfecto_token): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( "PRIVATE:Folder/Test.xml", - "unknown", - {"device_id": "x"}, + {"DUT": {"device_type": "unknown", "device_id": "x"}}, )) - assert result.error == "Invalid device_type or device_under_test value." + assert "Invalid device_type 'unknown' for devices_under_test['DUT']" in result.error + assert "Accepted values: desktop, real, virtual" in result.error def test_desktop_device_serializes_capabilities_json(self, perfecto_token, monkeypatch): captured: dict = {} @@ -232,15 +596,15 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( TEST_ID, - "desktop", - { + {"DUT": { + "device_type": "desktop", "platform_name": "Windows", "platform_version": "11", "browser_name": "Chrome", "browser_version": "120", "resolution": "1920x1080", "location": "US", - }, + }}, )) assert result.error is None @@ -261,8 +625,7 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): manager = AiScriptlessManager(make_ctx(perfecto_token)) result = asyncio.run(manager.execute_test( TEST_ID, - "desktop", - {"platform_name": "Windows"}, + {"DUT": {"device_type": "desktop", "platform_name": "Windows"}}, )) assert result.error is None @@ -1331,13 +1694,20 @@ async def fake_api_request(_token, method, endpoint=None, **kwargs): captured["json"] = kwargs.get("json") return BaseResult(result={"executionId": "exec-dispatch"}) + async def fake_fetch_script_payload(_token, _test_id): + return BaseResult(result={"script": new_empty_script()}) + + async def no_inventory(_self): + return None + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch_script_payload) + monkeypatch.setattr(AiScriptlessManager, "_real_device_inventory", no_inventory) tool = _register_tool(perfecto_token) result = asyncio.run(_call_tool(tool, "execute_test", { "test_id": TEST_ID, - "device_type": "real", - "device_under_test": {"device_id": "DEV-1"}, + "devices_under_test": {"DUT": {"device_id": "DEV-1"}}, })) assert result.error is None diff --git a/tests/test_ai_scriptless_script.py b/tests/test_ai_scriptless_script.py index 6fc141c..b2b6467 100644 --- a/tests/test_ai_scriptless_script.py +++ b/tests/test_ai_scriptless_script.py @@ -318,6 +318,45 @@ def test_add_and_find_variable(self): assert entry["data"]["value"] == 0 assert len(script["variables"]) == 1 + def test_add_device_variable_matches_what_the_ui_writes(self): + # The Select device dialog stores the device identifier under the IMEI selector, + # the same shape DUT has once a device is picked. + script = new_empty_script() + entry = add_script_variable(script, "android", "device", "DEVICE-123", set_at_runtime=True) + assert entry["@type"] == "Parameter" + assert entry["data"] == { + "@type": "HandsetData", + "description": None, + "displayName": None, + "key": "IMEI", + "name": "android", + "secured": False, + "value": "DEVICE-123", + } + + def test_device_variable_without_a_device_is_the_runtime_picker(self): + # Empty means "Select device" in the UI: null, not "", exactly as DUT is declared. + script = new_empty_script() + entry = add_script_variable(script, "android", "device", "", set_at_runtime=True) + assert entry["data"]["value"] is None + assert entry["data"]["key"] is None + + def test_device_variable_accepts_capabilities_for_a_virtual_or_desktop_device(self): + script = new_empty_script() + entry = add_script_variable( + script, "web", "device", {"platformName": "Windows", "browserName": "Chrome"}) + assert entry["data"]["value"] == '{"platformName":"Windows","browserName":"Chrome"}' + assert entry["data"]["key"] == "IMEI" + + def test_modify_variable_converts_a_string_to_a_device(self): + # The conversion the UI's "Replace variable type" dialog performs. + script = new_empty_script() + add_script_variable(script, "phone", "string", "DEVICE-123") + variable = modify_script_variable(script, "phone", variable_type="device") + assert variable["data"]["@type"] == "HandsetData" + assert variable["data"]["key"] == "IMEI" + assert variable["data"]["value"] == "DEVICE-123" + def test_add_variable_rejects_duplicate(self): script = new_empty_script() add_script_variable(script, "token", "string", "abc") @@ -694,11 +733,21 @@ def test_delete_finds_a_runtime_parameter(self): delete_script_variable(script, "runtime") assert [p["data"]["name"] for p in script["parameters"]] == ["DUT"] - def test_dut_is_listed_but_protected(self): + def test_dut_takes_a_default_device_but_keeps_its_role(self): + # The UI's Select device dialog sets a device on DUT; what it never does is turn DUT + # into another type or take it out of the runtime parameters. script = new_empty_script() assert [v["data"]["name"] for v in list_script_variables(script)] == ["DUT"] - with pytest.raises(ValueError, match="cannot be modified"): - modify_script_variable(script, "DUT", value="DEVICE-1") + + variable = modify_script_variable(script, "DUT", value="DEVICE-1") + assert variable["data"]["value"] == "DEVICE-1" + assert variable["data"]["key"] == "IMEI" + assert script["parameters"][0]["data"]["name"] == "DUT" + + with pytest.raises(ValueError, match="cannot become a string"): + modify_script_variable(script, "DUT", variable_type="string") + with pytest.raises(ValueError, match="cannot stop being a runtime parameter"): + modify_script_variable(script, "DUT", set_at_runtime=False) with pytest.raises(ValueError, match="breaks the test"): delete_script_variable(script, "DUT") diff --git a/tools/ai_scriptless/__init__.py b/tools/ai_scriptless/__init__.py index 205793d..dcacc2d 100644 --- a/tools/ai_scriptless/__init__.py +++ b/tools/ai_scriptless/__init__.py @@ -11,6 +11,7 @@ coerce_argument_value, command_contract, empty_mandatory_note, + handset_constant_note, reset_command_contract_cache, restriction_allowed_values, restriction_range, @@ -73,6 +74,8 @@ VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE, bindable_values, describe_bindable_values, + device_declarations, + referenced_variable_names, variable_type_label, VARIABLE_TYPE_ALIASES, add_script_variable, @@ -102,6 +105,8 @@ "VARIABLE_TYPE_ALIASES", "bindable_values", "describe_bindable_values", + "device_declarations", + "referenced_variable_names", "variable_type_label", "VISIBILITY_UI_ROOT", "_persist_script", @@ -128,6 +133,7 @@ "delete_element_by_path", "delete_script_variable", "empty_mandatory_note", + "handset_constant_note", "fetch_script_payload", "find_container_by_path", "find_element_by_path", diff --git a/tools/ai_scriptless/definitions.py b/tools/ai_scriptless/definitions.py index 5a5dacb..7a30024 100644 --- a/tools/ai_scriptless/definitions.py +++ b/tools/ai_scriptless/definitions.py @@ -534,6 +534,41 @@ def validate_variable_bindings( return f"Invalid cmd_arguments for command '{command_id}': " + " ".join(errors) +def handset_constant_note( + command_id: str, + cmd_arguments: Optional[dict[str, Any]], + contract: Optional[CommandContract], +) -> Optional[str]: + """Note when a device parameter is pinned to a literal device id. + + Perfecto reserves one device per device declaration, not per argument, so a CONSTANT + device id is accepted on save and then fails the run with "No handset is available". + The step has to name a device variable the test declares. + """ + if not cmd_arguments or not contract or not contract.parameters: + return None + + spec = get_command_spec(command_id) + pinned = [] + for name, raw in spec.normalize_argument_names(cmd_arguments).items(): + if not isinstance(raw, dict) or "data_source" not in raw: + continue + if str(raw.get("data_source") or "").upper() != "CONSTANT": + continue + parameter = contract.parameter(name) + if parameter is None or (parameter.data_type or "").upper() != "HANDSET": + continue + pinned.append(name) + if not pinned: + return None + return ( + f"Device parameter(s) set to a literal device id on '{command_id}': {', '.join(sorted(pinned))}. " + "The step saves, but the run fails with 'No handset is available': Perfecto reserves a device per " + "device declaration, not per argument. Bind it to a device variable instead " + "(add_test_variable with variable_type='device', then data_source VARIABLE)." + ) + + def empty_mandatory_note( command_id: str, cmd_arguments: Optional[dict[str, Any]], diff --git a/tools/ai_scriptless/variables.py b/tools/ai_scriptless/variables.py index 210c26c..1bc448a 100644 --- a/tools/ai_scriptless/variables.py +++ b/tools/ai_scriptless/variables.py @@ -1,3 +1,4 @@ +import json from typing import Any, Optional @@ -11,7 +12,12 @@ "datatable": "TableData", } -SUPPORTED_VARIABLE_TYPES = frozenset({"string", "secured_string", "number", "boolean"}) +SUPPORTED_VARIABLE_TYPES = frozenset({"string", "secured_string", "number", "boolean", "device"}) + +# A device is stored as the identifier Perfecto selects the handset by; "IMEI" is the +# selector the Select device dialog writes for every device it offers (real, virtual and +# desktop web alike), with the identifier itself in value. +HANDSET_SELECTOR_KEY = "IMEI" def validate_variable_name(name: str) -> None: @@ -42,6 +48,15 @@ def _coerce_variable_value(variable_type: str, value: Any) -> Any: except (TypeError, ValueError) as exc: raise ValueError("number value must be an integer") from exc return numeric + if variable_type == "device": + # An unset device is the runtime picker the UI shows as "Select device": null, not "". + if value is None or value == "": + return None + # Real devices are a device_id string; virtual and desktop web are the capabilities + # object, serialized the way execute_test sends it. + if isinstance(value, dict): + return json.dumps(value, separators=(',', ':')) + return str(value) return "" if value is None else str(value) @@ -63,7 +78,7 @@ def build_variable_data(variable_type: str, name: str, value: Any) -> dict[str, "value": coerced_value, } if data_type == "HandsetData": - data["key"] = None + data["key"] = HANDSET_SELECTOR_KEY if coerced_value else None if data_type == "TableData": data["columns"] = [] return data @@ -153,11 +168,21 @@ def modify_script_variable( if located is None: raise ValueError(f"variable not found: {variable_name}") array_key, index, variable = located - if variable_name == DUT_NAME: - raise ValueError( - "DUT is the device parameter execute_test fills in; it cannot be modified here" - ) current_type = _variable_type_from_data(variable.get("data", {})) + if variable_name == DUT_NAME: + # DUT can hold a default device, the way the UI's Select device dialog sets it, but it + # stays the runtime device parameter every step binds handsetId to: changing its type + # or moving it out of the runtime parameters breaks every step at once. + if variable_type not in (None, current_type): + raise ValueError( + f"DUT is the device parameter every step binds handsetId to; " + f"it cannot become a {variable_type}" + ) + if set_at_runtime is False: + raise ValueError( + "DUT is supplied when the run starts; it cannot stop being a runtime parameter" + ) + set_at_runtime = None target_type = variable_type or current_type if target_type not in SUPPORTED_VARIABLE_TYPES: raise ValueError( @@ -236,6 +261,56 @@ def bindable_values(script: dict[str, Any]) -> dict[str, dict[str, Any]]: return bindable +def device_declarations(script: dict[str, Any]) -> dict[str, Any]: + """Every device declaration by name, with the device it stores (None when unset).""" + declarations: dict[str, Any] = {} + for entry in list_script_variables(script): + data = entry.get("data", {}) if isinstance(entry, dict) else {} + if data.get("@type") != "HandsetData": + continue + name = data.get("name") + if name: + declarations[name] = data.get("value") + return declarations + + +def referenced_variable_names(script: dict[str, Any]) -> set[str]: + """Every variable name the steps bind an argument to, at any depth. + + For a device parameter this answers only half the question: one holding a device is + reserved whether or not a step names it, so a caller weighing what a run costs asks + device_declarations first and this second, for the empty ones a step still needs. + """ + referenced: set[str] = set() + + def read_arguments(element: dict[str, Any]) -> None: + for argument in element.get("arguments") or []: + if not isinstance(argument, dict): + continue + data = argument.get("data") or {} + if str(data.get("dataSource") or "").upper() != "VARIABLE": + continue + value = data.get("value") + if isinstance(value, str) and value: + referenced.add(value) + for validation in element.get("validations") or []: + if isinstance(validation, dict): + read_arguments(validation) + + def walk(elements: Optional[list]) -> None: + for element in elements or []: + if not isinstance(element, dict): + continue + read_arguments(element) + walk(element.get("flowElements")) + for branch in element.get("branches") or []: + if isinstance(branch, dict): + walk(branch.get("flowElements")) + + walk(script.get("flowElements")) + return referenced + + def describe_bindable_values(script: dict[str, Any]) -> str: described = [ f"{name} ({variable_type_label(data)})" diff --git a/tools/ai_scriptless_manager.py b/tools/ai_scriptless_manager.py index 1a90f1a..4074885 100644 --- a/tools/ai_scriptless_manager.py +++ b/tools/ai_scriptless_manager.py @@ -19,6 +19,7 @@ from models.result import BaseResult, PaginationResult from telemetry import run_tool from tools.ai_scriptless_script import ( + DUT_NAME, add_script_variable, build_flow_element, build_if_statement, @@ -35,9 +36,12 @@ fetch_script_payload, find_element_by_path, find_step_path_for_element, + handset_constant_note, insert_flow_element, bindable_values, describe_bindable_values, + device_declarations, + referenced_variable_names, list_script_variables, variable_type_label, load_and_mutate, @@ -184,73 +188,242 @@ async def list_filter_values(self, filter_names: list[str]) -> BaseResult: warning=warnings, ) - @token_verify - async def execute_test(self, test_id: str, device_type: str, device_under_test: dict[str, Any]) -> BaseResult: - execute_url = perfecto.get_ai_scriptless_execution_api_url(self.token.cloud_name) - - # This mapping allows us to detect when the AI gets confused and uses Perfecto-style capabilities. - # It also allows for reverse mapping from internal to capabilities from Perfecto. - att_map = { - "real": { - "device_id": "deviceId" - }, - "virtual": { - "platform_name": "platformName", - "manufacturer": "manufacturer", - "model": "model", - "platform_version": "platformVersion" - }, - "desktop": { - "platform_name": "platformName", - "platform_version": "platformVersion", - "browser_name": "browserName", - "browser_version": "browserVersion", - "resolution": "resolution", - "location": "location" - } + # This mapping allows us to detect when the AI gets confused and uses Perfecto-style capabilities. + # It also allows for reverse mapping from internal to capabilities from Perfecto. + DEVICE_ATTRIBUTE_MAP = { + "real": { + "device_id": "deviceId" + }, + "virtual": { + "platform_name": "platformName", + "manufacturer": "manufacturer", + "model": "model", + "platform_version": "platformVersion" + }, + "desktop": { + "platform_name": "platformName", + "platform_version": "platformVersion", + "browser_name": "browserName", + "browser_version": "browserVersion", + "resolution": "resolution", + "location": "location" } + } + + def _resolve_device_param( + self, + device_type: str, + device: dict[str, Any], + label: str, + ) -> tuple[Optional[str], Optional[str]]: + """The value a device parameter takes at run start: (value, error). + + A real device is its device_id; virtual and desktop are the capabilities object, + serialized the way the test stores a device variable's value. + """ + att_map = self.DEVICE_ATTRIBUTE_MAP.get(device_type) + if att_map is None: + accepted = ", ".join(sorted(self.DEVICE_ATTRIBUTE_MAP)) + return None, ( + f"Invalid device_type '{device_type}' for {label}. Accepted values: {accepted}." + ) - dut = None - remapped_device_under_test = {} + remapped = {} # Remap the attributes to Perfecto Capabilities format - if device_type in att_map.keys(): - for key in att_map[device_type].keys(): - alt_key = att_map[device_type][key] - remapped_device_under_test[alt_key] = device_under_test.get(key, device_under_test.get(alt_key, None)) + for key, alt_key in att_map.items(): + remapped[alt_key] = device.get(key, device.get(alt_key, None)) if device_type == "real": - dut = remapped_device_under_test.get("deviceId", None) - if dut is None: - return BaseResult( - error="Invalid value for device_under_test. The key device_id could not be found." + device_id = remapped.get("deviceId", None) + if device_id is None: + return None, f"Invalid value for {label}. The key device_id could not be found." + return device_id, None + + # Verify if all the needed keys exist on the remapped version + key_not_found = [key for key, alt_key in att_map.items() if alt_key not in remapped] + if key_not_found: + keys_not_found_str = ",".join(key_not_found) + return None, f"Invalid value for {label}. The keys [{keys_not_found_str}] could not be found." + return json.dumps(remapped, separators=(',', ':')), None + + async def _real_device_inventory(self) -> Optional[dict[str, dict[str, Any]]]: + """Every handset this cloud knows, by device id, or None when it cannot be read. + + list_real_devices only returns the available ones, and a device that was retired has + to be told apart from one that is merely busy, so this reads the unfiltered listing. + """ + devices_url = perfecto.get_real_device_management_api_url(self.token.cloud_name) + result = await api_request(self.token, "POST", endpoint=devices_url, json={"device": {}}) + if result.error or not isinstance(result.result, dict): + return None + inventory: dict[str, dict[str, Any]] = {} + for group in result.result.values(): + if not isinstance(group, dict): + continue + for handset in group.get("handset") or []: + device_id = handset.get("deviceId") if isinstance(handset, dict) else None + if device_id: + inventory[device_id] = handset + return inventory + + @staticmethod + def _device_unusable_reason(device_id: Any, inventory: dict[str, dict[str, Any]]) -> Optional[str]: + """Why this device cannot be reserved, or None when it can.""" + # Virtual and desktop devices are stored as capabilities, not as a handset id. + if not isinstance(device_id, str) or device_id.strip().startswith("{"): + return None + handset = inventory.get(device_id) + if handset is None: + return "this cloud does not list it (it may have been retired)" + if str(handset.get("available", "")).lower() != "true": + return f"it is {handset.get('status') or 'not available'}" + return None + + async def _device_preflight( + self, + test_id: str, + supplied: dict[str, Any], + ) -> tuple[Optional[str], Optional[str], Optional[str]]: + """What this run will do with devices, before it starts: (error, warning, summary). + + Perfecto answers an unreservable device with "No handset is available", which says + nothing about which parameter or which device. + + What gets reserved: a device parameter holding a device, whether or not a step names + it, plus every parameter a run supplies. One left empty is skipped, and only breaks + the run when a step names it. Anything that cannot be established leaves the run alone. + """ + payload = await fetch_script_payload(self.token, test_id) + if payload.error or not isinstance(payload.result, dict): + return None, None, None + script = payload.result.get("script", {}) + # Perfecto can answer with an IfStatement whose branches[] is empty and whose + # thenClause holds the children; without this the steps inside it are invisible. + normalize_if_statement_aliases(script) + declarations = device_declarations(script) + referenced = referenced_variable_names(script) + + # Perfecto drops a params entry whose name it does not know, without a word, and runs + # on the stored device instead: a misspelled parameter would pass as a green run. + unknown = sorted(name for name in supplied if name not in declarations) + if unknown: + declared = ", ".join(sorted(declarations)) or "none" + return ( + f"devices_under_test names {', '.join(repr(name) for name in unknown)}, which this " + f"test does not declare, and Perfecto would ignore it and run on the stored device. " + f"Device parameters this test declares: {declared}.", + None, + None, + ) + + missing_device = [ + name for name, value in declarations.items() + if name in referenced and name not in supplied and not value + ] + to_verify = {name: value for name, value in supplied.items() if value} + to_verify.update({ + name: value for name, value in declarations.items() + if name not in supplied and value + }) + + inventory = await self._real_device_inventory() if to_verify else None + unusable: list[str] = [] + busy: list[str] = [] + for name, device_id in sorted(to_verify.items()) if inventory is not None else []: + reason = self._device_unusable_reason(device_id, inventory) + source = f"device parameter '{name}'" + if reason: + unusable.append(f"{source} is set to device id '{device_id}', but {reason}.") + elif str((inventory.get(device_id) or {}).get("inUse", "")).lower() == "true": + busy.append(f"{source} ('{device_id}')") + + problems = list(unusable) + for name in sorted(missing_device): + if name == DUT_NAME: + problems.append( + "DUT holds no device and devices_under_test has no entry for it; " + "add one, or store a device on DUT with modify_test_variable." ) - elif device_type in ["virtual", "desktop"]: - # Verify if all the needed keys exist on the remapped version - key_not_found = [] - for key in att_map[device_type].keys(): - alt_key = att_map[device_type][key] - if alt_key not in remapped_device_under_test: - key_not_found.append(key) - if len(key_not_found) == 0: - dut = json.dumps(remapped_device_under_test, separators=(',', ':')) else: - keys_not_found_str = ",".join(key_not_found) - return BaseResult( - error=f"Invalid value for device_under_test. The keys [{keys_not_found_str}] could not be found." + example = ('{"%s": {"device_type": "real", ' + '"device_id": ""}}') % name + problems.append( + f"device parameter '{name}' is used by a step but has no device and none " + f"was passed; add it to devices_under_test, as {example}." ) - if dut is not None and len(dut) > 0: - body = { - "params": { - "DUT": dut - }, - "testKey": test_id, - "triggerType": "Manual" - } - return await api_request(self.token, "POST", endpoint=execute_url, json=body) - else: - return BaseResult( - error="Invalid device_type or device_under_test value." + error = None + if problems: + error = ( + f"This run would fail with 'No handset is available'. " + " ".join(problems) + + " Perfecto reserves a device for every device parameter that holds one, " + "whether or not a step uses it, plus the ones this call supplies. " + "Pick a device with list_real_devices, then either pass it in devices_under_test " + "for this run or store it with modify_test_variable(variable_type='device')." ) + warning = None + if busy: + warning = ( + f"In use right now: {', '.join(busy)}. The run may fail to reserve them." + ) + # Which device each parameter ended up on, so a run can be read back without guessing. + summary = None + if to_verify and not error: + described = [] + for name, device_id in sorted(to_verify.items()): + origin = "passed in this call" if name in supplied else "stored in the test" + described.append(f"{name}={device_id} ({origin})") + summary = "Devices this run uses: " + ", ".join(described) + "." + return error, warning, summary + + DEVICES_SHAPE_HINT = ( + "devices_under_test maps a device parameter name to the device it runs on, as " + '{"DUT": {"device_type": "real", "device_id": ""}}.' + ) + + @token_verify + async def execute_test( + self, + test_id: str, + devices_under_test: Optional[dict[str, dict[str, Any]]] = None, + ) -> BaseResult: + execute_url = perfecto.get_ai_scriptless_execution_api_url(self.token.cloud_name) + + if devices_under_test is not None and not isinstance(devices_under_test, dict): + return BaseResult(error=f"Invalid devices_under_test. {self.DEVICES_SHAPE_HINT}") + + # Perfecto takes one device per parameter name, DUT included: it is the parameter every + # test starts with, not a different kind of argument. A parameter this call leaves out + # keeps the device the test stores, and one that resolves to nothing is simply skipped. + params: dict[str, str] = {} + for name, device in (devices_under_test or {}).items(): + label = f"devices_under_test['{name}']" + if not isinstance(device, dict): + return BaseResult(error=f"Invalid {label}. {self.DEVICES_SHAPE_HINT}") + value, error = self._resolve_device_param( + device.get("device_type", "real"), device, label) + if error: + return BaseResult(error=error) + if not value: + return BaseResult(error=f"Invalid device_type or device value for {label}.") + params[name] = value + + preflight_error, preflight_warning, device_summary = await self._device_preflight( + test_id, params) + if preflight_error: + return BaseResult(error=preflight_error) + + body = { + "params": params, + "testKey": test_id, + "triggerType": "Manual" + } + result = await api_request(self.token, "POST", endpoint=execute_url, json=body) + if not result.error: + if preflight_warning: + result.append_warnings([preflight_warning]) + if device_summary: + result.append_info([device_summary]) + return result @token_verify async def list_commands(self, checkpoint: bool = False) -> BaseResult: @@ -356,6 +529,9 @@ def mutator(script: dict[str, Any]) -> None: empty_note = empty_mandatory_note(command_id, cmd_arguments, contract) if empty_note: result.result.setdefault("notes", []).append(empty_note) + constant_note = handset_constant_note(command_id, cmd_arguments, contract) + if constant_note: + result.result.setdefault("notes", []).append(constant_note) return _append_step_path_refresh_notes(result) @token_verify @@ -367,6 +543,8 @@ async def modify_command(self, test_id: str, step_path: str, cmd_arguments: dict if not cmd_arguments: return BaseResult(error="cmd_arguments is required") + constant_note: dict[str, Optional[str]] = {"note": None} + async def mutator(script: dict[str, Any]) -> None: located = find_element_by_path(script, step_path) if located is None: @@ -384,11 +562,13 @@ async def mutator(script: dict[str, Any]) -> None: bindings_error = validate_variable_bindings(command_id, cmd_arguments, contract, script) if bindings_error: raise ValueError(bindings_error) + constant_note["note"] = handset_constant_note(command_id, cmd_arguments, contract) update_element_arguments(element, cmd_arguments, contract) - return _append_step_path_refresh_notes( - await load_and_mutate(self.token, test_id, mutator) - ) + result = await load_and_mutate(self.token, test_id, mutator) + if not result.error and constant_note["note"]: + result.result.setdefault("notes", []).append(constant_note["note"]) + return _append_step_path_refresh_notes(result) @token_verify async def delete_command(self, test_id: str, step_path: str) -> BaseResult: @@ -835,14 +1015,33 @@ def register(mcp, runtime: AppRuntime): args(dict): Dictionary with the following required filter parameters: filter_names (list[str], values=['test_name', 'owner_list']): The filter name list. - execute_test: Execute a preconfigured AI Scriptless Test. - args(dict): Dictionary with the following required parameters: - test_id (str): Test ID from list_tests - device_type (str, default='real', values=['real', 'virtual', 'desktop']: The device type. - device_under_test (dict, required): Device configuration object. - When device_type='real': {device_id: str} (Get from list_real_devices). - When device_type='virtual': {platform_name: str, manufacturer: str, model: str, platform_version: str} (Get from list_virtual_devices). - When device_type='desktop': {platform_name: str, platform_version: str, browser_name: str, - browser_version: str, resolution: str, location: str} (Get from list_desktop_devices). + A test runs on the devices its own device parameters name. Every test declares at least one, called + DUT; a test driving two phones declares one more per phone (list_test_variables shows them as type + 'device'). Each parameter takes its device from devices_under_test when this call names it, and + otherwise from the device the test stores. A parameter that ends up with no device is skipped, and + only fails the run when a step uses it. What this call passes applies to this run alone: it never + changes what the test stores, so one test runs on different devices without being edited. + args(dict): Dictionary with the following parameters: + test_id (str, required): Test ID from list_tests. + devices_under_test (dict, optional): The device each device parameter runs on, keyed by the + parameter's name exactly as list_test_variables reports it. DUT is one of those names, not a + separate argument. Omit the whole argument to run on the devices the test already stores. + Each value describes one device: + device_type (str, default='real', values=['real', 'virtual', 'desktop']) + When device_type='real': {device_id: str} (Get from list_real_devices). + When device_type='virtual': {platform_name: str, manufacturer: str, model: str, platform_version: str} (Get from list_virtual_devices). + When device_type='desktop': {platform_name: str, platform_version: str, browser_name: str, + browser_version: str, resolution: str, location: str} (Get from list_desktop_devices). + Entries may mix device types, and work on any device parameter whether or not it is + set_at_runtime. Example, for a test declaring DUT plus the parameters 'android' and 'ios' to + place a call from one phone to another, running DUT and 'android' on chosen devices and + leaving 'ios' on the device the test stores: + {"DUT": {"device_type": "real", "device_id": "RF8N30QRX7K"}, + "android": {"device_type": "real", "device_id": "R3CN206EJ7B"}} + Before launching, every device the run would reserve is checked against the cloud's device list, so a + device that was retired or disconnected since the test was authored is named, together with the + parameter holding it, instead of surfacing as 'No handset is available' after the run already failed. + The result reports which device each parameter resolved to and where it came from. - view_test_structure: View the hierarchical structure of an AI Scriptless test. Each step has step_path (dot-separated positional path, e.g. 0, 2.0, 5.b0.1). args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests (e.g. PRIVATE:My Folder/My Test.xml). @@ -972,6 +1171,8 @@ def register(mcp, runtime: AppRuntime): snapshot_id (str): UUID key from list_snapshots (not ''; use view_test_structure for the live script). - list_test_variables: List everything the test declares, exactly as the UI's Configure test variables dialog: runtime parameters first (set_at_runtime=true, including the DUT device parameter), then plain variables. + A 'device' variable reports the device_id it holds, or null when its device is chosen at run start; + read_real_device_info tells whether that device is still connected. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - add_test_variable: Add a script variable or runtime parameter and persist. @@ -979,9 +1180,16 @@ def register(mcp, runtime: AppRuntime): test_id (str): Test itemKey from list_tests. name (str): Variable name (letters, numbers, underscore; cannot start with a number). Must be unique across runtime parameters and variables alike; Perfecto rejects a name declared twice. - variable_type (str, default='string', values=['string', 'secured_string', 'number', 'boolean']): Variable type. + variable_type (str, default='string', values=['string', 'secured_string', 'number', 'boolean', 'device']): + Variable type. 'device' is the type DUT has: it holds a device, and a handsetId argument only binds + to a device variable, so a test that drives more than one device needs one per device. value (any, default=''): Variable value. For secured_string, pass the plaintext: it is encrypted through Perfecto before being stored (the UI's lock button) and never echoed back. + For device, pass the device_id from list_real_devices, or the capabilities object for a virtual or + desktop device ({platform_name, ...}, same keys as execute_test). A device parameter holding a + device is reserved on every run, even when no step uses it, so an unused one costs a device. + Leave it empty for a device chosen at run start, the way DUT is: it then costs nothing until + execute_test names it in devices_under_test. set_at_runtime (bool, default=false): True makes it a runtime variable: the stored value is only the default, it is supplied when the run starts (UI: the 'Set at runtime' checkbox, then the 'Enter runtime values' dialog; execute_test is that same channel, the way DUT receives its device) and it @@ -990,9 +1198,13 @@ def register(mcp, runtime: AppRuntime): - modify_test_variable: Update a script variable and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - name (str): Existing variable name (runtime parameters included; DUT cannot be modified). - value (any, optional): New value. - variable_type (str, optional): New type (string, secured_string, number, boolean). + name (str): Existing variable name, runtime parameters included. DUT accepts a device, which is the + device it runs on when execute_test supplies none, but it cannot change type or stop being a + runtime parameter: every step binds handsetId to it. + value (any, optional): New value. Passing only variable_type keeps the current value. + variable_type (str, optional): New type (string, secured_string, number, boolean, device). Converting a + string variable to device is how a test authored with plain strings becomes multi-device: the + device_id it held is kept. set_at_runtime (bool, optional): Toggle between runtime parameter and variable. - delete_test_variable: Remove a script variable and persist. args(dict): Dictionary with the following required parameters: @@ -1001,6 +1213,9 @@ def register(mcp, runtime: AppRuntime): Hints: - LICENSE: AI Scriptless actions require a Perfecto AI license on your cloud (administrator opt-in via feature toggle). Without it, AI commands and related MCP operations will not work. Desktop web test authoring additionally requires the Desktop Web license. - COVERAGE: DataTables, Scheduler (scheduled jobs), Embedded tests, and other advanced UI capabilities (folder management, rename test, restore snapshot, download as Appium, AI Assistant, Object Spy, per-step error policy, etc.) are not yet supported by this MCP tool. +- The variable types this tool creates are string, secured_string, number, boolean and device. A test may also + declare media and datatable variables, which list_test_variables reports and view_test_step binds, but + add_test_variable and modify_test_variable cannot create or change them: those are authored in the UI. - HELP: For product behavior and workarounds, use the perfecto_skills tool (skill `perfecto-ai-scriptless`) and/or the perfecto_help tool: Filter by category_id='perfecto', subcategory_id_list=['ide']. - UI_ACCESS: No per-test URL exists. Only UI entry: cloud_url/lab/scriptless-mobile/ (cloud_url from perfecto_user read_user). For debugging or unsupported MCP tasks, link the lab URL and tell the user to open the test via Tests → Open or Manage tests using the folder tree and test name from list_tests (itemKey is MCP-only; the UI shows folders and names, not itemKey). Never invent other scriptless URLs. - When authoring or editing test steps, call list_commands first and follow the command selection policy in the info field. @@ -1014,6 +1229,15 @@ def register(mcp, runtime: AppRuntime): - A VARIABLE binding is checked against the test: the variable must exist and its type must match the parameter (a string variable cannot feed a Number parameter), the same rule the UI enforces by only offering compatible variables. The error lists the variables the test defines with their types. +- MULTI-DEVICE: a step runs on the device its handsetId names, and handsetId only accepts a device variable. + To drive a second device (a call from one phone to another, a message between two apps): add_test_variable + with variable_type='device' per device, then add_command with + {"handsetId": {"data_source": "VARIABLE", "value": ""}}. Give each variable its device with + its value, or leave it empty and name it in execute_test's devices_under_test, which keeps the + test reusable across devices. +- Never bind handsetId to a CONSTANT device id. Perfecto accepts it when saving and then fails the run with + 'No handset is available': a device is reserved for the device parameters the test declares, never for a + literal in a step. The only binding that reserves a device is VARIABLE on a device variable. - A step's failure semantics also come from the command definition: Action steps abort the test (ABORT) while Validation steps are only reported (IGNORE). No need to set it, add_command applies what the command declares. - step_path is a dot-separated positional path without spaces (0-based indices; b0=Then branch, b1=Else). Example: root step 3 is "3"; first step inside Then of condition at 5 is "5.b0.0". Perfecto does not persist paths; they change when steps are inserted, moved, or deleted. Always call view_test_structure before the next structure edit; do not reuse step_path from a previous mutation response. @@ -1031,14 +1255,16 @@ def register(mcp, runtime: AppRuntime): - If in any result has_next_page is true, ask the user if they want to see the next page or access all pages before making a subsequent call. - Before executing a test, follow this validation workflow: 1. list_tests (get and validate test_id). - 2. Get device configuration based on device_type: + 2. list_test_variables (the device parameters this test runs on are the ones typed 'device', and their + value is the device each already holds; only the ones being chosen for this run need an entry). + 3. Get the device configuration for each entry, based on its device_type: - 'real': list_real_devices (get device_id). - 'virtual': list_virtual_devices (get platform_name, manufacturer, model, platform_version). - 'desktop': list_desktop_devices (get platform_name, platform_version, browser_name, browser_version, resolution, location). - 3. On real device use read_real_device_info (verify device is available and not in use). - 4. execute_test (execute the test). - 5. list_report_executions with report name equal to test name and list_live_executions when the device it's in use (monitor execution progress). -- Always check before running a test_id if the device_type and device_under_test exist and is available (when it's a real device), not use device in use or malfunctioning. + 4. On real device use read_real_device_info (verify device is available and not in use). + 5. execute_test (execute the test) with devices_under_test. + 6. list_report_executions with report name equal to test name and list_live_executions when the device it's in use (monitor execution progress). +- Always check before running a test_id that every device in devices_under_test exists and is available (when it's a real device), not use device in use or malfunctioning. - Always monitor a real device's operation while it's in use by checking the information with read_real_device_info. - Always stop the execution by stopping the live execution (make sure it's the correct execution, such as the execution name or user ID). """ @@ -1058,9 +1284,19 @@ async def _dispatch(): case "list_filter_values": return await ai_scriptless_manager.list_filter_values(args.get("filter_names", [])) case "execute_test": - return await ai_scriptless_manager.execute_test(args.get("test_id", ""), - args.get("device_type", ""), - args.get("device_under_test", {})) + # device_type/device_under_test/devices were one concept split in three; + # naming them now is a mistake worth a straight answer, not a silent run. + retired = [key for key in ("device_type", "device_under_test", "devices") + if key in args] + if retired: + return BaseResult( + error=f"{', '.join(retired)} no longer exist on execute_test. " + f"{AiScriptlessManager.DEVICES_SHAPE_HINT}" + ) + return await ai_scriptless_manager.execute_test( + args.get("test_id", ""), + args.get("devices_under_test", None), + ) case "view_test_structure": return await ai_scriptless_manager.view_test_structure(args.get("test_id", "")) case "view_test_step":