chore(eval): containerize built-in evaluations

[TomCC7]

Part of the linear evaluation stack: #3439 → #3581 → #3582 → #3583.

Summary

• make dimos eval run container-only for every built-in Evaluation
• replace the LIBERO-specific wrapper with one shared, digest-pinned evaluation image and launcher used by LIBERO-PRO and VLNCE-R2R
• keep executable Evaluations explicit and built-in; remove the unused external entry-point plugin seam
• require task manifests to be relative to their run specification
• run the existing LIBERO autoresearch command through the same container automatically
• execute by immutable Podman image ID with rootless sibling containers, run-scoped NVIDIA CDI, read-only inputs, allowlisted environment names, and atomically published staged outputs

Verification

• 56 focused tests pass across the shared framework, CLI, LIBERO-PRO, VLNCE-R2R, and autoresearch bootstrap
• Ruff check and format check pass on all changed Python files
• OpenSpec strict validation passes
• generic outer-container preflight passes: candidate import, Pi build, rootless Podman connection, and RTX 3090 visibility
• real checked-in evaluation cases remain pending until the previously exposed API credential is rotated

Depends on #3582.

[codecov]

❌ 3 Tests Failed:

Tests completed	Failed	Passed	Skipped
3910	3	3907	175

View the top 2 failed test(s) by shortest run time

dimos.codebase_checks.test_no_dunder_new::test_no_dunder_new

Stack Traces | 1.93s run time

def test_no_dunder_new() -> None:
        """Fail if any test file calls `__new__` to bypass `__init__`."""
        dimos_dir = DIMOS_PROJECT_ROOT / "dimos"
        hits = find_dunder_new_calls()
        if hits:
            listing = "\n".join(
                f"  - {p.relative_to(dimos_dir)}:{lineno}: {line.strip()}" for p, lineno, line in hits
            )
>           raise AssertionError(
                f"Found __new__ call(s) in test files:\n{listing}\n\n"
                "Tests must construct objects with the real constructor: __init__ is "
                "code under test too, and an object assembled by hand silently rots "
                "when the constructor changes. If __init__ does heavy work, mock the "
                "collaborators it needs instead of skipping it. Only if that is truly "
                "impossible, add the call to the WHITELIST in "
                "dimos/codebase_checks/test_no_dunder_new.py."
            )
E           AssertionError: Found __new__ call(s) in test files:
E             - benchmark/libero_pro/test_server_runtime.py:114: runtime = object.__new__(SERVER.Runtime)
E             - benchmark/libero_pro/test_server_runtime.py:126: runtime = object.__new__(SERVER.Runtime)
E           
E           Tests must construct objects with the real constructor: __init__ is code under test too, and an object assembled by hand silently rots when the constructor changes. If __init__ does heavy work, mock the collaborators it needs instead of skipping it. Only if that is truly impossible, add the call to the WHITELIST in dimos/codebase_checks/test_no_dunder_new.py.

dimos_dir  = PosixPath('.../dimos/dimos/dimos')
hits       = [(PosixPath('....../benchmark/libero_pro/test_server_runtime.py'), 114, '    runtime = ob....../benchmark/libero_pro/test_server_runtime.py'), 126, '    runtime = object.__new__(SERVER.Runtime)')]
listing    = '  - benchmark/libero_pro/test_server_runtime.py:114: runtime = object.__new__(SERVER.Runtime)\n  - benchmark/libero_pro/test_server_runtime.py:126: runtime = object.__new__(SERVER.Runtime)'

dimos/codebase_checks/test_no_dunder_new.py:67: AssertionError

dimos.robot.test_all_blueprints_generation::test_all_blueprints_is_current

Stack Traces | 5.31s run time

def test_all_blueprints_is_current() -> None:
        root = DIMOS_PROJECT_ROOT / "dimos"
        all_blueprints, all_modules = _scan_for_blueprints(root)
    
        common = set(all_blueprints.keys()) & set(all_modules.keys())
        assert not common, (
            f"Names must be unique across blueprints and modules, "
            f"but these appear in both: {sorted(common)}"
        )
    
        generated_content = _generate_all_blueprints_content(all_blueprints, all_modules)
    
        file_path = root / "robot" / "all_blueprints.py"
    
        if "CI" in os.environ:
            if not file_path.exists():
                pytest.fail(f"all_blueprints.py does not exist at {file_path}")
    
            current_content = file_path.read_text()
            if current_content != generated_content:
                diff = difflib.unified_diff(
                    current_content.splitlines(keepends=True),
                    generated_content.splitlines(keepends=True),
                    fromfile="all_blueprints.py (current)",
                    tofile="all_blueprints.py (generated)",
                )
                diff_str = "".join(diff)
>               pytest.fail(
                    f"all_blueprints.py is out of date. Run "
                    f"`pytest dimos/robot/test_all_blueprints_generation.py` locally to update.\n\n"
                    f"Diff:\n{diff_str}"
                )
E               Failed: all_blueprints.py is out of date. Run `pytest dimos/robot/test_all_blueprints_generation.py` locally to update.
E               
E               Diff:
E               --- all_blueprints.py (current)
E               +++ all_blueprints.py (generated)
E               @@ -219,6 +219,7 @@
E                    "goal-relay": "dimos.navigation.nav_3d.mls_planner.goal_relay.GoalRelay",
E                    "google-maps-skill-container": "dimos.agents.skills.google_maps_skill_container.GoogleMapsSkillContainer",
E                    "gps-nav-skill-container": "dimos.agents.skills.gps_nav_skill.GpsNavSkillContainer",
E               +    "grasp-execution-module": "dimos.manipulation.grasp_execution.GraspExecutionModule",
E                    "grasp-gen-x-module": "dimos.manipulation.grasping.grasp_gen_x.GraspGenXModule",
E                    "grasping-module": "dimos.manipulation.grasping.grasping.GraspingModule",
E                    "grounded-segmentation-module": "dimos.perception.grounded_segmentation.GroundedSegmentationModule",

all_blueprints = {'a1z-planner-coordinator': 'dimos.robot.manipulators.a1z.blueprints.basic:a1z_planner_coordinator', 'alfred-nav': 'di...rs.a1z.blueprints.basic:coordinator_a1z', 'coordinator-basic': 'dimos.control.blueprints.basic:coordinator_basic', ...}
all_modules = {'alfred-high-level': 'dimos.robot.diy.alfred.effector_high_level.AlfredHighLevel', 'arm-command-module': 'dimos.teleo...oordinator', 'arm-pose-twist-coordinator': 'dimos.robot.manipulators.common.coordinators.ArmPoseTwistCoordinator', ...}
common     = set()
current_content = '# Copyright 2025-2026 Dimensional Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the "License");\n# you m...sted.blueprints.cloudflare.WristCamera",\n    "zed-camera": "dimos.hardware.sensors.camera.zed.camera.ZEDCamera",\n}\n'
diff       = <generator object unified_diff at 0xff2803392740>
diff_str   = '--- all_blueprints.py (current)\n+++ all_blueprints.py (generated)\n@@ -219,6 +219,7 @@\n     "goal-relay": "dimos.na...gModule",\n     "grounded-segmentation-module": "dimos.perception.grounded_segmentation.GroundedSegmentationModule",\n'
file_path  = PosixPath('.../dimos/robot/all_blueprints.py')
generated_content = '# Copyright 2025-2026 Dimensional Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the "License");\n# you m...sted.blueprints.cloudflare.WristCamera",\n    "zed-camera": "dimos.hardware.sensors.camera.zed.camera.ZEDCamera",\n}\n'
root       = PosixPath('.../dimos/dimos/dimos')

dimos/robot/test_all_blueprints_generation.py:76: Failed

View the full list of 1 ❄️ flaky test(s)

dimos.benchmark.evaluation.test_policy_runtime::test_policy_result_larger_than_pipe_buffer_does_not_deadlock

Flake rate in main: 100.00% (Passed 0 times, Failed 1 times)

Stack Traces | 2.01s run time

tmp_path = PosixPath('.../pytest-0/popen-gw1/test_policy_result_larger_than0')

    def test_policy_result_larger_than_pipe_buffer_does_not_deadlock(tmp_path: Path) -> None:
        context = multiprocessing.get_context("spawn")
        messages, worker_messages = context.Pipe(duplex=False)
        start_event = context.Event()
        result_sending = context.Event()
        output_path = tmp_path / "policy-output.log"
        process = context.Process(
            target=_send_large_policy_error,
            args=(worker_messages, start_event, result_sending, str(output_path)),
            daemon=True,
        )
        process.start()
        worker_messages.close()
        execution = _PolicyExecutionProcess(process, messages, start_event, output_path)
        execution.start()
>       assert result_sending.wait(timeout=2)
E       assert False
E        +  where False = wait(timeout=2)
E        +    where wait = <Event at 0xff2803241070 unset>.wait

context    = <multiprocessing.context.SpawnContext object at 0xff2900dd8e00>
execution  = <dimos.benchmark.evaluation.runtime._PolicyExecutionProcess object at 0xff2803e3d4c0>
messages   = <multiprocessing.connection.Connection object at 0xff28032416d0>
output_path = PosixPath('.../pytest-0/popen-gw1/test_policy_result_larger_than0/policy-output.log')
process    = <SpawnProcess name='SpawnProcess-5' pid=14860 parent=3156 started daemon>
result_sending = <Event at 0xff2803241070 unset>
start_event = <Event at 0xff280263af90 set>
tmp_path   = PosixPath('.../pytest-0/popen-gw1/test_policy_result_larger_than0')
worker_messages = <multiprocessing.connection.Connection object at 0xff2803240e60>

.../benchmark/evaluation/test_policy_runtime.py:297: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}