From 9399a7efb6c81015026d4e462d6ffa767d07dfdc Mon Sep 17 00:00:00 2001 From: Aditya kumar singh <143548997+Adityakk9031@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:44:02 +0530 Subject: [PATCH 1/3] setup: offer the agent's on-demand camera mode, defaulting to on_demand (#179) --- docs/guide/cli.md | 3 +- docs/guide/quickstart.md | 8 +-- plugins/inspect-robots-agent/README.md | 7 +- src/inspect_robots/_setup.py | 62 ++++++++++++++++- tests/test_setup.py | 93 ++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 10 deletions(-) diff --git a/docs/guide/cli.md b/docs/guide/cli.md index ad8492015..26a076b36 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -115,7 +115,8 @@ failure with "no operator judgement recorded". The interactive first-run wizard: it prompts for each `[defaults]` key with a suggested value (Enter accepts, typing overrides), warns when a chosen -policy or embodiment is not registered in the current environment, and then +policy or embodiment is not registered in the current environment, offers +the `agent` policy's on-demand camera mode (`images = on_demand`), and then helps assign camera devices by listing `/dev/v4l/by-id`. If you do not know which physical camera a device path belongs to, answer `u` and unplug that camera when asked: the wizard rescans and identifies it from the entry that diff --git a/docs/guide/quickstart.md b/docs/guide/quickstart.md index 12b8ebdda..3cd81cbf6 100644 --- a/docs/guide/quickstart.md +++ b/docs/guide/quickstart.md @@ -92,10 +92,10 @@ uv pip install inspect-robots-yam # provides the molmoact2 policy + yam_arms r inspect-robots setup ``` -The wizard picks your defaults and finds your cameras, then writes -`~/.config/inspect-robots/config.ini`. On a different rig, install its plugin -instead and type its component names at the prompts; to write the config file -by hand, see [the CLI guide](cli.md). +The wizard picks your defaults (suggesting on-demand camera mode for `agent`), +finds your cameras, then writes `~/.config/inspect-robots/config.ini`. On a +different rig, install its plugin instead and type its component names at the +prompts; to write the config file by hand, see [the CLI guide](cli.md). The `molmoact2` policy is only a client: nothing moves until the MolmoAct2 server is listening, and the server does not start itself or survive a reboot diff --git a/plugins/inspect-robots-agent/README.md b/plugins/inspect-robots-agent/README.md index 3a0097956..65d8d0202 100644 --- a/plugins/inspect-robots-agent/README.md +++ b/plugins/inspect-robots-agent/README.md @@ -74,8 +74,9 @@ user reads these notes live and in the saved transcript to follow what the agent sees and decides. Camera images are attached to every observation by default -(`-P images=always`). Set `-P images=on_demand` to send state without image -payloads and give the model a `take_pic` tool instead: +(`-P images=always`), though `inspect-robots setup` suggests `on_demand`. Set +`-P images=on_demand` to send state without image payloads and give the model a +`take_pic` tool instead: ```bash inspect-robots "pick up the cube" --policy agent \ @@ -218,7 +219,7 @@ be distinguishable, encode it in a named factory's qualname, for example Configuration knobs (all `-P key=value`): `model`, `base_url`, `api_key_env`, `wire`, `speed`, `max_output_tokens`, `max_llm_calls` (default `100`), `temperature`, `effort`, `max_speed_frac`, `transcript_echo`, `images` -(default `always`; use `on_demand` for model-requested frames), +(default `always`; use `on_demand` for model-requested frames; `inspect-robots setup` suggests `on_demand`), `image_horizon`, `depth` (default `render`; use `off` to omit depth renders), and `prior_learnings`. `speed` and `max_output_tokens` apply to `-P wire=anthropic` only, and passing diff --git a/src/inspect_robots/_setup.py b/src/inspect_robots/_setup.py index d7fb9adc2..05ed5ecf5 100644 --- a/src/inspect_robots/_setup.py +++ b/src/inspect_robots/_setup.py @@ -109,6 +109,10 @@ def _valid_bool(value: str) -> bool: return isinstance(_parse_value(value), bool) +def _valid_images_mode(value: str) -> bool: + return value in ("on_demand", "always") + + def _ask( prompt: str, default: str, @@ -895,8 +899,12 @@ def _render_config( embodiment_args: dict[str, str], carried: dict[str, dict[str, str]], managed_args: tuple[str, ...] = CAMERA_KEYS, + policy_args: dict[str, str] | None = None, + managed_policy_args: tuple[str, ...] = (), ) -> str: """Render a full commented config while carrying unmanaged raw values.""" + if policy_args is None: + policy_args = {} sections: list[str] = [] default_lines: list[str] = [] @@ -931,8 +939,20 @@ def _render_config( if embodiment_lines: sections.append("[embodiment.args]\n" + "\n".join(embodiment_lines)) + policy_lines: list[str] = [] + for key in managed_policy_args: + if key in policy_args: + value = policy_args[key].replace("\n", "\n\t") + policy_lines.append(f"{key} = {value}") + for key, value in carried.get("policy.args", {}).items(): + if key not in managed_policy_args: + value = value.replace("\n", "\n\t") + policy_lines.append(f"{key} = {value}") + if policy_lines: + sections.append("[policy.args]\n" + "\n".join(policy_lines)) + for section, values in carried.items(): - if section in ("defaults", "embodiment.args"): + if section in ("defaults", "embodiment.args", "policy.args"): continue if values: lines = [] @@ -1008,6 +1028,37 @@ def run_setup( input_fn=input_fn, out=out, ) + configured_policy = defaults["policy"] + policy_args: dict[str, str] = {} + managed_policy_args: tuple[str, ...] = () + if configured_policy == "agent": + managed_policy_args = ("images",) + existing_policy_args = carried.get("policy.args", {}) + default_images = "on_demand" + if "images" in existing_policy_args: + configured_images = existing_policy_args["images"] + if _valid_images_mode(configured_images): + default_images = configured_images + print( + _paint( + "agent camera mode: 'on_demand' lets the model call take_pic when " + "it needs a frame (cuts tokens, but model must remember to look); " + "'always' attaches frames to every step", + _DIM, + out, + ), + file=out, + ) + images_value = _ask( + "agent camera mode", + default_images, + _valid_images_mode, + "camera mode must be 'on_demand' or 'always'", + input_fn=input_fn, + out=out, + ) + policy_args["images"] = images_value + from inspect_robots.registry import registered embodiment_factories = registered("embodiment") @@ -1044,7 +1095,14 @@ def run_setup( print(_paint("setup aborted; nothing written", _YELLOW, out), file=out) return 1 - text = _render_config(defaults, embodiment_args, carried, managed_args) + text = _render_config( + defaults, + embodiment_args, + carried, + managed_args, + policy_args=policy_args, + managed_policy_args=managed_policy_args, + ) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(path.name + ".tmp") tmp.write_text(text, encoding="utf-8") diff --git a/tests/test_setup.py b/tests/test_setup.py index efae84e3e..06ebe2337 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2654,3 +2654,96 @@ def test_suggest_can_pinning_pinned_names_or_no_assigned_kernel_name_are_silent( _suggest_can_pinning(order_net, slots, {"left_channel": "can9"}, out=unassigned_out) assert pinned_out.getvalue() == "" assert unassigned_out.getvalue() == "" + + +def test_render_config_renders_policy_args() -> None: + defaults = { + "policy": "agent", + "embodiment": "yam_arms", + "scorer": "success_at_end", + "max_steps": "1200", + "rerun": "true", + "store_frames": "true", + } + carried = { + "policy.args": {"model": "anthropic/claude-fable-5"}, + "custom": {"key": "val"}, + } + rendered = _render_config( + defaults, + {}, + carried, + policy_args={"images": "on_demand"}, + managed_policy_args=("images",), + ) + assert "[policy.args]\nimages = on_demand\nmodel = anthropic/claude-fable-5" in rendered + assert "[custom]\nkey = val" in rendered + + +def test_run_setup_prompts_agent_images_mode(tmp_path: Path) -> None: + config_file = _config_path(tmp_path) + env = {"XDG_CONFIG_HOME": str(tmp_path)} + # Input sequence: + # 1. policy: agent + # 2. embodiment: yam_arms + # 3. scorer: Enter (success_at_end) + # 4. max steps: Enter (1200) + # 5. rerun: Enter (true) + # 6. store frames: Enter (true) + # 7. agent camera mode: Enter (default on_demand) + # 8. configure cameras: n + input_fn, _prompts = _scripted_input(["agent", "yam_arms", "", "", "", "", "", "n"]) + out = io.StringIO() + + exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True) + assert exit_code == 0 + text = config_file.read_text(encoding="utf-8") + assert "[policy.args]\nimages = on_demand" in text + assert "agent camera mode" in out.getvalue() + + +def test_run_setup_prompts_agent_images_mode_always(tmp_path: Path) -> None: + config_file = _config_path(tmp_path) + env = {"XDG_CONFIG_HOME": str(tmp_path)} + # Input sequence: + # 1. policy: agent + # 2. embodiment: yam_arms + # 3. scorer: Enter (success_at_end) + # 4. max steps: Enter (1200) + # 5. rerun: Enter (true) + # 6. store frames: Enter (true) + # 7. agent camera mode: always + # 8. configure cameras: n + input_fn, _prompts = _scripted_input(["agent", "yam_arms", "", "", "", "", "always", "n"]) + out = io.StringIO() + + exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True) + assert exit_code == 0 + text = config_file.read_text(encoding="utf-8") + assert "[policy.args]\nimages = always" in text + + +def test_run_setup_preserves_existing_agent_images_mode(tmp_path: Path) -> None: + config_file = _config_path(tmp_path) + config_file.parent.mkdir(parents=True) + config_file.write_text( + "[defaults]\npolicy = agent\nembodiment = yam_arms\n\n[policy.args]\nimages = always\n", + encoding="utf-8", + ) + env = {"XDG_CONFIG_HOME": str(tmp_path)} + # Input sequence: + # 1. policy: Enter (agent) + # 2. embodiment: Enter (yam_arms) + # 3. scorer: Enter + # 4. max steps: Enter + # 5. rerun: Enter + # 6. store frames: Enter + # 7. agent camera mode: Enter (preserves always) + # 8. configure cameras: n + input_fn, _prompts = _scripted_input(["", "", "", "", "", "", "", "n"]) + out = io.StringIO() + + exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True) + assert exit_code == 0 + text = config_file.read_text(encoding="utf-8") + assert "[policy.args]\nimages = always" in text From 298524a247541df6f425e94d01ded0eaf2069316 Mon Sep 17 00:00:00 2001 From: Aditya kumar singh <143548997+Adityakk9031@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:50:28 +0530 Subject: [PATCH 2/3] test: add branch coverage for agent policy setup prompts and policy.args rendering (#179) --- tests/test_setup.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_setup.py b/tests/test_setup.py index 06ebe2337..ae385be37 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2747,3 +2747,37 @@ def test_run_setup_preserves_existing_agent_images_mode(tmp_path: Path) -> None: assert exit_code == 0 text = config_file.read_text(encoding="utf-8") assert "[policy.args]\nimages = always" in text + + +def test_run_setup_ignores_invalid_existing_agent_images_mode(tmp_path: Path) -> None: + config_file = _config_path(tmp_path) + config_file.parent.mkdir(parents=True) + config_file.write_text( + "[defaults]\npolicy = agent\nembodiment = yam_arms\n\n[policy.args]\nimages = invalid\n", + encoding="utf-8", + ) + env = {"XDG_CONFIG_HOME": str(tmp_path)} + input_fn, _prompts = _scripted_input(["", "", "", "", "", "", "", "n"]) + out = io.StringIO() + + exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True) + assert exit_code == 0 + text = config_file.read_text(encoding="utf-8") + assert "[policy.args]\nimages = on_demand" in text + + +def test_render_config_managed_policy_args_missing_from_policy_args() -> None: + defaults = { + "policy": "agent", + "embodiment": "yam_arms", + } + carried = {"empty_section": {}} + rendered = _render_config( + defaults, + {}, + carried, + policy_args={}, + managed_policy_args=("images",), + ) + assert "[policy.args]" not in rendered + assert "[empty_section]" not in rendered From 23cffd17283eba601f07b90a9e0cf6449274e90e Mon Sep 17 00:00:00 2001 From: Aditya kumar singh <143548997+Adityakk9031@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:54:50 +0530 Subject: [PATCH 3/3] test: add type annotation for carried dictionary in test_setup.py (#179) --- tests/test_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_setup.py b/tests/test_setup.py index ae385be37..97d55719a 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2771,7 +2771,7 @@ def test_render_config_managed_policy_args_missing_from_policy_args() -> None: "policy": "agent", "embodiment": "yam_arms", } - carried = {"empty_section": {}} + carried: dict[str, dict[str, str]] = {"empty_section": {}} rendered = _render_config( defaults, {},