setup: offer the agent's on-demand camera mode, defaulting to on_demand (#179)

docs/guide/cli.md

@@ -115,7 +115,8 @@ failure with "no operator judgement recorded".
 
 The interactive first-run wizard: it prompts for each `[defaults]` key with
 a suggested value (Enter accepts, typing overrides), warns when a chosen
-policy or embodiment is not registered in the current environment, and then
+policy or embodiment is not registered in the current environment, offers
+the `agent` policy's on-demand camera mode (`images = on_demand`), and then
 helps assign camera devices by listing `/dev/v4l/by-id`. If you do not know
 which physical camera a device path belongs to, answer `u` and unplug that
 camera when asked: the wizard rescans and identifies it from the entry that

docs/guide/quickstart.md

@@ -92,10 +92,10 @@ uv pip install inspect-robots-yam   # provides the molmoact2 policy + yam_arms r
 inspect-robots setup
 ```
 
-The wizard picks your defaults and finds your cameras, then writes
-`~/.config/inspect-robots/config.ini`. On a different rig, install its plugin
-instead and type its component names at the prompts; to write the config file
-by hand, see [the CLI guide](cli.md).
+The wizard picks your defaults (suggesting on-demand camera mode for `agent`),
+finds your cameras, then writes `~/.config/inspect-robots/config.ini`. On a
+different rig, install its plugin instead and type its component names at the
+prompts; to write the config file by hand, see [the CLI guide](cli.md).
 
 The `molmoact2` policy is only a client: nothing moves until the MolmoAct2
 server is listening, and the server does not start itself or survive a reboot

plugins/inspect-robots-agent/README.md

@@ -74,8 +74,9 @@ user reads these notes live and in the saved transcript to follow what the
 agent sees and decides.
 
 Camera images are attached to every observation by default
-(`-P images=always`). Set `-P images=on_demand` to send state without image
-payloads and give the model a `take_pic` tool instead:
+(`-P images=always`), though `inspect-robots setup` suggests `on_demand`. Set
+`-P images=on_demand` to send state without image payloads and give the model a
+`take_pic` tool instead:
 
 ```bash
 inspect-robots "pick up the cube" --policy agent \
@@ -218,7 +219,7 @@ be distinguishable, encode it in a named factory's qualname, for example
 Configuration knobs (all `-P key=value`): `model`, `base_url`, `api_key_env`,
 `wire`, `speed`, `max_output_tokens`, `max_llm_calls` (default `100`),
 `temperature`, `effort`, `max_speed_frac`, `transcript_echo`, `images`
-(default `always`; use `on_demand` for model-requested frames),
+(default `always`; use `on_demand` for model-requested frames; `inspect-robots setup` suggests `on_demand`),
 `image_horizon`, `depth` (default `render`; use `off` to omit depth
 renders), and `prior_learnings`.
 `speed` and `max_output_tokens` apply to `-P wire=anthropic` only, and passing

src/inspect_robots/_setup.py

@@ -109,6 +109,10 @@ def _valid_bool(value: str) -> bool:
     return isinstance(_parse_value(value), bool)
 
 
+def _valid_images_mode(value: str) -> bool:
+    return value in ("on_demand", "always")
+
+
 def _ask(
     prompt: str,
     default: str,
@@ -895,8 +899,12 @@ def _render_config(
     embodiment_args: dict[str, str],
     carried: dict[str, dict[str, str]],
     managed_args: tuple[str, ...] = CAMERA_KEYS,
+    policy_args: dict[str, str] | None = None,
+    managed_policy_args: tuple[str, ...] = (),
 ) -> str:
     """Render a full commented config while carrying unmanaged raw values."""
+    if policy_args is None:
+        policy_args = {}
     sections: list[str] = []
 
     default_lines: list[str] = []
@@ -931,8 +939,20 @@ def _render_config(
     if embodiment_lines:
         sections.append("[embodiment.args]\n" + "\n".join(embodiment_lines))
 
+    policy_lines: list[str] = []
+    for key in managed_policy_args:
+        if key in policy_args:
+            value = policy_args[key].replace("\n", "\n\t")
+            policy_lines.append(f"{key} = {value}")
+    for key, value in carried.get("policy.args", {}).items():
+        if key not in managed_policy_args:
+            value = value.replace("\n", "\n\t")
+            policy_lines.append(f"{key} = {value}")
+    if policy_lines:
+        sections.append("[policy.args]\n" + "\n".join(policy_lines))
+
     for section, values in carried.items():
-        if section in ("defaults", "embodiment.args"):
+        if section in ("defaults", "embodiment.args", "policy.args"):
             continue
         if values:
             lines = []
@@ -1008,6 +1028,37 @@ def run_setup(
             input_fn=input_fn,
             out=out,
         )
+        configured_policy = defaults["policy"]
+        policy_args: dict[str, str] = {}
+        managed_policy_args: tuple[str, ...] = ()
+        if configured_policy == "agent":
+            managed_policy_args = ("images",)
+            existing_policy_args = carried.get("policy.args", {})
+            default_images = "on_demand"
+            if "images" in existing_policy_args:
+                configured_images = existing_policy_args["images"]
+                if _valid_images_mode(configured_images):
+                    default_images = configured_images
+            print(
+                _paint(
+                    "agent camera mode: 'on_demand' lets the model call take_pic when "
+                    "it needs a frame (cuts tokens, but model must remember to look); "
+                    "'always' attaches frames to every step",
+                    _DIM,
+                    out,
+                ),
+                file=out,
+            )
+            images_value = _ask(
+                "agent camera mode",
+                default_images,
+                _valid_images_mode,
+                "camera mode must be 'on_demand' or 'always'",
+                input_fn=input_fn,
+                out=out,
+            )
+            policy_args["images"] = images_value
+
         from inspect_robots.registry import registered
 
         embodiment_factories = registered("embodiment")
@@ -1044,7 +1095,14 @@ def run_setup(
         print(_paint("setup aborted; nothing written", _YELLOW, out), file=out)
         return 1
 
-    text = _render_config(defaults, embodiment_args, carried, managed_args)
+    text = _render_config(
+        defaults,
+        embodiment_args,
+        carried,
+        managed_args,
+        policy_args=policy_args,
+        managed_policy_args=managed_policy_args,
+    )
     path.parent.mkdir(parents=True, exist_ok=True)
     tmp = path.with_name(path.name + ".tmp")
     tmp.write_text(text, encoding="utf-8")

tests/test_setup.py

@@ -2654,3 +2654,130 @@ def test_suggest_can_pinning_pinned_names_or_no_assigned_kernel_name_are_silent(
     _suggest_can_pinning(order_net, slots, {"left_channel": "can9"}, out=unassigned_out)
     assert pinned_out.getvalue() == ""
     assert unassigned_out.getvalue() == ""
+
+
+def test_render_config_renders_policy_args() -> None:
+    defaults = {
+        "policy": "agent",
+        "embodiment": "yam_arms",
+        "scorer": "success_at_end",
+        "max_steps": "1200",
+        "rerun": "true",
+        "store_frames": "true",
+    }
+    carried = {
+        "policy.args": {"model": "anthropic/claude-fable-5"},
+        "custom": {"key": "val"},
+    }
+    rendered = _render_config(
+        defaults,
+        {},
+        carried,
+        policy_args={"images": "on_demand"},
+        managed_policy_args=("images",),
+    )
+    assert "[policy.args]\nimages = on_demand\nmodel = anthropic/claude-fable-5" in rendered
+    assert "[custom]\nkey = val" in rendered
+
+
+def test_run_setup_prompts_agent_images_mode(tmp_path: Path) -> None:
+    config_file = _config_path(tmp_path)
+    env = {"XDG_CONFIG_HOME": str(tmp_path)}
+    # Input sequence:
+    # 1. policy: agent
+    # 2. embodiment: yam_arms
+    # 3. scorer: Enter (success_at_end)
+    # 4. max steps: Enter (1200)
+    # 5. rerun: Enter (true)
+    # 6. store frames: Enter (true)
+    # 7. agent camera mode: Enter (default on_demand)
+    # 8. configure cameras: n
+    input_fn, _prompts = _scripted_input(["agent", "yam_arms", "", "", "", "", "", "n"])
+    out = io.StringIO()
+
+    exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True)
+    assert exit_code == 0
+    text = config_file.read_text(encoding="utf-8")
+    assert "[policy.args]\nimages = on_demand" in text
+    assert "agent camera mode" in out.getvalue()
+
+
+def test_run_setup_prompts_agent_images_mode_always(tmp_path: Path) -> None:
+    config_file = _config_path(tmp_path)
+    env = {"XDG_CONFIG_HOME": str(tmp_path)}
+    # Input sequence:
+    # 1. policy: agent
+    # 2. embodiment: yam_arms
+    # 3. scorer: Enter (success_at_end)
+    # 4. max steps: Enter (1200)
+    # 5. rerun: Enter (true)
+    # 6. store frames: Enter (true)
+    # 7. agent camera mode: always
+    # 8. configure cameras: n
+    input_fn, _prompts = _scripted_input(["agent", "yam_arms", "", "", "", "", "always", "n"])
+    out = io.StringIO()
+
+    exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True)
+    assert exit_code == 0
+    text = config_file.read_text(encoding="utf-8")
+    assert "[policy.args]\nimages = always" in text
+
+
+def test_run_setup_preserves_existing_agent_images_mode(tmp_path: Path) -> None:
+    config_file = _config_path(tmp_path)
+    config_file.parent.mkdir(parents=True)
+    config_file.write_text(
+        "[defaults]\npolicy = agent\nembodiment = yam_arms\n\n[policy.args]\nimages = always\n",
+        encoding="utf-8",
+    )
+    env = {"XDG_CONFIG_HOME": str(tmp_path)}
+    # Input sequence:
+    # 1. policy: Enter (agent)
+    # 2. embodiment: Enter (yam_arms)
+    # 3. scorer: Enter
+    # 4. max steps: Enter
+    # 5. rerun: Enter
+    # 6. store frames: Enter
+    # 7. agent camera mode: Enter (preserves always)
+    # 8. configure cameras: n
+    input_fn, _prompts = _scripted_input(["", "", "", "", "", "", "", "n"])
+    out = io.StringIO()
+
+    exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True)
+    assert exit_code == 0
+    text = config_file.read_text(encoding="utf-8")
+    assert "[policy.args]\nimages = always" in text
+
+
+def test_run_setup_ignores_invalid_existing_agent_images_mode(tmp_path: Path) -> None:
+    config_file = _config_path(tmp_path)
+    config_file.parent.mkdir(parents=True)
+    config_file.write_text(
+        "[defaults]\npolicy = agent\nembodiment = yam_arms\n\n[policy.args]\nimages = invalid\n",
+        encoding="utf-8",
+    )
+    env = {"XDG_CONFIG_HOME": str(tmp_path)}
+    input_fn, _prompts = _scripted_input(["", "", "", "", "", "", "", "n"])
+    out = io.StringIO()
+
+    exit_code = run_setup(env, input_fn=input_fn, out=out, interactive=True)
+    assert exit_code == 0
+    text = config_file.read_text(encoding="utf-8")
+    assert "[policy.args]\nimages = on_demand" in text
+
+
+def test_render_config_managed_policy_args_missing_from_policy_args() -> None:
+    defaults = {
+        "policy": "agent",
+        "embodiment": "yam_arms",
+    }
+    carried: dict[str, dict[str, str]] = {"empty_section": {}}
+    rendered = _render_config(
+        defaults,
+        {},
+        carried,
+        policy_args={},
+        managed_policy_args=("images",),
+    )
+    assert "[policy.args]" not in rendered
+    assert "[empty_section]" not in rendered