WIP: Add self-update tests

.github/actions/ci/action.yml

@@ -19,6 +19,7 @@ runs:
           ~/.dylint_drivers/
           ~/.rustup/toolchains/
           target/dylint/
+          target/necessist-audit-cache/
         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
     - uses: taiki-e/install-action@v2.85.13
@@ -45,6 +46,24 @@ runs:
       if: ${{ env.TESTS != 'ci' }}
       uses: ./.github/actions/install-testing-tools
 
+    - name: Install LLM tools
+      if: ${{ (runner.os == 'Linux' || runner.os == 'macOS') && contains(env.TESTS, 'necessist_audit') }}
+      run: |
+        curl -fsSL https://claude.ai/install.sh | bash
+        curl -fsSL https://chatgpt.com/codex/install.sh | sh
+        printenv OPENAI_API_KEY | codex login --with-api-key
+      shell: bash
+
+    # smoelius: Codex requires Bubblewrap.
+    - name: Install Bubblewrap on Ubuntu
+      if: ${{ runner.os == 'Linux' && contains(env.TESTS, 'necessist_audit') }}
+      run: |
+        sudo apt install bubblewrap
+        sudo apt install apparmor-profiles
+        sudo cp /usr/share/apparmor/extra-profiles/bwrap-userns-restrict /etc/apparmor.d || true
+        sudo systemctl reload apparmor
+      shell: bash
+
     - name: Install sqlite3 on Ubuntu
       if: ${{ runner.os == 'Linux' }}
       run: |
@@ -82,6 +101,11 @@ runs:
       run: |
         for TEST in $TESTS; do
           if [[ "$TEST" != 'other' ]]; then
+            if [[ "$TEST" = 'necessist_audit' && '${{ github.event_name }}' != 'schedule' && '${{ github.event_name }}' != 'workflow_dispatch' ]] &&
+              ! git diff --name-only ${{ github.event.pull_request.base.sha }} | grep 'necessist_audit' >/dev/null
+            then
+              exit 0
+            fi
             $CARGO_TEST -p necessist --test "$TEST"
           else
             $CARGO_TEST -p necessist --test general

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
       fail-fast: ${{ github.event_name == 'merge_group' }}
       matrix:
         environment: [ubuntu-latest, macos-latest, windows-latest]
-        test: [third_party_0, third_party_1, trycmd, other]
+        test: [necessist_audit, third_party_0, third_party_1, trycmd, other]
         include:
           - environment: ubuntu-latest
             test: ci
@@ -48,6 +48,8 @@ jobs:
       - uses: ./.github/actions/ci
         env:
           TESTS: ${{ matrix.test }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
   all-checks:
     needs: [test]

core/skills/necessist-audit/SKILL.md

@@ -16,6 +16,16 @@ Use passing removals as leads for finding bugs in the code or tests being audite
 
 Do not modify project source unless the user explicitly requests changes. Running Necessist and allowing it to create `necessist.db` is permitted.
 
+## Preflight
+
+Run `necessist --check-skill <path>`, where `<path>` is the file containing these instructions. If the path is unknown, try `~/.claude/skills/necessist-audit/SKILL.md` and `~/.codex/skills/necessist-audit/SKILL.md`.
+
+Never pass `--write`. Replacing this skill is the user's decision, not yours.
+
+If the user asks only to check whether the skill is up to date, relay the `necessist --check-skill` output verbatim and stop without locating or auditing Necessist results.
+
+During an audit, relay the output verbatim only when it reports that the skill is an old version or a newer version. Stay silent when the skill is current, when the path does not exist, or when the command fails. Proceed with the audit in every case.
+
 ## Scope
 
 Analyze only removals whose outcome is `passed`.

necessist/tests/necessist_audit.rs

@@ -0,0 +1,91 @@
+use assert_cmd::{cargo::cargo_bin, output::OutputError};
+use elaborate::std::{
+    env::{join_paths_wc, var_os_wc},
+    path::PathContext,
+};
+use std::{
+    env::split_paths,
+    io::{Write, stderr},
+    path::{Path, PathBuf},
+    process::Output,
+    time::Duration,
+};
+
+mod necessist_audit_support;
+use necessist_audit_support::{tool::Tool, update_check::TESTS};
+
+const TIMEOUT: Duration = Duration::from_mins(5);
+
+#[test]
+#[cfg_attr(dylint_lib = "general", allow(non_thread_safe_call_in_test))]
+fn necessist_audit_update_claude() {
+    let Some(tool_path) = command_on_path("claude") else {
+        #[allow(clippy::explicit_write)]
+        writeln!(
+            stderr(),
+            "Skipping `necessist_audit_update_claude` because `claude` is not on PATH"
+        )
+        .unwrap();
+        return;
+    };
+
+    let tool = Tool::claude(tool_path);
+    for test in &TESTS {
+        test.run(&tool);
+    }
+}
+
+#[test]
+#[cfg_attr(dylint_lib = "general", allow(non_thread_safe_call_in_test))]
+fn necessist_audit_update_codex() {
+    let Some(tool_path) = command_on_path("codex") else {
+        #[allow(clippy::explicit_write)]
+        writeln!(
+            stderr(),
+            "Skipping `necessist_audit_update_codex` because `codex` is not on PATH"
+        )
+        .unwrap();
+        return;
+    };
+
+    let tool = Tool::codex(tool_path);
+    for test in &TESTS {
+        test.run(&tool);
+    }
+}
+
+fn command_on_path(tool: &str) -> Option<PathBuf> {
+    let path_var = var_os_wc("PATH").ok()?;
+    split_paths(&path_var).find_map(|dir| {
+        let candidate = dir.join(tool);
+        candidate.is_file().then_some(candidate)
+    })
+}
+
+fn format_output(output: &Output) -> String {
+    OutputError::new(output.clone()).to_string()
+}
+
+#[cfg_attr(dylint_lib = "supplementary", allow(abs_home_path))]
+fn workspace_root() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent_wc()
+        .unwrap()
+        .to_owned()
+}
+
+fn skill_path() -> PathBuf {
+    workspace_root().join("core/skills/necessist-audit/SKILL.md")
+}
+
+fn path_with_necessist() -> std::ffi::OsString {
+    let necessist = cargo_bin("necessist");
+    let necessist_dir = necessist.parent_wc().unwrap();
+    let paths = std::iter::once(necessist_dir.to_owned()).chain(
+        var_os_wc("PATH")
+            .ok()
+            .map(|path| split_paths(&path).collect::<Vec<_>>())
+            .unwrap_or_default(),
+    );
+    join_paths_wc(paths).unwrap()
+}

necessist/tests/necessist_audit_support/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod tool;
+pub(crate) mod update_check;

necessist/tests/necessist_audit_support/tool.rs

@@ -0,0 +1,104 @@
+use super::super::{TIMEOUT, format_output, path_with_necessist};
+use elaborate::std::process::{ChildContext, CommandContext};
+use std::{
+    env::vars_os,
+    path::{Path, PathBuf},
+    process::{Command, Output, Stdio},
+    thread::sleep,
+    time::{Duration, Instant},
+};
+
+const POLL_INTERVAL: Duration = Duration::from_millis(100);
+
+pub(crate) struct Tool {
+    pub(crate) name: &'static str,
+    tool_path: PathBuf,
+    args_fn: fn(&Path, String) -> Vec<String>,
+    stdin_null: bool,
+}
+
+impl Tool {
+    pub(crate) fn claude(tool_path: PathBuf) -> Self {
+        Self {
+            name: "claude",
+            tool_path,
+            args_fn: claude_args,
+            stdin_null: false,
+        }
+    }
+
+    pub(crate) fn codex(tool_path: PathBuf) -> Self {
+        Self {
+            name: "codex",
+            tool_path,
+            args_fn: codex_args,
+            stdin_null: true,
+        }
+    }
+
+    pub(crate) fn run(&self, skill_dir: &Path, prompt: String) -> Result<Output, String> {
+        run_tool_command(self, skill_dir, prompt)
+    }
+}
+
+fn claude_args(skill_dir: &Path, prompt: String) -> Vec<String> {
+    vec![
+        format!("--add-dir={}", skill_dir.display()),
+        "--allowedTools=Bash,Read".to_owned(),
+        "--permission-mode=acceptEdits".to_owned(),
+        "--print".to_owned(),
+        prompt,
+    ]
+}
+
+fn codex_args(skill_dir: &Path, prompt: String) -> Vec<String> {
+    vec![
+        "exec".to_owned(),
+        format!("--add-dir={}", skill_dir.to_string_lossy()),
+        "--config=sandbox_workspace_write.network_access=false".to_owned(),
+        "--config=shell_environment_policy.inherit=\"all\"".to_owned(),
+        "--sandbox=workspace-write".to_owned(),
+        prompt,
+    ]
+}
+
+fn run_tool_command(tool: &Tool, skill_dir: &Path, prompt: String) -> Result<Output, String> {
+    let mut command = Command::new(&tool.tool_path);
+    command
+        .args((tool.args_fn)(skill_dir, prompt))
+        .env_clear()
+        .envs(vars_os())
+        .env("PATH", path_with_necessist());
+
+    if tool.stdin_null {
+        command.stdin(Stdio::null());
+    }
+
+    let mut child = command
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn_wc()
+        .unwrap();
+    let start = Instant::now();
+
+    loop {
+        if child.try_wait_wc().unwrap().is_some() {
+            let output = child.wait_with_output_wc().unwrap();
+            if output.status.success() {
+                return Ok(output);
+            }
+            return Err(format!("skill command failed\n{}", format_output(&output)));
+        }
+
+        if start.elapsed() >= TIMEOUT {
+            drop(child.kill_wc());
+            let output = child.wait_with_output_wc().unwrap();
+            return Err(format!(
+                "skill command timed out after {TIMEOUT:?}\n{}",
+                format_output(&output)
+            ));
+        }
+
+        sleep(POLL_INTERVAL);
+    }
+}