Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions .agents/skills/nemoclaw-maintainer-e2e/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,18 @@ case "$E2E_JOBS" in
esac
REVIEW_REASON='Reviewed the PR head commit for credentialed E2E.'
CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')"
INFERENCE_MODE=mock
ALLOW_JETSON_RUNNER_QUEUE=false
ALLOW_DGX_SPARK_RUNNER_QUEUE=false
gh workflow run .github/workflows/e2e.yaml \
--repo NVIDIA/NemoClaw \
--ref main \
-f targets= \
-f "jobs=${E2E_JOBS}" \
-f inference_mode=mock \
-f "inference_mode=${INFERENCE_MODE}" \
-f include_staging_brev_launchable=false \
-f allow_jetson_runner_queue=false \
-f allow_dgx_spark_runner_queue=false \
-f "allow_jetson_runner_queue=${ALLOW_JETSON_RUNNER_QUEUE}" \
-f "allow_dgx_spark_runner_queue=${ALLOW_DGX_SPARK_RUNNER_QUEUE}" \
-f "pr_number=${PR_NUMBER}" \
-f "checkout_sha=${HEAD_SHA}" \
-f "checkout_repository=${HEAD_REPOSITORY}" \
Expand All @@ -99,7 +102,8 @@ gh workflow run .github/workflows/e2e.yaml \
-f "correlation_id=${CORRELATION_ID}"
```

The trusted pre-checkout step requires current `maintain` or `admin` permission and validates the open PR, repository, head SHA, base SHA, workflow SHA, review reason, and selected mode.
The trusted pre-checkout step requires current `maintain` or `admin` permission.
It validates the actor, open PR, repository, head SHA, base SHA, workflow SHA, review reason, and allowed jobs, targets, and Launchable combination.
A second validation after checkout rejects a changed PR identity before preparation.

Find and verify the correlated run with bounded GitHub reads:
Expand Down
6 changes: 1 addition & 5 deletions test/e2e-risk-signal-reporter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ vi.mock("node:child_process", () => ({
}));

const EXPECTED_SHA = "a".repeat(40);
const PLAN_HASH = "b".repeat(64);
const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc";

function moduleWithStates(states: Array<"passed" | "failed" | "skipped" | "pending">): TestModule {
Expand Down Expand Up @@ -79,7 +78,6 @@ function environment(artifactDir: string): RiskSignalEnvironment {
shardId: "default",
expectedSha: EXPECTED_SHA,
testedSha: EXPECTED_SHA,
planHash: PLAN_HASH,
correlationId: CORRELATION_ID,
};
}
Expand All @@ -99,13 +97,12 @@ describe("E2E risk signal reporter", () => {
);
});

it("attests the checked-out HEAD instead of echoing only the expected SHA", () => {
it("attests the checked-out HEAD without plan metadata", () => {
const env = {
E2E_ARTIFACT_DIR: "/tmp/e2e-risk-signal-test",
E2E_TARGET_ID: "onboard-resume",
GITHUB_WORKSPACE: "/workspace",
NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA,
NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH,
NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID,
NEMOCLAW_E2E_SHARD: "default",
};
Expand Down Expand Up @@ -161,7 +158,6 @@ describe("E2E risk signal reporter", () => {
vi.stubEnv("E2E_ARTIFACT_DIR", dir);
vi.stubEnv("E2E_TARGET_ID", "network-policy");
vi.stubEnv("NEMOCLAW_E2E_EXPECTED_SHA", EXPECTED_SHA);
vi.stubEnv("NEMOCLAW_E2E_PLAN_HASH", PLAN_HASH);
vi.stubEnv("NEMOCLAW_E2E_CORRELATION_ID", CORRELATION_ID);
vi.stubEnv("NEMOCLAW_E2E_SHARD", "live-probes");

Expand Down
9 changes: 6 additions & 3 deletions test/e2e/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,8 @@ test/e2e/
A maintainer can also dispatch the trusted `main` workflow against the exact
head of an open internal or fork PR. The manual path validates the actor, PR
number, head repository, head SHA, base SHA, workflow SHA, review reason, and
selected mode before candidate checkout. For a PR revision run, leave `jobs` and
allowed jobs, targets, and Launchable combination before candidate checkout.
For a PR revision run, leave `jobs` and
`targets` empty. The run selects every default-selected free-standing workflow
E2E except `Exact staging Brev Launchable`. It also selects all shared credential-free tests and
these controller-selected registry targets:
Expand Down Expand Up @@ -331,8 +332,10 @@ test/e2e/
The Slack and GitHub scorecard timing comparison remains scoped to the
dedicated `cloud-onboard` artifact.
Manual PR runs attach `test/e2e/risk-signal-reporter.ts` to live Vitest
invocations and suppress PR reporting and scorecards. The workflow boundary
requires every selected job shard to upload its evidence artifact.
invocations and suppress PR reporting and scorecards. Each risk signal binds
its result counts to the expected and tested candidate SHA, correlation ID,
job ID, and shard ID. The workflow boundary requires every selected job shard
to upload its evidence artifact.
- `.github/workflows/platform-vitest-main.yaml` runs the full Vitest suite in
four independent shards on each of macOS and WSL, with `fail-fast` disabled.
Each macOS shard installs the pinned OpenShell formula and has a 30-minute
Expand Down
1 change: 0 additions & 1 deletion test/e2e/risk-signal-reporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ function mergeSignal(previous: E2eRiskSignal | null, current: E2eRiskSignal): E2
previous.shardId !== current.shardId ||
previous.expectedSha !== current.expectedSha ||
previous.testedSha !== current.testedSha ||
previous.planHash !== current.planHash ||
previous.correlationId !== current.correlationId
) {
throw new Error("risk signal metadata changed between Vitest invocations");
Expand Down
3 changes: 0 additions & 3 deletions test/e2e/support/mcp-bridge-runtime-compatibility-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { MCP_BRIDGE_RUNTIME_COMPATIBILITY_ARTIFACT } from "../../../tools/e2e/mc
import { RISK_SIGNAL_FILE } from "../../../tools/e2e/risk-signal.ts";

const COMPATIBILITY_TOOL = path.resolve("tools/e2e/mcp-bridge-runtime-compatibility.mts");
const PLAN_HASH = "b".repeat(64);
const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000";

function gatedEnvironment(): Record<string, string> {
Expand All @@ -26,7 +25,6 @@ function gatedEnvironment(): Record<string, string> {
GITHUB_WORKSPACE: process.cwd(),
NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID,
NEMOCLAW_E2E_EXPECTED_SHA: expectedSha,
NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH,
NEMOCLAW_E2E_SHARD: "openclaw",
};
}
Expand Down Expand Up @@ -153,7 +151,6 @@ describe.skipIf(process.platform === "win32")("MCP bridge compatibility CLI", ()
shardId: "openclaw",
expectedSha: gateEnv.NEMOCLAW_E2E_EXPECTED_SHA,
testedSha: gateEnv.NEMOCLAW_E2E_EXPECTED_SHA,
planHash: PLAN_HASH,
correlationId: CORRELATION_ID,
passed: 1,
failed: 0,
Expand Down
5 changes: 1 addition & 4 deletions test/e2e/support/mcp-bridge-runtime-compatibility.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import {
import { RISK_SIGNAL_FILE } from "../../../tools/e2e/risk-signal.ts";

const EXPECTED_SHA = "a".repeat(40);
const PLAN_HASH = "b".repeat(64);
const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000";

function riskSignalEnv(artifactDirectory: string): NodeJS.ProcessEnv {
Expand All @@ -31,7 +30,6 @@ function riskSignalEnv(artifactDirectory: string): NodeJS.ProcessEnv {
GITHUB_WORKSPACE: "/test/workspace",
NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID,
NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA,
NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH,
NEMOCLAW_E2E_SHARD: "hermes",
};
}
Expand Down Expand Up @@ -94,7 +92,7 @@ describe("MCP bridge dev runtime compatibility", () => {
}
});

it("reports an exact unsupported-version rejection as passing compatibility evidence (#6426)", () => {
it("reports an exact unsupported-version rejection without a plan hash (#6426)", () => {
const result = classifyMcpBridgeRuntimeCompatibility(
assertRuntimeVersion("0.0.78-dev.6+ga7271169"),
);
Expand Down Expand Up @@ -146,7 +144,6 @@ describe("MCP bridge dev runtime compatibility", () => {
shardId: "hermes",
expectedSha: EXPECTED_SHA,
testedSha: EXPECTED_SHA,
planHash: PLAN_HASH,
correlationId: CORRELATION_ID,
passed: 1,
failed: 0,
Expand Down
3 changes: 0 additions & 3 deletions test/e2e/support/retired-selector-compatibility.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import {
} from "../../../tools/e2e/retired-selector-compatibility.mts";

const EXPECTED_SHA = "a".repeat(40);
const PLAN_HASH = "b".repeat(64);
const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000";
const REPLACEMENT_FILES = [
"src/lib/actions/sandbox/rebuild-flow-helpers.test.ts",
Expand Down Expand Up @@ -63,7 +62,6 @@ function environment(
TARGETS: targets,
NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID,
NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA,
NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH,
NEMOCLAW_E2E_SHARD: "default",
};
}
Expand Down Expand Up @@ -127,7 +125,6 @@ describe("retired E2E selector compatibility", () => {
shardId: "default",
expectedSha: EXPECTED_SHA,
testedSha: EXPECTED_SHA,
planHash: PLAN_HASH,
correlationId: CORRELATION_ID,
passed: 1,
failed: 0,
Expand Down
8 changes: 0 additions & 8 deletions tools/e2e/risk-signal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ export type E2eRiskSignal = {
shardId: string;
expectedSha: string;
testedSha: string;
planHash: string;
correlationId: string;
passed: number;
failed: number;
Expand All @@ -27,7 +26,6 @@ export type RiskSignalEnvironment = {
shardId: string;
expectedSha: string;
testedSha: string;
planHash: string;
correlationId: string;
};

Expand All @@ -37,7 +35,6 @@ export type RiskSignalCounts = Pick<
>;

const SHA_PATTERN = /^[a-f0-9]{40}$/u;
const HASH_PATTERN = /^[a-f0-9]{64}$/u;
const CORRELATION_PATTERN =
/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u;
const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u;
Expand All @@ -63,7 +60,6 @@ export function configuredRiskSignalEnvironment(
jobId: env.E2E_TARGET_ID ?? "",
shardId: env.NEMOCLAW_E2E_SHARD ?? "",
expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA,
planHash: env.NEMOCLAW_E2E_PLAN_HASH ?? "",
correlationId: env.NEMOCLAW_E2E_CORRELATION_ID ?? "",
};
if (!values.artifactDir) throw new Error("risk signal requires E2E_ARTIFACT_DIR");
Expand All @@ -74,9 +70,6 @@ export function configuredRiskSignalEnvironment(
if (!SHA_PATTERN.test(values.expectedSha)) {
throw new Error("risk signal requires a 40-character lowercase expected SHA");
}
if (!HASH_PATTERN.test(values.planHash)) {
throw new Error("risk signal requires a 64-character lowercase plan hash");
}
if (!CORRELATION_PATTERN.test(values.correlationId)) {
throw new Error("risk signal requires a lowercase UUIDv4 correlation id");
}
Expand All @@ -97,7 +90,6 @@ export function buildRiskSignal(
shardId: environment.shardId,
expectedSha: environment.expectedSha,
testedSha: environment.testedSha,
planHash: environment.planHash,
correlationId: environment.correlationId,
...counts,
};
Expand Down
Loading