diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index 6b54aa5e7ec..e8bdb7e2f02 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-cut-release-tag -description: Creates deterministic NemoClaw semver release tags on origin/main after verifying the pre-tag dated changelog entry, handles release housekeeping, drafts announcement release notes, and verifies the maintainer-published Announcement. Use when cutting a release, tagging a version, shipping a build, creating vX.Y.Z tags, publishing release announcements, or completing release communication. +description: Prepares and cuts NemoClaw's semi-automatic daily release tag after verifying release-prep docs and housekeeping, then drafts announcement release notes and verifies the maintainer-published Announcement. Use when cutting a release, tagging a version, shipping a build, creating vX.Y.Z tags, or completing release communication. user_invocable: true --- @@ -9,90 +9,54 @@ user_invocable: true # Cut Release Tag -Use the release scripts for normal release operations. Do not run raw `git tag`, `git push`, `gh api`, or version-bump commands by hand for the normal release flow. +At 4 PM America/Los_Angeles, prepare and cut the daily release with an authorized maintainer. This is semi-automatic: the agent verifies that the dated changelog PR is merged, reviews the release plan and housekeeping, exercises the maintainer's local signer, and asks for the exact confirmation phrase before pushing the tag. -The release is one signed annotated semver tag on an already-merged `origin/main` commit. The GitHub workflow requires that tag to be GitHub-Verified, points `latest` at the exact verified tag object, carries remaining open issues/PRs to the next patch label, and deletes the released label while holding the shared release-label coordination queue; release admins promote `lkg` manually after validation. After the workflow is verified, draft release notes, then verify the maintainer-published Announcement before final handoff. +Every push to `main` separately starts the complete E2E workflow. Treat those results as asynchronous regression evidence: from 4 PM through 8 AM, keep merging normally while agents consolidate failures, remove redundant coverage, and fix broken or flaky E2Es. E2E does not select, delay, cancel, or authorize the daily tag. -## LKG Production Image Dispatch - -When a release admin creates or moves `lkg` to a commit carrying a `vX.Y.Z` tag, the `Release / LKG Brev Image` workflow dispatches the `Release Production Image` workflow in `brevdev/nemoclaw-image` on its `main` branch. -The dispatch passes the immutable semver tag instead of the mutable `lkg` tag. -The source workflow requires the `NEMOCLAW_IMAGE_DISPATCH_TOKEN` Actions secret with Actions read/write access to `brevdev/nemoclaw-image`; a missing secret fails before the API request, and the workflow summary never includes its value. -The trigger summary records the selected release tag, full commit SHA, target workflow, dispatch result, downstream run ID, and a direct link to the downstream run. -After `lkg` promotion, find and wait for the source trigger run using the promoted commit: - -```bash -gh run list --repo NVIDIA/NemoClaw --workflow release-lkg-brev-image.yaml --commit --event push --limit 1 --json databaseId,status,conclusion,url -gh run watch --repo NVIDIA/NemoClaw --exit-status -gh run view --repo NVIDIA/NemoClaw --log -``` - -Extract the exact `https://github.com/brevdev/nemoclaw-image/actions/runs/` URL printed by the source run, give that link to the maintainer immediately, and tell them to follow it to terminal success. -Treat dispatch acceptance as an intermediate state, not proof of production image promotion: the downstream run must succeed and its summary must show successful runtime E2E validation and promotion of the `nemoclaw-brev-cpu` image family. -A rejected dispatch fails the trigger run but does not move or roll back `lkg`. -Deleting `lkg` does not dispatch an image build. -The downstream scheduled reconciliation remains available if the event-driven dispatch fails or is delayed. +Use the release scripts for release operations. Do not run raw `git tag`, `git push`, `gh api`, or version-bump commands by hand for the normal release flow. ## Hard Rules -- Tag only the commit captured in a generated release plan. -- Do not generate the release plan until the release-prep docs PR containing `docs/changelog/YYYY-MM-DD.mdx` and the exact planned `## vX.Y.Z` heading is merged or explicitly waived. -- Treat the dated MDX entry as the canonical release history. A conventional Release Notes page or post-tag Announcement draft cannot replace it. -- If `origin/main` changes after plan generation, regenerate the plan before cutting the tag. -- Before asking for release confirmation, satisfy the canonical [pre-tag E2E evidence policy](../nemoclaw-maintainer-policies/references/release-train.md#pre-tag-e2e-evidence) for that commit. -- Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E and `Exact staging Brev Launchable` evidence. -- Ask the maintainer to paste the confirmation phrase from the plan before cutting the tag. -- Push only the semver tag (`vX.Y.Z`) from the agent-controlled step. -- Never push `latest` or `lkg` from this skill. -- Never move, delete, or force-push an existing remote semver tag unless the maintainer explicitly starts protected-tag remediation. +- Do not generate the release plan until the release-prep docs PR containing `docs/changelog/YYYY-MM-DD.mdx` and the exact planned `## vX.Y.Z` heading is merged or a maintainer explicitly waives it. +- Tag only the `origin/main` commit captured by the generated release plan. If `origin/main` moves before the cut, regenerate the plan and request its new confirmation phrase; do not stop merging. +- Do not consult E2E state before cutting the tag. +- Ask the maintainer to paste the exact confirmation phrase from the plan before cutting the tag. +- Sign on the maintainer's workstation with its configured OpenPGP, SSH, or X.509 signer. Never put the release signing key in a GitHub Actions secret or use a release bot to sign tags. +- Push only one new signed annotated semver tag. The tag must become GitHub-Verified. +- Never move, delete, or force-push an existing remote semver tag unless a maintainer explicitly starts protected-tag remediation. +- Never push `latest` or `lkg` from this skill. Let `release-latest-tag` move `latest`; release admins promote `lkg` after validation. - Delete the released version label only after open work moves forward and a final query finds no open stragglers. Never rename or reuse a released label. -- Keep label retirement inside the `release-latest-tag` workflow so it cannot overlap the post-merge labeler. Do not run the retirement script directly. -- Draft release notes locally. Do not create the GitHub Discussion; the maintainer does that. -- Do not mark the announcement step complete until the maintainer provides a valid Discussion URL and the published Announcement is verified. -- Follow the shared [Git and GitHub Access Hard Stop](../_shared/git-github-hard-stop.md) for SSH, authentication, remote access, authorization, or permission failures. +- Keep label retirement inside the `release-latest-tag` workflow. Do not run the retirement script directly. +- Draft release notes locally. Do not create the GitHub Discussion; the maintainer publishes it. +- Follow the shared [Git and GitHub Access Hard Stop](../_shared/git-github-hard-stop.md) for access or permission failures. ## Workflow -Copy this checklist and update it as you proceed: - -```text -Release Progress: -- [ ] Step 1: Preflight and generate release plan -- [ ] Step 2: Show plan, E2E evidence, and confirmation phrase -- [ ] Step 3: Cut the semver tag from the confirmed plan -- [ ] Step 4: Wait for workflow-managed latest -- [ ] Step 5: Carry open work forward and retire the released label -- [ ] Step 6: Generate release-note data and draft Markdown -- [ ] Step 7: Wait for maintainer-published Announcement -- [ ] Step 8: Verify Announcement and hand off sharing -``` - -### Step 1: Preflight and Generate Release Plan - -Start with one read-only pass that checks these prerequisites together: - -- refresh `origin/main` and resolve its full SHA; -- check the target changelog heading and release-prep docs state; and -- inventory existing E2E runs for the same SHA before deciding what to dispatch. +### Step 1: Verify release prep and generate the plan -Do not wait for merges to stop. The plan captures one candidate SHA for evidence; a late drift check advances it when `origin/main` moves. -Do not dispatch or poll a workflow during this pass. - -Before this step, confirm release-prep docs are merged or explicitly waived. -Return to `nemoclaw-maintainer-evening` if docs are still pending. - -For the planned version, inspect `origin/main` before generating the plan: +Refresh `origin/main` and confirm the release-prep docs PR is merged. Set `NEXT_TAG` to the selected release version, then require exactly one matching heading in a directly nested dated changelog: ```bash -git grep -n '^## vX\.Y\.Z$' origin/main -- 'docs/changelog/*.mdx' +NEXT_TAG="" +[[ "$NEXT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "NEXT_TAG must be a semver tag such as v0.0.58" >&2 + exit 1 +} +CHANGELOG_MATCHES="$( + git grep -n -E "^## ${NEXT_TAG//./\\.}$" origin/main -- \ + ':(glob)docs/changelog/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].mdx' || true +)" +CHANGELOG_MATCH_COUNT="$(printf '%s\n' "$CHANGELOG_MATCHES" | sed '/^$/d' | wc -l | tr -d ' ')" +if [[ "$CHANGELOG_MATCH_COUNT" != "1" ]]; then + echo "Expected exactly one dated changelog heading for $NEXT_TAG; found $CHANGELOG_MATCH_COUNT" >&2 + exit 1 +fi +printf '%s\n' "$CHANGELOG_MATCHES" ``` -Require exactly one match in a dated file directly under `docs/changelog/`. -Confirm that a newly created file begins with the parser-safe MDX SPDX comment and that the entry contains its summary and detailed bullets. -If the entry is missing or malformed, return to `nemoclaw-contributor-update-docs`; do not substitute the post-tag announcement workflow. -If the maintainer explicitly waives the entry, preserve the reason in the release-plan presentation and confirmation handoff. +Unless Step 1 records an explicit waiver, require the plan's next tag to match that dated changelog heading. When waived, show the recorded waiver reason in the plan presentation. A conventional Release Notes page or post-tag Announcement draft cannot replace the dated changelog. -Run one of: +Generate the requested plan: ```bash npm run release:plan -- --bump patch @@ -100,178 +64,39 @@ npm run release:plan -- --bump minor npm run release:plan -- --bump major ``` -Patch is the default if the maintainer says "yes", "go", or similar without choosing. - -The script writes a plan outside the checkout root, for example: - -```text -../nemoclaw-release-v0.0.58/plan.json -``` - -### Step 2: Show Plan, E2E Evidence, and Ask for Confirmation - -Read the generated `plan.json` and show the maintainer: - -- previous tag, -- next tag, -- target `origin/main` commit and headline, -- plan hash, -- forbidden operations, -- confirmation phrase, -- open issue/PR housekeeping plan for the release label, including deletion of the released label after carry-forward succeeds. - -Unless Step 1 records an explicit waiver, verify that the plan's next tag matches the H2 version heading in the dated changelog entry at the candidate SHA. -When the entry is waived, show the recorded waiver reason in the plan presentation and confirmation handoff instead. - -For the plan's full `origin/main` SHA, review `.github/workflows/e2e.yaml` at that commit and build the evidence ledger required by the canonical [pre-tag E2E evidence policy](../nemoclaw-maintainer-policies/references/release-train.md#pre-tag-e2e-evidence). The workflow is the sole source of truth; do not substitute or maintain a separate release-gating test list. - -From a checkout whose `HEAD` is the plan candidate SHA and whose `git status --short` is empty, generate one release E2E preflight: - -```bash -CANDIDATE_SHA="" -npm run release:e2e-evidence -- \ - --candidate-sha "$CANDIDATE_SHA" \ - >"$EVIDENCE_DIR/preflight.json" -``` - -The preflight derives every required execution from one empty-selector dispatch. -The full run includes every default-selected workflow E2E plus `Exact staging Brev Launchable`. -Accepted release evidence requires `allow_jetson_runner_queue=false` and `allow_dgx_spark_runner_queue=false`. -The required denominator excludes `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification`. -Each job that declares `RELEASE_E2E_ACTIVATION_PATH` requires that path at the candidate SHA. -A missing activation path is a preflight failure. - -Check whether one existing full run for the candidate SHA contains complete evidence. If it does not, load `nemoclaw-maintainer-e2e` and dispatch one full run. Do not combine evidence from different workflow run IDs. Do not substitute a selective run for full-run evidence. - -Monitor the dispatched correlation ID with one bounded status query. - -Before accepting full-mode exact Brev evidence, require: - -- the workflow `head_sha` to equal the plan candidate SHA; -- the trusted dispatch receipt to prove empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`; -- the workflow conclusion to be `success`; -- the `Exact staging Brev Launchable` job conclusion to be `success`; -- the job URL and selected successful Launchable job attempt; -- Launchable E2E identity for the same SHA; and -- cleanup evidence that reports the qualified workspace as `ABSENT`. - -Treat a skipped job as missing evidence even when the workflow concludes `success`. -If the plan candidate SHA changes, discard the run and Launchable E2E evidence. -Run full mode again for the new candidate SHA. -No release-note-only delta exception is currently defined. - -For the accepted full run, reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` returned by `nemoclaw-maintainer-e2e`, and collect the workflow-produced dispatch receipt. -If those files were not returned, collect them once: - -```bash -gh api "repos/NVIDIA/NemoClaw/actions/runs/$RUN_ID" \ - >"$EVIDENCE_DIR/run-$RUN_ID.json" -gh api --paginate --slurp \ - "repos/NVIDIA/NemoClaw/actions/runs/$RUN_ID/jobs?filter=all&per_page=100" \ - >"$EVIDENCE_DIR/jobs-$RUN_ID.json" -ARTIFACT_PAGES="$(gh api --paginate --slurp \ - "repos/NVIDIA/NemoClaw/actions/runs/$RUN_ID/artifacts?per_page=100")" -DISPATCH_ARTIFACT_NAME="$(jq -r --arg prefix "e2e-dispatch-$RUN_ID-" \ - '[.[] | .artifacts[] | select(.expired != true and (.name | startswith($prefix)))] - | sort_by(.created_at) | last | .name // empty' <<<"$ARTIFACT_PAGES")" -test -n "$DISPATCH_ARTIFACT_NAME" -gh run download "$RUN_ID" \ - --repo NVIDIA/NemoClaw \ - --name "$DISPATCH_ARTIFACT_NAME" \ - --dir "$EVIDENCE_DIR/dispatch-$RUN_ID" -``` - -Use the latest existing receipt artifact, not the run's latest attempt number. A partial rerun can leave `generate-matrix` successful and therefore reuse its earlier receipt; the ledger permits that earlier receipt only when it binds the same run and its attempt does not exceed the run's latest attempt. - -Successful workflow E2E and `Exact staging Brev Launchable` evidence may accumulate across rerun attempts of that workflow run. Evidence from another workflow run does not satisfy the ledger. - -Create `manifest.json` in the private evidence directory: - -```json -{ - "candidateSha": "", - "runs": [ - { - "runJson": "run-123.json", - "jobsJson": "jobs-123.json", - "dispatchJson": "dispatch-123/dispatch.json" - } - ] -} -``` - -Do not type empty-selector claims or selector lists into the manifest. The helper derives them from the workflow-produced receipt and rejects a receipt whose selector fields disagree with its empty-selector flag. -Build the ledger with `npm run release:e2e-evidence -- --manifest "$EVIDENCE_DIR/manifest.json"`. -The helper derives the denominator from the workflow, preserves matrix rows as separate semantic identifiers, binds every run and its actual dispatch inputs to the candidate SHA, and keeps an earlier successful attempt when a later attempt fails. -The manifest and helper cover the workflow-derived test execution ledger only. They do not replace exact Brev Launchable E2E acceptance: keep the raw `dispatch.json`, `launchable-e2e.json`, and `cleanup.json` validation in `nemoclaw-maintainer-e2e`, and carry its validated return beside this ledger or record the required Launchable E2E exception. +Patch is the default daily bump. Show the previous tag, next tag, target commit, plan hash, forbidden operations, carry-forward plan, label-retirement plan, and confirmation phrase. Do not show or request E2E evidence. -Reject a failed workflow run before presenting the ledger. Rerun its failed jobs until the same workflow run concludes with `success`. Exceptions apply only to missing or skipped executions in that otherwise successful run. +### Step 2: Verify local signing and request confirmation -Before showing the confirmation prompt, present: - -- the candidate SHA; -- the number of tests with successful evidence out of the number required by the workflow; -- each required test mapped to a successful run or job URL and attempt; and -- when accepted full-mode exact Brev evidence exists, its workflow URL, `Exact staging Brev Launchable` job URL, selected evidence attempt, Launchable E2E identity, and cleanup result; and -- a separate itemized maintainer exception for each missing or skipped execution in the accepted successful workflow run, including its test identifier, run links, current result, and rationale; and -- a separate itemized maintainer exception for missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run, including run and job URLs, the missing or invalid receipt, and rationale. - -Do not ask for the phrase until the workflow run concludes with `success` and each test and the exact Brev Launchable E2E job has successful evidence or its own permitted itemized exception. -Immediately before asking, refresh `origin/main` once and compare its full SHA with the plan. If it moved, discard all prior candidate-bound evidence, regenerate the plan, rerun preflight and the full E2E workflow for the new SHA, capture a new manifest, and rebuild the ledger before requesting confirmation. - -Exercise the configured Git signing backend before asking for confirmation: +Run: ```bash npm run release:cut -- --plan --preflight-only ``` -Require status 0. This preflight creates and deletes one local temporary tag. It does not push a ref. Git selects the maintainer's configured OpenPGP, SSH, or X.509 signer. - -Ask the maintainer to paste this phrase: +This exercises the maintainer workstation's configured signer without publishing a ref. Require status 0, then ask the maintainer to paste the exact phrase from the plan: ```text CONFIRM RELEASE vX.Y.Z ``` -Do not proceed on a generic "yes" at this step. - -### Step 3: Cut the Semver Tag - -Run the cut script with the plan and the maintainer's phrase: - -```bash -npm run release:cut -- --plan --confirm "CONFIRM RELEASE vX.Y.Z " -``` - -The script verifies a clean worktree, unchanged `origin/main`, tag availability, target reachability, and remote peeled tag state, then creates and pushes the signed annotated tag using the configured signing key. It writes: +Do not proceed on a generic confirmation. -```text -/cut-result.json -``` - -If the script fails because of SSH, authentication, remote access, authorization, or permissions, follow [Git and GitHub Access Hard Stop](../_shared/git-github-hard-stop.md). For other precondition failures, report the failed precondition and use the recovery guidance below. Do not improvise git commands. - -### Step 4: Wait for Workflow-Managed `latest` +### Step 3: Cut the signed tag Run: ```bash -npm run release:wait-latest -- --plan -``` - -The script waits until `vX.Y.Z` and `latest` reference the same tag object, verifies both peel to the planned commit, and verifies `lkg` did not change from the plan. It writes: - -```text -/latest-result.json +npm run release:cut -- --plan --confirm "CONFIRM RELEASE vX.Y.Z " ``` -If it fails, report the failed workflow/status. Do not manually move `latest`. +The script verifies the plan hash, clean worktree, unchanged `origin/main`, target reachability, tag availability, local signer, and remote peeled state before writing `cut-result.json`. If `origin/main` moved, regenerate the plan and ask for its new phrase. This refresh does not freeze or gate merging. -### Step 5: Verify Carry-Forward and Label Retirement +### Step 4: Verify `latest` and housekeeping -The `release-latest-tag` workflow continues after moving `latest`: it moves every remaining open issue or PR carrying the released version to the next patch label, verifies none remain, and deletes the released label. The workflow and post-merge labeler share one queued concurrency group, so assignment cannot overlap the verification-and-delete window. +Run `npm run release:wait-latest -- --plan `. Require the semver tag and `latest` to reference the same tag object and peel to the planned commit. Verify `lkg` did not move. -Find the workflow run started by Step 3 and wait for it to finish: +Find exactly one `release-latest-tag` push run for the release commit and wait for it: ```bash RELEASE_SHA="" @@ -286,90 +111,26 @@ fi gh run watch "${RELEASE_RUN_IDS[0]}" --repo NVIDIA/NemoClaw --exit-status ``` -This automatic post-tag housekeeping is covered by the release plan and confirmation in Step 2. Do not run `scripts/retire-release-label.mts` directly; doing so would bypass the coordination boundary. - -Then verify the released version label no longer exists: - -```bash -gh label list --repo NVIDIA/NemoClaw --search --json name \ - --jq '.[] | select(.name == "")' -``` - -The command must return no output. Never rename the released label into a future version; a future target must be a separately created label with its own GitHub identity. - -Summarize: - -- open issues/PRs moved to ``; -- released label deleted; -- any items that need manual maintainer attention. - -### Step 6: Generate Release-Note Data and Draft Markdown - -Collect deterministic release-note input: - -```bash -npm run release:notes-data -- --plan -``` - -This writes: - -```text -/notes-data.json -``` - -If `notes-data.json` has `status: "partial"` or non-empty `pullRequestWarnings`, report the warnings and ask the maintainer whether to fetch/fill the missing PR metadata before drafting. - -Load and follow `nemoclaw-maintainer-release-notes`, then use its output as the draft. Save only Markdown, outside the checkout root: - -```text -/release-note-draft.md -``` - -Before continuing to Step 7, verify the draft has three lead paragraphs, categorized shipped changes, one what-changed-and-why-it-matters bullet with a visible `#NNNN` link for every included change, and thanks for external contributors only. - -Do not create or update a GitHub Discussion. -Do not edit `docs/changelog/` in this post-tag step; the canonical entry must already be present in the tagged commit. - -### Step 7: Wait for Maintainer-Published Announcement +Confirm open items moved to the next patch label, no open item retains the released label, and the released label was deleted. -Return: +## Release Notes and Announcement -- release tag, -- confirmed release commit, -- plan path and plan hash, -- `cut-result.json`, `latest-result.json`, and `notes-data.json` paths, -- Markdown draft path, -- issue/PR housekeeping summary, -- suggested discussion title: `NemoClaw is out`. +Run `npm run release:notes-data -- --plan `, then load `nemoclaw-maintainer-release-notes`. Save the Markdown draft outside the checkout. Do not edit `docs/changelog/` after tagging and do not publish the Discussion. -Ask the maintainer to publish the draft in the `Announcements` Discussion category and return the resulting Discussion URL. Do not create or update the Discussion. Keep Step 7 in progress until the maintainer provides the URL. +Ask the maintainer to publish the draft in the `Announcements` category with title `NemoClaw vX.Y.Z is out` and return its URL. Accept only `https://github.com/NVIDIA/NemoClaw/discussions/` without a query or fragment. Verify the title, category, compare link, PR links, contributor names, and substantive draft content before completing the handoff. -### Step 8: Verify Announcement and Hand Off Sharing - -Before making any network request, reject the maintainer-provided URL unless it matches `https://github.com/NVIDIA/NemoClaw/discussions/` with no query string or fragment. Only then open it using a read-only GitHub or web capability and verify: - -- the title is `NemoClaw is out`; -- the category is `Announcements`; -- the body preserves the draft's three lead paragraphs, category headings, every included PR link, comparison URL, and external contributor usernames; formatting-only edits are acceptable; -- the comparison link targets `...` and visible PR links target `github.com/NVIDIA/NemoClaw/pull/`. +## LKG Production Image Dispatch -If the Announcement is valid, return its URL with the release artifacts and mark the release workflow complete. Remind the maintainer to share that Discussion URL in the appropriate external channels. Do not create a duplicate Announcement. +When a release admin creates or moves `lkg` to a commit carrying a semver tag, `Release / LKG Brev Image` dispatches `Release Production Image` in `brevdev/nemoclaw-image`. Treat dispatch acceptance as intermediate; verify the downstream run succeeds, runtime E2E validation passes, and the `nemoclaw-brev-cpu` image family is promoted. A rejected dispatch does not move or roll back `lkg`. ## Recovery -- Plan generation fails: fix the named precondition, then regenerate the plan. -- Planned changelog entry is missing or malformed: stop before plan generation and run the pre-tag `nemoclaw-contributor-update-docs` workflow. Use post-release recovery only when the tag already exists. -- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable E2E job finishes. -- Full-mode E2E ran for another SHA: reject the run and dispatch full mode for the plan candidate SHA. -- `Exact staging Brev Launchable` was skipped in an otherwise successful candidate run: dispatch full mode again or record the required itemized maintainer exception. -- Launchable E2E or cleanup evidence is missing or invalid in an otherwise successful candidate run: dispatch full mode again or record the separate itemized maintainer exception. Do not infer Launchable E2E success from the workflow conclusion. -- `origin/main` moved after plan generation: regenerate the plan and ask for the new confirmation phrase. -- Remote semver tag already exists: stop; do not retag unless the maintainer explicitly starts protected-tag remediation. -- Signing preflight fails: fix the reported Git signer or signing-key failure. Run the preflight again before requesting confirmation. -- `latest` workflow fails or times out: report the workflow/status; do not move `latest` manually. -- `latest` workflow rejects a rollback: keep `latest` unchanged, inspect the plan target commit, and regenerate the plan for the current `origin/main` tip if appropriate. -- `lkg` changed: stop and escalate to a release admin. -- Post-tag housekeeping fails: report the workflow error and list items still carrying the released label. After the failure is fixed, rerun `release-latest-tag.yaml` with `` through `workflow_dispatch`; the promotion and retirement steps are idempotent, already-moved items no longer match the source label, and an already-deleted released label is treated as success. Do not run the retirement script outside the workflow. -- Announcement is not published yet: keep Step 7 in progress and return the draft path and suggested title; the tag and housekeeping remain complete. -- Announcement title, category, body, or links are wrong: ask the maintainer to edit the existing Discussion, then verify the same URL again. Do not create a replacement. After three failed verification attempts for the same Discussion, stop and escalate to a release admin. -- Announcement cannot be inspected: report the read failure and ask the maintainer to confirm access or provide a public URL; do not mark Step 8 complete. +- Dated changelog is missing: merge the release-prep docs entry, then regenerate the plan. Record an explicit maintainer waiver only when directed. +- `origin/main` moved after plan generation: regenerate the plan and request its new confirmation phrase. +- Latest remote semver changed: regenerate the plan; never race or overwrite the existing tag. +- Signing preflight fails: repair the maintainer workstation's configured signer. Do not move the signing key into GitHub Actions. +- Tag push fails: follow the Git and GitHub access hard stop; do not improvise another signer or remote. +- `latest` or label retirement fails: rerun `release-latest-tag.yaml` for the immutable semver tag. Do not move `latest` or retire labels by hand. +- E2E fails: leave the tag in place and continue the asynchronous stabilization loop. +- `lkg` changes unexpectedly: stop and escalate to a release admin. +- Announcement is not published: return the draft and keep announcement verification pending. diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts deleted file mode 100644 index 5c3f9dd07d9..00000000000 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts +++ /dev/null @@ -1,647 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -import YAML from "yaml"; -import { readFreeStandingJobsInventory } from "../../../../tools/e2e/workflow-boundary.mts"; -import { - buildE2eWorkflowPlan, - type E2eWorkflowPlan, -} from "../../../../tools/e2e/workflow-plan.mts"; - -type JsonRecord = Record; -type ExecutionGroup = "default"; - -export type ReleaseE2eExecution = { - id: string; - jobId: string; - expectedName: string; - group: ExecutionGroup; -}; - -export type ReleaseE2ePreflight = { - candidateSha: string; - dispatches: { - completeRun: { - includeStagingBrevLaunchable: true; - jobs: ""; - mode: "full"; - targets: ""; - }; - }; - exceptionsRequired: string[]; - executions: ReleaseE2eExecution[]; - launchableE2eJobId: string; - requiredExecutionCount: number; -}; - -export type ReleaseE2eRunEvidence = { - dispatch: unknown; - jobs: unknown; - run: unknown; -}; - -export type ReleaseE2eLedgerEntry = ReleaseE2eExecution & { - attempts: Array<{ - attempt: number; - conclusion: string; - status: string; - jobUrl: string; - runUrl: string; - }>; - successfulEvidence?: { - attempt: number; - jobUrl: string; - runUrl: string; - }; - status: "missing" | "successful"; -}; - -export type ReleaseE2eLedger = { - candidateSha: string; - entries: ReleaseE2eLedgerEntry[]; - successfulCount: number; - missingCount: number; - requiredCount: number; -}; - -type ReleaseEvidenceManifest = { - candidateSha: string; - runs: Array<{ - dispatchJson: string; - jobsJson: string; - runJson: string; - }>; -}; - -type CliOptions = { - candidateSha?: string; - manifest?: string; - workflowPath: string; -}; - -const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", ".."); -const DEFAULT_WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; -const SAFE_REPO_PATH_PATTERN = /^(?!\/)(?!.*(?:^|\/)\.\.(?:\/|$))[^\\]+$/u; -const MATRIX_EXPRESSION_PATTERN = /\$\{\{\s*matrix\.([A-Za-z0-9_-]+)\s*\}\}/gu; -const OPT_IN_HARDWARE_JOB_IDS = new Set([ - "jetson-nvmap-gpu", - "llama-cpp-dgx-spark-plan", - "llama-cpp-dgx-spark-qualification", -]); - -function record(value: unknown, label: string): JsonRecord { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error(`${label} must be an object`); - } - return value as JsonRecord; -} - -function stringField(value: JsonRecord, field: string, label: string): string { - const result = value[field]; - if (typeof result !== "string" || result.length === 0) { - throw new Error(`${label}.${field} must be a non-empty string`); - } - return result; -} - -function numberField(value: JsonRecord, field: string, label: string): number { - const result = value[field]; - if (!Number.isInteger(result) || (result as number) < 1) { - throw new Error(`${label}.${field} must be a positive integer`); - } - return result as number; -} - -function booleanField(value: JsonRecord, field: string, label: string): boolean { - const result = value[field]; - if (typeof result !== "boolean") { - throw new Error(`${label}.${field} must be a boolean`); - } - return result; -} - -function requireEqual(actual: unknown, expected: unknown, label: string): void { - if (actual !== expected) { - throw new Error(`${label} must equal ${JSON.stringify(expected)}`); - } -} - -function requireSha(value: JsonRecord, field: string, label: string): string { - const sha = stringField(value, field, label); - if (!SHA_PATTERN.test(sha)) { - throw new Error(`${label}.${field} must be a lowercase 40-character commit SHA`); - } - return sha; -} - -function requireRepository(value: JsonRecord, field: string, label: string): string { - const repository = stringField(value, field, label); - if (!REPOSITORY_PATTERN.test(repository)) { - throw new Error(`${label}.${field} must be an owner/repository name`); - } - return repository; -} - -function validateDispatchIdentity( - dispatch: JsonRecord, - candidateSha: string, - label: string, -): string { - requireEqual(dispatch.candidateSha, candidateSha, `${label}.candidateSha`); - const kind = stringField(dispatch, "kind", label); - if (kind === "nemoclaw-e2e-dispatch-v1") return candidateSha; - if (kind !== "nemoclaw-e2e-dispatch-v2") { - throw new Error( - `${label}.kind must equal "nemoclaw-e2e-dispatch-v1" or "nemoclaw-e2e-dispatch-v2"`, - ); - } - - requireEqual(dispatch.repository, "NVIDIA/NemoClaw", `${label}.repository`); - const candidateRepository = requireRepository(dispatch, "candidateRepository", label); - const baseSha = requireSha(dispatch, "baseSha", label); - const workflowSha = requireSha(dispatch, "workflowSha", label); - if (dispatch.prNumber === null) { - requireEqual(candidateRepository, "NVIDIA/NemoClaw", `${label}.candidateRepository`); - requireEqual(baseSha, candidateSha, `${label}.baseSha`); - requireEqual(workflowSha, candidateSha, `${label}.workflowSha`); - } else { - numberField(dispatch, "prNumber", label); - } - return workflowSha; -} - -function matrixRows(rawMatrix: unknown, jobId: string): JsonRecord[] { - const matrix = record(rawMatrix, `${jobId}.strategy.matrix`); - if (typeof matrix.include === "string") { - throw new Error(`${jobId} has a dynamic matrix that needs a planner-specific expansion`); - } - - const axes = Object.entries(matrix).filter( - ([key]) => - key !== "exclude" && key !== "include" && key !== "fail-fast" && key !== "max-parallel", - ); - let rows: JsonRecord[] = [{}]; - for (const [key, rawValues] of axes) { - if (!Array.isArray(rawValues) || rawValues.length === 0) { - throw new Error(`${jobId} matrix axis ${key} must be a non-empty array`); - } - rows = rows.flatMap((row) => - rawValues.map((value) => { - if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") { - throw new Error(`${jobId} matrix axis ${key} contains an unsupported value`); - } - return { ...row, [key]: value }; - }), - ); - } - - const excludes = Array.isArray(matrix.exclude) - ? matrix.exclude.map((row) => record(row, "exclude")) - : []; - rows = rows.filter( - (row) => - !excludes.some((excluded) => - Object.entries(excluded).every(([key, value]) => row[key] === value), - ), - ); - - if (Array.isArray(matrix.include)) { - if (axes.length > 0) { - throw new Error( - `${jobId} combines matrix axes and include rows; add explicit expansion support`, - ); - } - rows = matrix.include.map((row) => record(row, `${jobId}.strategy.matrix.include`)); - } - return rows; -} - -function renderMatrixJobName(jobId: string, rawJob: JsonRecord, row: JsonRecord): string { - const configuredName = rawJob.name; - if (configuredName !== undefined && typeof configuredName !== "string") { - throw new Error(`${jobId}.name must be a string when set`); - } - if (configuredName) { - const rendered = configuredName.replace(MATRIX_EXPRESSION_PATTERN, (_match, key: string) => { - if (!Object.hasOwn(row, key)) { - throw new Error(`${jobId}.name references missing matrix dimension ${key}`); - } - return String(row[key]); - }); - if (rendered.includes("${{ matrix.")) { - throw new Error(`${jobId}.name contains an unsupported matrix expression`); - } - return rendered; - } - return `${jobId} (${Object.values(row) - .map((value) => String(value)) - .join(", ")})`; -} - -function executionId(jobId: string, row: JsonRecord): string { - if (typeof row.id === "string" && row.id.length > 0) return `${jobId}[id=${row.id}]`; - const dimensions = Object.entries(row) - .map(([key, value]) => `${key}=${JSON.stringify(value)}`) - .join(","); - return `${jobId}[${dimensions}]`; -} - -function jobExecutions( - jobId: string, - rawJob: JsonRecord, - group: ExecutionGroup, - plan: E2eWorkflowPlan, -): ReleaseE2eExecution[] { - let rows: JsonRecord[] = []; - if (jobId === "live") rows = plan.matrix as unknown as JsonRecord[]; - else if (jobId === "shared-e2e") rows = plan.testMatrix as unknown as JsonRecord[]; - else { - const strategy = record(rawJob.strategy ?? {}, `${jobId}.strategy`); - if (strategy.matrix !== undefined) rows = matrixRows(strategy.matrix, jobId); - } - - if (rows.length === 0) { - const configuredName = rawJob.name; - return [ - { - expectedName: typeof configuredName === "string" ? configuredName : jobId, - group, - id: jobId, - jobId, - }, - ]; - } - return rows.map((row) => ({ - expectedName: renderMatrixJobName(jobId, rawJob, row), - group, - id: executionId(jobId, row), - jobId, - })); -} - -function workflowJobs(workflowPath: string): JsonRecord { - const workflow = record(YAML.parse(readFileSync(workflowPath, "utf8")), "workflow"); - return record(workflow.jobs, "workflow.jobs"); -} - -function isLaunchableE2eJob(jobId: string, job: JsonRecord): boolean { - const condition = job.if; - return ( - jobId === "staging-brev-launchable" && - typeof condition === "string" && - condition.includes("inputs.include_staging_brev_launchable") - ); -} - -function releaseActivationPath(job: JsonRecord, jobId: string): string | undefined { - const rawEnvironment = job.env; - if (rawEnvironment === undefined) return undefined; - const environment = record(rawEnvironment, `workflow.jobs.${jobId}.env`); - const activationPath = environment.RELEASE_E2E_ACTIVATION_PATH; - if (activationPath === undefined) return undefined; - if ( - typeof activationPath !== "string" || - activationPath.length === 0 || - !SAFE_REPO_PATH_PATTERN.test(activationPath) - ) { - throw new Error( - `${jobId}.env.RELEASE_E2E_ACTIVATION_PATH must be a nonempty relative repository path without backslashes or parent-directory segments`, - ); - } - return activationPath; -} - -function candidatePathExists(candidateSha: string, candidatePath: string): boolean { - try { - const output = execFileSync( - "git", - ["ls-tree", "--name-only", candidateSha, "--", candidatePath], - { - cwd: REPO_ROOT, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }, - ); - return output.trim() === candidatePath; - } catch (error) { - throw new Error( - `could not inspect release E2E activation path ${candidatePath} at candidate ${candidateSha}`, - { cause: error }, - ); - } -} - -export function buildReleaseE2ePreflight(input: { - candidateSha: string; - candidatePathExists?: (candidateSha: string, candidatePath: string) => boolean; - plan?: E2eWorkflowPlan; - workflowPath?: string; -}): ReleaseE2ePreflight { - if (!SHA_PATTERN.test(input.candidateSha)) { - throw new Error("candidateSha must be a lowercase 40-character commit SHA"); - } - const workflowPath = input.workflowPath ?? DEFAULT_WORKFLOW_PATH; - const jobs = workflowJobs(workflowPath); - const inventory = readFreeStandingJobsInventory(workflowPath); - const plan = input.plan ?? buildE2eWorkflowPlan(); - const pathExists = input.candidatePathExists ?? candidatePathExists; - const defaultJobIds = inventory.workflowJobs.filter( - (jobId) => jobId !== "shared-e2e" && !OPT_IN_HARDWARE_JOB_IDS.has(jobId), - ); - for (const jobId of defaultJobIds) { - const activationPath = releaseActivationPath( - record(jobs[jobId], `workflow.jobs.${jobId}`), - jobId, - ); - if (activationPath !== undefined && !pathExists(input.candidateSha, activationPath)) { - throw new Error( - `candidate commit is missing required E2E activation path ${activationPath} for ${jobId}`, - ); - } - } - const launchableE2eJobs = defaultJobIds.filter((jobId) => - isLaunchableE2eJob(jobId, record(jobs[jobId], `workflow.jobs.${jobId}`)), - ); - if (launchableE2eJobs.length !== 1) { - throw new Error(`expected exactly one Launchable E2E job, found ${launchableE2eJobs.length}`); - } - const launchableE2eJobId = launchableE2eJobs[0]!; - const executions = [ - ...defaultJobIds.flatMap((jobId) => - jobExecutions(jobId, record(jobs[jobId], `workflow.jobs.${jobId}`), "default", plan), - ), - ...jobExecutions("live", record(jobs.live, "workflow.jobs.live"), "default", plan), - ...jobExecutions( - "shared-e2e", - record(jobs["shared-e2e"], "workflow.jobs.shared-e2e"), - "default", - plan, - ), - ]; - const duplicateIds = executions - .map((execution) => execution.id) - .filter((id, index, ids) => ids.indexOf(id) !== index); - if (duplicateIds.length > 0) { - throw new Error(`release E2E execution identifiers are not unique: ${duplicateIds.join(",")}`); - } - - const exceptionsRequired: string[] = []; - - return { - candidateSha: input.candidateSha, - dispatches: { - completeRun: { - includeStagingBrevLaunchable: true, - jobs: "", - mode: "full", - targets: "", - }, - }, - exceptionsRequired, - executions, - launchableE2eJobId, - requiredExecutionCount: executions.length, - }; -} - -function flattenJobs(value: unknown): JsonRecord[] { - const pages = Array.isArray(value) ? value : [value]; - return pages.flatMap((page, pageIndex) => { - const jobs = record(page, `jobs page ${pageIndex}`).jobs; - if (!Array.isArray(jobs)) throw new Error(`jobs page ${pageIndex}.jobs must be an array`); - return jobs.map((job, jobIndex) => record(job, `jobs page ${pageIndex}.jobs[${jobIndex}]`)); - }); -} - -function matchesExpectedName(actual: string, expected: string): boolean { - if (actual === expected) return true; - if (!actual.endsWith("...")) return false; - return expected.startsWith(actual.slice(0, -3)); -} - -export function buildReleaseE2eLedger( - preflight: ReleaseE2ePreflight, - runs: readonly ReleaseE2eRunEvidence[], -): ReleaseE2eLedger { - if (runs.length !== 1) { - throw new Error( - `release E2E evidence requires exactly one workflow run, received ${runs.length}`, - ); - } - const attempts = new Map(); - - for (const [runIndex, evidence] of runs.entries()) { - const label = `runs[${runIndex}]`; - const run = record(evidence.run, `${label}.run`); - requireEqual(run.head_branch, "main", `${label}.run.head_branch`); - requireEqual(run.event, "workflow_dispatch", `${label}.run.event`); - requireEqual(run.path, ".github/workflows/e2e.yaml", `${label}.run.path`); - requireEqual(run.status, "completed", `${label}.run.status`); - requireEqual(run.conclusion, "success", `${label}.run.conclusion`); - const runId = numberField(run, "id", `${label}.run`); - const runAttempt = numberField(run, "run_attempt", `${label}.run`); - const runUrl = stringField(run, "html_url", `${label}.run`); - - const dispatch = record(evidence.dispatch, `${label}.dispatch`); - const expectedWorkflowSha = validateDispatchIdentity( - dispatch, - preflight.candidateSha, - `${label}.dispatch`, - ); - requireEqual(run.head_sha, expectedWorkflowSha, `${label}.run.head_sha`); - requireEqual(dispatch.eventName, "workflow_dispatch", `${label}.dispatch.eventName`); - requireEqual(dispatch.workflowRunId, String(runId), `${label}.dispatch.workflowRunId`); - const receiptAttempt = numberField(dispatch, "workflowRunAttempt", `${label}.dispatch`); - if (receiptAttempt > runAttempt) { - throw new Error(`${label}.dispatch.workflowRunAttempt exceeds the workflow run attempt`); - } - const jobsInput = dispatch.jobs; - const targetsInput = dispatch.targets; - if (typeof jobsInput !== "string" || typeof targetsInput !== "string") { - throw new Error(`${label}.dispatch jobs and targets must be strings`); - } - requireEqual(jobsInput, "", `${label}.dispatch.jobs`); - requireEqual(targetsInput, "", `${label}.dispatch.targets`); - requireEqual( - booleanField(dispatch, "emptySelectors", `${label}.dispatch`), - true, - `${label}.dispatch.emptySelectors`, - ); - requireEqual( - booleanField(dispatch, "includeStagingBrevLaunchable", `${label}.dispatch`), - true, - `${label}.dispatch.includeStagingBrevLaunchable`, - ); - requireEqual( - booleanField(dispatch, "allowJetsonRunnerQueue", `${label}.dispatch`), - false, - `${label}.dispatch.allowJetsonRunnerQueue`, - ); - requireEqual( - booleanField(dispatch, "allowDgxSparkRunnerQueue", `${label}.dispatch`), - false, - `${label}.dispatch.allowDgxSparkRunnerQueue`, - ); - - const selectedExecutions = preflight.executions; - for (const job of flattenJobs(evidence.jobs)) { - const jobRunId = numberField(job, "run_id", `runs[${runIndex}].job`); - const jobAttempt = numberField(job, "run_attempt", `runs[${runIndex}].job`); - if (jobRunId !== runId || jobAttempt > runAttempt) continue; - const name = stringField(job, "name", `runs[${runIndex}].job`); - const matches = selectedExecutions.filter((execution) => - matchesExpectedName(name, execution.expectedName), - ); - if (matches.length > 1) { - throw new Error( - `GitHub job name ${JSON.stringify(name)} ambiguously matches ${matches - .map((execution) => execution.id) - .join(",")}`, - ); - } - if (matches.length === 0) continue; - const execution = matches[0]!; - const values = attempts.get(execution.id) ?? []; - values.push({ - attempt: jobAttempt, - conclusion: stringField(job, "conclusion", `runs[${runIndex}].job`), - status: stringField(job, "status", `runs[${runIndex}].job`), - jobUrl: stringField(job, "html_url", `runs[${runIndex}].job`), - runUrl, - }); - attempts.set(execution.id, values); - } - } - - const entries = preflight.executions.map((execution): ReleaseE2eLedgerEntry => { - const executionAttempts = [...(attempts.get(execution.id) ?? [])].sort( - (left, right) => right.attempt - left.attempt || right.jobUrl.localeCompare(left.jobUrl), - ); - const successful = executionAttempts.find( - (attempt) => attempt.status === "completed" && attempt.conclusion === "success", - ); - return { - ...execution, - attempts: executionAttempts, - ...(successful - ? { - successfulEvidence: { - attempt: successful.attempt, - jobUrl: successful.jobUrl, - runUrl: successful.runUrl, - }, - } - : {}), - status: successful ? "successful" : "missing", - }; - }); - const successfulCount = entries.filter((entry) => entry.status === "successful").length; - return { - candidateSha: preflight.candidateSha, - entries, - successfulCount, - missingCount: entries.length - successfulCount, - requiredCount: entries.length, - }; -} - -function parseArgs(argv: readonly string[]): CliOptions { - const options: CliOptions = { - workflowPath: DEFAULT_WORKFLOW_PATH, - }; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - const value = argv[index + 1]; - if (arg !== "--candidate-sha" && arg !== "--manifest" && arg !== "--workflow") { - throw new Error(`Unknown argument: ${arg}`); - } - if (value === undefined) throw new Error(`${arg} requires a value`); - if (arg === "--candidate-sha") options.candidateSha = value; - else if (arg === "--manifest") options.manifest = value; - else options.workflowPath = value; - index += 1; - } - return options; -} - -function readManifest(manifestPath: string): { - manifest: ReleaseEvidenceManifest; - runs: ReleaseE2eRunEvidence[]; -} { - const directory = path.dirname(path.resolve(manifestPath)); - const raw = record(JSON.parse(readFileSync(manifestPath, "utf8")), "manifest"); - const manifest = raw as ReleaseEvidenceManifest; - if (!SHA_PATTERN.test(manifest.candidateSha) || !Array.isArray(manifest.runs)) { - throw new Error("release E2E evidence manifest has an invalid schema"); - } - const runs = manifest.runs.map((entry, index) => { - if ( - typeof entry.dispatchJson !== "string" || - typeof entry.jobsJson !== "string" || - typeof entry.runJson !== "string" - ) { - throw new Error(`manifest.runs[${index}] has an invalid schema`); - } - return { - dispatch: JSON.parse(readFileSync(path.resolve(directory, entry.dispatchJson), "utf8")), - jobs: JSON.parse(readFileSync(path.resolve(directory, entry.jobsJson), "utf8")), - run: JSON.parse(readFileSync(path.resolve(directory, entry.runJson), "utf8")), - }; - }); - return { manifest, runs }; -} - -function requireCandidateCheckout(candidateSha: string): void { - const headSha = execFileSync("git", ["rev-parse", "HEAD"], { - cwd: REPO_ROOT, - encoding: "utf8", - }).trim(); - if (headSha !== candidateSha) { - throw new Error(`checkout HEAD ${headSha} does not match candidate SHA ${candidateSha}`); - } -} - -export function runReleaseE2eEvidenceCli(argv = process.argv.slice(2)): void { - const options = parseArgs(argv); - if (options.manifest) { - const { manifest, runs } = readManifest(options.manifest); - requireCandidateCheckout(manifest.candidateSha); - const preflight = buildReleaseE2ePreflight({ - candidateSha: manifest.candidateSha, - workflowPath: options.workflowPath, - }); - process.stdout.write(`${JSON.stringify(buildReleaseE2eLedger(preflight, runs), null, 2)}\n`); - return; - } - if (options.candidateSha === undefined) { - throw new Error("--candidate-sha is required for preflight"); - } - requireCandidateCheckout(options.candidateSha); - process.stdout.write( - `${JSON.stringify( - buildReleaseE2ePreflight({ - candidateSha: options.candidateSha, - workflowPath: options.workflowPath, - }), - null, - 2, - )}\n`, - ); -} - -const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; -if (invokedFile === fileURLToPath(import.meta.url)) { - try { - runReleaseE2eEvidenceCli(); - } catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - } -} diff --git a/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md b/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md index 5257c16f5d1..6b0eb11ef6e 100644 --- a/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md +++ b/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md @@ -57,7 +57,7 @@ The team follows a daily ship cycle. All maintainer skills operate within this r 2. **During the day** (`/nemoclaw-maintainer-day`) — land PRs using the maintainer loop. Version labels make progress visible on dashboards. 3. **Evening** (`/nemoclaw-maintainer-evening`) — Check shipped work and the pre-tag changelog PR. Confirm that the pre-tag changelog PR contains `docs/changelog/YYYY-MM-DD.mdx` for the release. - Identify open items and prepare the QA summary. Record the release SHA and required E2E evidence. + Identify open items and prepare the QA summary. Record the release SHA and asynchronous E2E runs that still need overnight triage; E2E does not block the tag or merging. Cut the tag after confirmation. Move open items to the next patch label and delete the released label. Prepare the Announcement. 4. **Overnight** — A QA team in another time zone validates the tag. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 59e43ff6219..806863c868c 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -9,9 +9,7 @@ description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw mai # Run Maintainer E2E Use `.github/workflows/e2e.yaml` from trusted `main`. -Every push to `main` selects the default workflow E2E jobs. -Push runs skip `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` because push events cannot set the required workflow dispatch flags. -Pre-tag evidence still requires the full `workflow_dispatch` mode described below. +Every push to `main` selects every workflow E2E. A selected job can remain queued until its configured runner is available. No E2E job is excluded from trusted `main` push selection. Pre-tag evidence still requires the full `workflow_dispatch` mode described below. Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly requests local execution. ## Manual PR E2E @@ -65,10 +63,9 @@ Require a review reason containing 10 to 500 printable characters. Choose exactly one mode: - For a PR revision run, leave `E2E_JOBS` empty. The run selects: - - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; + - every free-standing workflow E2E except `Exact staging Brev Launchable`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. - The run skips `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` unless their separate runner-queue flags are `true`. - For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. Leave `targets` empty and keep Launchable disabled: @@ -161,10 +158,9 @@ A generic E2E request must not authorize the Brev Launchable path. Do not infer full mode from words such as “all” or “complete.” Ask for clarification only when the request contains conflicting mode phrases. -Ordinary mode selects every default-selected workflow E2E except `Exact staging Brev Launchable`. +Ordinary mode selects every workflow E2E except `Exact staging Brev Launchable`. Launchable mode runs only `Exact staging Brev Launchable`. -Full mode adds `Exact staging Brev Launchable` to the default E2E selection in the same workflow run. -The documented invocations for all three modes keep both hardware runner-queue flags set to `false`. +Full mode selects every workflow E2E, including `Exact staging Brev Launchable`, in the same workflow run. ## Resolve the Candidate @@ -201,8 +197,6 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs= \ -f inference_mode=mock \ -f include_staging_brev_launchable=false \ - -f allow_jetson_runner_queue=false \ - -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` @@ -216,8 +210,6 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs=staging-brev-launchable \ -f inference_mode=mock \ -f include_staging_brev_launchable=false \ - -f allow_jetson_runner_queue=false \ - -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` @@ -231,29 +223,16 @@ gh workflow run .github/workflows/e2e.yaml \ -f jobs= \ -f inference_mode=mock \ -f include_staging_brev_launchable=true \ - -f allow_jetson_runner_queue=false \ - -f allow_dgx_spark_runner_queue=false \ -f "correlation_id=${CORRELATION_ID}" ``` Do not set `jobs=staging-brev-launchable` for full mode. -Empty `jobs` and `targets` select every default-selected workflow E2E except `Exact staging Brev Launchable`. -The `include_staging_brev_launchable` input adds the Launchable E2E job to that same run. +Empty `jobs` and `targets` select every workflow E2E except `Exact staging Brev Launchable`. +The boolean input adds the Launchable E2E job to that same run. The trusted `main` workflow verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That role check is the authorization. -A user permitted to dispatch this workflow may set -`allow_jetson_runner_queue=true` to add `jetson-nvmap-gpu` to an empty-selector -manual run or enable its explicit selection. Set it only after a repository -administrator confirms an online Jetson runner in the authoritative runner -inventory. -A permitted dispatcher may set `allow_dgx_spark_runner_queue=true` to add -`llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` to an -empty-selector manual run or enable explicit qualification selection. Set it -only after a repository administrator confirms an online DGX Spark runner in -the authoritative runner inventory. -If GitHub pauses the qualification job for the `approve-dgx-spark-image-qualification` environment, an authorized environment reviewer must approve it before qualification starts. -`Exact staging Brev Launchable` does not require environment approval. +Every push and every empty-selector manual run selects `llama-cpp-dgx-spark-qualification`. If GitHub pauses the job for the `approve-dgx-spark-image-qualification` environment, an authorized environment reviewer must approve it before qualification starts. `Exact staging Brev Launchable` does not require environment approval. Find the run by its unique title: @@ -311,7 +290,7 @@ gh api --paginate --slurp \ >"$EVIDENCE_DIR/jobs-$RUN_ID.json" ``` -Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` as the `nemoclaw-maintainer-cut-release-tag` manifest inputs and as the full-mode validator inputs. Do not fetch the same run again. `jobs-latest-$RUN_ID.json` is only for ordinary and Launchable modes. +Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` as the full-mode validator inputs. Do not fetch the same run again. `jobs-latest-$RUN_ID.json` is only for ordinary and Launchable modes. For ordinary and Launchable modes, require `run-$RUN_ID.json` to report: @@ -351,7 +330,7 @@ node --experimental-strip-types --no-warnings \ The validator requires: - the workflow run to succeed for the selected SHA; -- `dispatch.json` to bind the same run, empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, `allowDgxSparkRunnerQueue: false`, and the selected successful Launchable job attempt; +- `dispatch.json` to bind the same run, empty selectors, `include_staging_brev_launchable=true`, and the selected successful Launchable job attempt; - `Exact staging Brev Launchable` to conclude `success` in the selected current or earlier attempt of the same workflow run; - `launchable-e2e.json` to identify the selected SHA in the repository and provision records; - the booted repository to be unmodified; @@ -364,7 +343,7 @@ A missing, mismatched, or failed cleanup receipt is not evidence. ## Bind Release Evidence -If no release plan exists, label a successful full run against `origin/main` as provisional release evidence. +Label a successful full run against `origin/main` as SHA-bound diagnostic evidence. Return: - candidate SHA; @@ -374,13 +353,11 @@ Return: - Launchable E2E identity; and - cleanup result. -If the release candidate SHA changes, discard the earlier full run and dispatch full mode for the new SHA. -No release-note-only delta exception is currently defined. +When validating another SHA, dispatch or locate a run bound to that exact SHA rather than reusing an earlier result. -When `nemoclaw-maintainer-cut-release-tag` invokes this skill, return the validated fields for its pre-tag E2E evidence ledger. -The trusted `dispatch.json` receipt proves that full mode used empty selectors, included `Exact staging Brev Launchable`, and disabled both optional hardware paths. -The release evidence ledger proves the result of each workflow E2E. -Do not ask for the release confirmation phrase in this skill. +Return the validated fields for post-merge triage or any explicitly requested full-run audit. +The trusted `dispatch.json` receipt proves that full mode used empty selectors and included `Exact staging Brev Launchable`. +The evidence proves the result of each workflow E2E; it does not gate merging or a release tag. ## Access Failures diff --git a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md index 330f2a7327b..5c94fc90b7c 100644 --- a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md @@ -6,7 +6,7 @@ user_invocable: true # NemoClaw Maintainer Evening -Wrap up the day: check progress, identify stragglers, summarize for QA, cut the tag, automatically carry stragglers to the next patch, retire the released label, and prepare release notes for posting. +Wrap up the day: prepare the dated changelog, check progress, identify stragglers, summarize for QA, cut the semi-automatic signed tag, and hand asynchronous E2E stabilization to the overnight loop. See [PR-REVIEW-PRIORITIES.md](../nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md) for the daily cadence. @@ -30,7 +30,7 @@ gh issue list --repo NVIDIA/NemoClaw --state open --label --limit 100 List open labeled PRs and issues as the post-tag housekeeping plan. Tell the maintainer that, after the tag and workflow-managed `latest` are verified, `cut-release-tag` will automatically move all of them to the next patch label and delete the released label. -If an item should leave the daily release flow instead of moving forward, remove it from the released-version label before asking for the release confirmation phrase. +If an item should leave the daily release flow instead of moving forward, remove it from the released-version label before the 4 PM tag. ## Step 3: Generate Handoff Summary @@ -46,32 +46,22 @@ Run `/nemoclaw-contributor-update-docs for ` before loading `cut-releas Confirm that the release-prep docs PR creates or updates one direct child of `docs/changelog/` for the planned date and contains the exact `## ` heading, a parser-safe MDX SPDX comment, the summary, and the detailed release bullets. An ordinary docs refresh or a post-tag Discussion draft does not satisfy this step. The release-prep docs PR, including the dated changelog entry, must be merged, or explicitly waived with a reason that names the missing changelog entry, before `release:plan` captures the release commit. -If a docs PR or any other intended PR merges after `release:plan`, regenerate the plan before cutting the tag. +Finish and merge the changelog before generating the 4 PM release plan. If another PR merges after `release:plan`, regenerate the plan before cutting the tag. -## Step 4: Cut the Tag and Publish Release Notes +## Step 4: Cut the Tag and Start Overnight Stabilization -Load `cut-release-tag`. -The version is already known, so use a patch bump unless the maintainer selects another bump. -Show the commit, changelog, carry-forward plan, label-retirement plan, and release notes draft. +Load `cut-release-tag` at 4 PM America/Los_Angeles. Let the agent verify the merged changelog and housekeeping, generate the plan for current `origin/main`, exercise the maintainer's local signer, and show the exact confirmation phrase. After maintainer confirmation, cut the signed tag regardless of E2E state. Let the workflow move `latest`, automatically carry stragglers to the next patch, and retire the released label. -After the release plan captures the candidate SHA, load `nemoclaw-maintainer-e2e`. -Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E and `Exact staging Brev Launchable` evidence. -Review the pre-tag E2E evidence ledger from `.github/workflows/e2e.yaml` at that commit. -Require the accepted workflow run to conclude with `success`. Require successful `Exact staging Brev Launchable` evidence with matching Launchable E2E identity and verified workspace absence, or record the permitted itemized exception described below. -Each missing or skipped execution in that successful run requires its own itemized maintainer exception. -Missing or invalid Launchable E2E evidence in that successful run requires a separate itemized exception with run and job URLs, the missing or invalid receipt, and rationale. -Do not ask for the release confirmation phrase until the run succeeds and each required execution has successful evidence or a permitted exception. +Every push to `main` already runs the complete E2E workflow. From 4 PM through 8 AM, keep merging normally while agents consolidate failures, remove redundant coverage, and fix broken or flaky E2Es. Do not delay or retry the tag because of those results. -Tag the confirmed release commit with `vX.Y.Z`. -Let the workflow move `latest`, carry open work forward, and delete the released label. -Prepare the Announcement draft for the maintainer to post. +Prepare the Announcement draft for the maintainer to post. Keep the signing key on the maintainer's workstation; do not add it to GitHub Actions. ## Step 5: Confirm and Share After the tag is cut and release notes are drafted or posted by the maintainer, present the final summary: - **Tag**: `v0.0.8` at commit `abc1234` -- **Pre-tag E2E evidence**: 12/13 tests and exact Brev Launchable E2E passing for the candidate SHA; 1 itemized maintainer exception +- **E2E stabilization**: links to remaining failed or flaky post-merge runs for overnight follow-up - **Release notes draft**: `../nemoclaw-release-v0.0.8/release-note-draft.md` - **Shipped**: 4 items (#1234, #1235, #1236, #1237) - **Moved to v0.0.9**: 1 item (#1238 — still needs CI fix) @@ -88,7 +78,7 @@ node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer ## Notes -- Never cut a tag or hand off release notes without user confirmation. +- Never cut a tag without the maintainer's exact confirmation phrase. E2E does not block that confirmation or the tag. - If nothing was labeled or nothing shipped, ask whether to skip the tag today. - A PR version label activates release work; it is not a readiness claim. - If an open item misses the tag, post-tag housekeeping moves its target to the next patch version. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md b/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md index 104df850c66..db953376ca0 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md @@ -68,7 +68,7 @@ Agents may recommend labels, assignments, Project field changes, comments, merge - A PR daily version label activates daily release work; it is not a readiness claim. - Release inclusion requires a PR to be both merged and carrying the relevant daily version label at release cutoff. - Issue daily version labels are tracking or coordination signals only. -- Before tag confirmation, capture the candidate SHA and review every E2E test declared by `.github/workflows/e2e.yaml` at that commit. Require the accepted workflow run's `head_sha` and all associated evidence to match the candidate SHA, and require the run to conclude with `success`. Each declared execution must have successful evidence from that run or an explicit itemized maintainer exception. Permit exceptions only for missing or skipped executions, not failed executions. If a late merge changes the candidate SHA, invalidate the prior run and evidence and repeat the complete workflow-wide validation. +- At 4 PM America/Los_Angeles, let the release agent verify the merged changelog and housekeeping, then have an authorized maintainer confirm and locally sign the current `main` tag regardless of E2E state. Keep merging while the 4 PM-8 AM loop triages each SHA-bound post-merge E2E run. - Open PRs and issues that miss a tagged release carry forward by automatically moving from the released version label to the next patch label after the tag and `latest` are verified. - After carry-forward leaves no open item on the released label, delete that repository label. Never rename or reuse it. - Durable release history belongs in releases, release notes, or manifests, not in long-lived labels. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 36f05b574cf..f4a6e90885d 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -31,7 +31,7 @@ If any merge lands after `release:plan`, generate a fresh plan before cutting th ## Cutoff -The daily cutoff is the maintainer-defined point where the release tag is prepared. +The daily cutoff is 4 PM America/Los_Angeles, when the release agent prepares the current `main` commit for an authorized maintainer to confirm and sign locally. Merging does not stop for cutoff, E2E, or overnight stabilization. At cutoff: @@ -40,39 +40,18 @@ At cutoff: 3. List open PRs and issues still carrying the target label as post-tag stragglers. 4. Confirm the merged release-note docs PR contains the dated changelog entry for the target version, or record an explicit waiver that names the missing entry. 5. Generate QA handoff from merged PRs. -6. Generate the release plan to capture the candidate commit. Merges may continue; a late drift check advances the candidate and invalidates evidence for the older SHA. -7. Review the candidate commit's pre-tag E2E evidence. -8. Cut the release tag only with explicit maintainer confirmation. -9. After the tag and workflow-managed `latest` are verified, automatically move every open straggler to the next patch label, verify none remain, and delete the released version label. - -## Pre-Tag E2E Evidence - -The release candidate is the full `origin/main` commit SHA captured by the generated release plan. At that commit, `.github/workflows/e2e.yaml` is the sole source of truth for the release E2E test set. Do not maintain a separate release-gating test list. - -Before asking for the release confirmation phrase, build and show an evidence ledger for that SHA: - -- Preflight the candidate workflow and existing candidate evidence before dispatching new work. -- Derive the denominator from the candidate workflow. Do not copy it into a second release test list. -- Require every declared `RELEASE_E2E_ACTIVATION_PATH` to exist at the candidate SHA. A missing path is a preflight failure. -- Require the workflow-produced trusted dispatch receipt to bind the accepted run candidate SHA, run ID, attempt, and selector inputs. -- Run `nemoclaw-maintainer-e2e` in full mode when the ledger lacks complete evidence for the candidate SHA. -- Require one completed, successful full workflow run for all default-selected workflow E2E jobs and the full-mode additions, including `Exact staging Brev Launchable`. -- Require the trusted dispatch receipt to record `allowJetsonRunnerQueue: false` and `allowDgxSparkRunnerQueue: false`. -- Exclude `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` from the required denominator. -- Require the trusted dispatch receipt to bind the workflow run and an attempt no later than the run's latest attempt. The receipt must record empty selectors and `include_staging_brev_launchable=true`. -- Require the Launchable E2E receipt to identify the candidate SHA in the repository and provision records. -- Require the cleanup receipt to identify the qualified workspace and report `ABSENT`. -- Every E2E execution selected by the accepted dispatch must have at least one completed, successful execution for the candidate SHA. -- Treat each expanded matrix execution as a separate ledger entry. Use its matrix `id`, or all distinguishing matrix dimensions when no single ID exists, in the test identifier so results for distinct expansions are never collapsed under the parent job. -- Successful evidence may accumulate across rerun attempts of that workflow run. Evidence from another workflow run does not satisfy the ledger. A later failure does not erase an earlier successful execution for the same test and SHA. -- Skipped, unexecuted, queued, in-progress, cancelled, and failing results do not count as successful evidence. -- Map each test with successful evidence to its successful run or job URL and attempt number. -- Each missing or skipped execution in the accepted successful workflow run requires its own itemized maintainer exception. Record the test identifier, relevant run links or available evidence, the current result, and the rationale. -- Missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run requires a separate itemized maintainer exception. Record the run and job URLs, the missing or invalid receipt, and the rationale. - -The accepted workflow run must be completed and have a `success` conclusion. A failed workflow run cannot supply the release ledger. Rerun its failed jobs until the workflow concludes with `success`. An itemized test exception applies only to a missing or skipped execution in that otherwise successful run. - -Each test and the exact Brev Launchable E2E job in the accepted successful workflow run must have successful evidence or its own permitted itemized exception before release confirmation. Immediately before confirmation, compare `origin/main` with the planned SHA. If the candidate SHA changes, discard the ledger and its exceptions, including Launchable E2E evidence. Regenerate the release plan and repeat the review for the new SHA. This does not freeze `main` or prevent merges. No release-note-only delta exception is currently defined. +6. Generate the release plan for current `origin/main`, exercise the maintainer's local signer, and show the exact confirmation phrase. If `main` moves before confirmation, regenerate the plan rather than stopping merges. +7. After explicit maintainer confirmation, cut the locally signed tag regardless of E2E state. Never put the release signing key in GitHub Actions or use a release bot to sign it. +8. After the tag and workflow-managed `latest` are verified, automatically move every open straggler to the next patch label, verify none remain, and delete the released version label. +9. From 4 PM through 8 AM, continue merging while agents consolidate failures, remove redundant coverage, and fix broken or flaky E2Es. + +## Asynchronous E2E Stabilization + +Every push to `main` starts the complete E2E workflow. Each run is bound to that push SHA, so a later merge does not cancel or replace the earlier result. + +E2E results are advisory release-health signals. They never block merging, select the release candidate, delay the 4 PM tag, or require a maintainer exception. Keep failed results attached to their workflow runs for asynchronous triage. + +From 4 PM through 8 AM, agents work the accumulated results methodically: group duplicate failures, remove redundant tests, repair broken or flaky tests, and merge fixes normally. At 8 AM, hand the remaining state to the next release doula. The daytime merge window continues from 8 AM through the next 4 PM tag. ## Carry Forward diff --git a/.agents/skills/nemoclaw-skills-guide/SKILL.md b/.agents/skills/nemoclaw-skills-guide/SKILL.md index e604761401f..c1a1a24d680 100644 --- a/.agents/skills/nemoclaw-skills-guide/SKILL.md +++ b/.agents/skills/nemoclaw-skills-guide/SKILL.md @@ -62,7 +62,7 @@ Component-specific guidance lives with the package it describes, not in a skill. | `nemoclaw-maintainer-cross-issue-sweep` | Find open issues that a PR can also fix or conflict with. Report file and line evidence. | | `nemoclaw-maintainer-day` | Run one daytime maintainer pass for the release version. Select a merge, salvage, security, test, conflict, or sequencing workflow. Designed for `/loop`. | | `nemoclaw-maintainer-evening` | End-of-day handoff: require the pre-tag dated changelog PR, check version progress, identify stragglers, generate a QA handoff summary, cut the release tag, carry stragglers forward, retire the released label, and hand off the Announcement. | -| `nemoclaw-maintainer-cut-release-tag` | Verify the dated changelog entry, cut an annotated semver tag on a maintainer-confirmed `origin/main` commit, wait for workflow-managed `latest`, carry remaining open items forward, and delete the released label; `lkg` stays manual. | +| `nemoclaw-maintainer-cut-release-tag` | At 4 PM, verify the dated changelog and housekeeping, prepare a maintainer-confirmed tag for local signing independent of E2E, wait for workflow-managed `latest`, carry remaining open items forward, and delete the released label; `lkg` stays manual. | | `nemoclaw-maintainer-e2e` | Describe default E2E triggered by pushes to `main`, dispatch exact-revision manual PR E2E, and verify applicable workflow evidence. | | `nemoclaw-maintainer-release-notes` | Draft the post-tag Announcement from live tag/compare data, with the three-paragraph narrative, categorized change list, and external-only contributor thanks. | | `nemoclaw-maintainer-find-review-pr` | Find open security PRs with Urgent or High Project Priority. Link each PR to its issue and identify competing PRs. | diff --git a/package.json b/package.json index 6de43ffcf79..0aa004fc3a7 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,6 @@ "test-conditionals:scan": "tsx scripts/find-test-conditionals.mts", "bump:version": "tsx scripts/bump-version.mts", "release:plan": "tsx scripts/release-plan.mts", - "release:e2e-evidence": "node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts", "release:cut": "bash scripts/release-cut-tag.sh", "release:wait-latest": "bash scripts/release-wait-latest.sh", "release:notes-data": "tsx scripts/release-notes-data.mts", diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index b425c2f5e80..6e222bbdd40 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -314,7 +314,7 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).not.toMatch(/variable (?:set|delete) NEMOCLAW_BREV_LAUNCHABLE_E2E_ENABLED/u); }); - it("binds dispatch, evidence, invalidation, and release handoff to one SHA (#7487)", () => { + it("binds dispatch and diagnostic evidence to one SHA without gating merges or tags (#7487)", () => { expect(skill).toContain("git rev-parse origin/main"); expect(skill).toContain("correlation_id=${CORRELATION_ID}"); expect(skill).toContain("head_sha"); @@ -323,8 +323,8 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).toContain("cleanup.json"); expect(skill).toContain("dispatch.json"); expect(skill).toContain("validate-full-e2e-evidence.mts"); - expect(skill).toContain("provisional release evidence"); - expect(skill).toContain("If the release candidate SHA changes"); - expect(skill).toContain("nemoclaw-maintainer-cut-release-tag"); + expect(skill).toContain("SHA-bound diagnostic evidence"); + expect(skill).toContain("When validating another SHA"); + expect(skill).toContain("does not gate merging or a release tag"); }); }); diff --git a/test/maintainer-skills-policy.test.ts b/test/maintainer-skills-policy.test.ts index 05616d6ac30..3e8364346f3 100644 --- a/test/maintainer-skills-policy.test.ts +++ b/test/maintainer-skills-policy.test.ts @@ -185,123 +185,33 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(taxonomy.quality_rules.post_merge_untagged_release_labeling_allowed).toBe(true); }); - it("requires E2E evidence for the release candidate commit or itemized maintainer exceptions", () => { + it("keeps the semi-automatic 4 PM tag and merging independent from asynchronous E2E stabilization", () => { const dailyFlow = read(".agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md"); const evening = read(".agents/skills/nemoclaw-maintainer-evening/SKILL.md"); const priorities = read(".agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md"); const release = read(".agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md"); const policy = read(".agents/skills/nemoclaw-maintainer-policies/references/release-train.md"); - expect(policy).toContain("full `origin/main` commit SHA"); - expect(policy).toContain("`.github/workflows/e2e.yaml` is the sole source of truth"); - expect(policy).toContain("Do not maintain a separate release-gating test list"); - expect(policy).toContain("at least one completed, successful execution"); - expect(policy).toContain("Successful evidence may accumulate across rerun attempts"); - expect(policy).toContain("Evidence from another workflow run does not satisfy the ledger"); - expect(policy).toContain("Require every declared `RELEASE_E2E_ACTIVATION_PATH`"); - expect(policy).toContain("A missing path is a preflight failure"); - expect(release).toContain("Each job that declares `RELEASE_E2E_ACTIVATION_PATH`"); - expect(release).toContain("A missing activation path is a preflight failure"); - expect(policy).toContain("each expanded matrix execution as a separate ledger entry"); - expect(policy).toContain("matrix `id`"); - expect(policy).toContain("A later failure does not erase an earlier successful execution"); - expect(policy).toContain( - "Skipped, unexecuted, queued, in-progress, cancelled, and failing results do not count as successful evidence", - ); - expect(policy).toContain("itemized maintainer exception"); - expect(policy).toContain("If the candidate SHA changes"); - expect(policy).toContain("This does not freeze `main` or prevent merges"); - expect(policy).toContain("Require one completed, successful full workflow run"); - expect(policy).toContain("discard the ledger and its exceptions"); - expect(policy).toContain("selector inputs"); - expect(release).toContain('"dispatchJson"'); - expect(release).toContain("the number of tests with successful evidence"); - expect(release).toContain("successful run or job URL and attempt"); - expect(release).toContain("npm run release:e2e-evidence"); - expect(release).toContain("filter=all"); - expect(release).toContain("actions/runs/$RUN_ID/artifacts"); - expect(release).toContain("sort_by(.created_at)"); - expect(release).not.toContain("RECEIPT_ATTEMPT"); - expect(release).toContain("rerun preflight and the full E2E workflow"); - expect(release).toContain("Immediately before asking, refresh `origin/main` once"); - const evidenceSummary = release.indexOf("Before showing the confirmation prompt"); - const confirmationPrompt = release.indexOf( - "Ask the maintainer to paste this phrase", - evidenceSummary, - ); - expect(evidenceSummary).toBeGreaterThanOrEqual(0); - expect(evidenceSummary).toBeLessThan(confirmationPrompt); - expect(evening).toContain( - "Each missing or skipped execution in that successful run requires its own itemized maintainer exception", - ); - expect(evening).toContain( - "Missing or invalid Launchable E2E evidence in that successful run requires a separate", - ); - expect(evening).toContain("Tag the confirmed release commit with `vX.Y.Z`"); - expect(evening).not.toContain("tag `main`"); - expect(dailyFlow).toContain("capture the candidate SHA and review every E2E test"); - expect(dailyFlow).toContain( - "`head_sha` and all associated evidence to match the candidate SHA", - ); - expect(dailyFlow).toContain("invalidate the prior run and evidence"); - expect(priorities).toContain("Record the release SHA and required E2E evidence"); - }); - - it("requires full-mode exact Brev Launchable evidence before release confirmation (#7487)", () => { - const e2e = read(".agents/skills/nemoclaw-maintainer-e2e/SKILL.md"); - const evening = read(".agents/skills/nemoclaw-maintainer-evening/SKILL.md"); - const release = read(".agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md"); - const policy = read(".agents/skills/nemoclaw-maintainer-policies/references/release-train.md"); - const skillsGuide = read(".agents/skills/nemoclaw-skills-guide/SKILL.md"); - - expect(e2e).toContain("include_staging_brev_launchable=true"); - expect(e2e).toContain("Exact staging Brev Launchable"); - expect(e2e).toContain("launchable-e2e.json"); - expect(e2e).toContain("cleanup.json"); - expect(e2e).toContain("dispatch.json"); - expect(e2e).toContain("If the release candidate SHA changes"); - expect(e2e).toContain("jobs?filter=all&per_page=100"); - expect(e2e).toContain("Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json`"); - expect(release).toContain("reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json`"); - expect(release).toContain("load `nemoclaw-maintainer-e2e` and dispatch one full run"); - expect(release).toContain("Treat a skipped job as missing evidence"); - expect(release).toContain("include_staging_brev_launchable=true"); - expect(release).toContain("cleanup evidence that reports the qualified workspace as `ABSENT`"); - expect(release).toContain( - "a separate itemized maintainer exception for each missing or skipped execution", - ); - expect(release).toContain( - "a separate itemized maintainer exception for missing or invalid exact Brev Launchable E2E evidence", - ); - expect(release).toContain("when accepted full-mode exact Brev evidence exists"); - expect( - release.indexOf("load `nemoclaw-maintainer-e2e` and dispatch one full run"), - ).toBeLessThan(release.indexOf("Ask the maintainer to paste this phrase")); - expect(evening).toContain("load `nemoclaw-maintainer-e2e`"); - expect(evening).toContain( - "Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E", - ); - expect(release).toContain( - "Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E", - ); - expect(policy).toContain("A failed workflow run cannot supply the release ledger"); - expect(release).toContain("Reject a failed workflow run before presenting the ledger"); - expect(evening).not.toContain("readiness variable"); - expect(policy).toContain("Require one completed, successful full workflow run"); - expect(policy).toContain( - "Run `nemoclaw-maintainer-e2e` in full mode when the ledger lacks complete evidence", - ); - expect(policy).toContain("including `Exact staging Brev Launchable`"); - expect(policy).toContain("cleanup receipt"); - expect(policy).toContain("trusted dispatch receipt"); - expect(policy).toContain( - "Each missing or skipped execution in the accepted successful workflow run", - ); - expect(policy).toContain( - "Missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run", - ); - expect(policy).toContain("No release-note-only delta exception is currently defined"); - expect(skillsGuide).toContain("`nemoclaw-maintainer-e2e`"); + expect(policy).toContain("4 PM America/Los_Angeles"); + expect(policy).toContain("Every push to `main` starts the complete E2E workflow"); + expect(policy).toContain("They never block merging"); + expect(policy).toContain("From 4 PM through 8 AM"); + expect(policy).toContain("At 8 AM, hand the remaining state"); + expect(policy).toContain("authorized maintainer to confirm and sign locally"); + expect(policy).toContain("Never put the release signing key in GitHub Actions"); + expect(release).toContain("This is semi-automatic"); + expect(release).toContain("Do not consult E2E state before cutting the tag"); + expect(release).toContain("Never put the release signing key in a GitHub Actions secret"); + expect(release).toContain("npm run release:cut -- --plan --preflight-only"); + expect(release).toContain("CONFIRM RELEASE vX.Y.Z "); + expect(release).not.toContain("release:e2e-evidence"); + expect(release).not.toContain("release-daily-tag"); + expect(release).not.toContain("NEMOCLAW_RELEASE_TAG_SIGNING_KEY"); + expect(evening).toContain("keep merging normally"); + expect(evening).toContain("Do not delay or retry the tag"); + expect(evening).toContain("maintainer's local signer"); + expect(dailyFlow).toContain("regardless of E2E state"); + expect(priorities).toContain("E2E does not block the tag or merging"); }); it("runs release-prep docs before generating the final release plan", () => { @@ -335,7 +245,15 @@ describe("maintainer skills follow canonical workflow policy", () => { evening.indexOf("Load `cut-release-tag`"), ); expect(evening).toContain("contains the exact `## ` heading"); - expect(release).toContain("git grep -n '^## vX\\.Y\\.Z$'"); + expect(release).toContain('NEXT_TAG=""'); + expect(release).toContain('[[ "$NEXT_TAG" =~ ^v[0-9]+\\.[0-9]+\\.[0-9]+$ ]]'); + expect(release).toContain('git grep -n -E "^## ${NEXT_TAG//.'); + expect(release).toContain( + "':(glob)docs/changelog/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].mdx'", + ); + expect(release).toContain('CHANGELOG_MATCH_COUNT="$(printf'); + expect(release).toContain('[[ "$CHANGELOG_MATCH_COUNT" != "1" ]]'); + expect(release).toContain("Expected exactly one dated changelog heading"); expect(release).toContain("Unless Step 1 records an explicit waiver"); expect(release).toContain("show the recorded waiver reason"); expect(release).toContain("A conventional Release Notes page or post-tag Announcement draft"); diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts deleted file mode 100644 index f5aef666837..00000000000 --- a/test/release-e2e-evidence.test.ts +++ /dev/null @@ -1,401 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - buildReleaseE2eLedger, - buildReleaseE2ePreflight, - type ReleaseE2eExecution, - type ReleaseE2ePreflight, - type ReleaseE2eRunEvidence, -} from "../.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts"; - -const candidateSha = "a".repeat(40); -const workflowSha = "b".repeat(40); - -function preflight( - input: { candidatePathExists?: (candidateSha: string, candidatePath: string) => boolean } = {}, -) { - return buildReleaseE2ePreflight({ - candidateSha, - candidatePathExists: input.candidatePathExists ?? (() => true), - }); -} - -function runEvidence( - plan: ReleaseE2ePreflight, - group: ReleaseE2eExecution["group"], - options: { - attempt?: number; - conclusion?: (execution: ReleaseE2eExecution) => string; - only?: (execution: ReleaseE2eExecution) => boolean; - receiptVersion?: 1 | 2; - sha?: string; - status?: (execution: ReleaseE2eExecution) => string; - } = {}, -): ReleaseE2eRunEvidence { - const attempt = options.attempt ?? 1; - const runId = 1001; - const receiptVersion = options.receiptVersion ?? 1; - const executions = plan.executions.filter( - (execution) => execution.group === group && (options.only?.(execution) ?? true), - ); - const selectors: string[] = []; - return { - dispatch: { - allowDgxSparkRunnerQueue: false, - allowJetsonRunnerQueue: false, - ...(receiptVersion === 2 - ? { - baseSha: "c".repeat(40), - candidateRepository: "NVIDIA/NemoClaw", - prNumber: 8583, - repository: "NVIDIA/NemoClaw", - workflowSha, - } - : {}), - candidateSha, - emptySelectors: true, - eventName: "workflow_dispatch", - includeStagingBrevLaunchable: plan.dispatches.completeRun.includeStagingBrevLaunchable, - jobs: selectors.join(","), - kind: `nemoclaw-e2e-dispatch-v${receiptVersion}`, - targets: "", - workflowRunAttempt: attempt, - workflowRunId: String(runId), - }, - jobs: { - jobs: executions.map((execution, index) => ({ - conclusion: options.conclusion?.(execution) ?? "success", - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${runId}/job/${index + 1}`, - name: execution.expectedName, - run_attempt: attempt, - run_id: runId, - status: options.status?.(execution) ?? "completed", - })), - }, - run: { - event: "workflow_dispatch", - head_branch: "main", - id: runId, - path: ".github/workflows/e2e.yaml", - status: "completed", - conclusion: "success", - run_attempt: attempt, - head_sha: options.sha ?? (receiptVersion === 2 ? workflowSha : candidateSha), - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${runId}`, - }, - }; -} - -function directMainV2Evidence( - plan: ReleaseE2ePreflight, - group: ReleaseE2eExecution["group"], -): ReleaseE2eRunEvidence { - const evidence = runEvidence(plan, group, { receiptVersion: 2 }); - Object.assign(evidence.dispatch as Record, { - baseSha: candidateSha, - candidateRepository: "NVIDIA/NemoClaw", - prNumber: null, - workflowSha: candidateSha, - }); - (evidence.run as Record).head_sha = candidateSha; - return evidence; -} - -describe("release E2E evidence", () => { - it("derives one complete release E2E run from the workflow", () => { - const plan = preflight(); - - expect(plan.dispatches.completeRun).toEqual({ - includeStagingBrevLaunchable: true, - jobs: "", - mode: "full", - targets: "", - }); - expect(plan.launchableE2eJobId).toBe("staging-brev-launchable"); - expect(plan.exceptionsRequired).toEqual([]); - expect(plan.executions.map((execution) => execution.jobId)).not.toEqual( - expect.arrayContaining([ - "jetson-nvmap-gpu", - "llama-cpp-dgx-spark-plan", - "llama-cpp-dgx-spark-qualification", - ]), - ); - }); - - it("accepts v2 evidence bound to the candidate and trusted workflow SHAs", () => { - const plan = preflight(); - const ledger = buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { receiptVersion: 2 }), - ]); - - expect(ledger.successfulCount).toBe(ledger.requiredCount); - expect(ledger.missingCount).toBe(0); - }); - - it("accepts direct-main v2 evidence with identical repository and SHA identities", () => { - const plan = preflight(); - const ledger = buildReleaseE2eLedger(plan, [directMainV2Evidence(plan, "default")]); - - expect(ledger.successfulCount).toBe(ledger.requiredCount); - expect(ledger.missingCount).toBe(0); - }); - - it.each([ - ["candidateRepository", "contributor/NemoClaw", "runs[0].dispatch.candidateRepository"], - ["baseSha", "c".repeat(40), "runs[0].dispatch.baseSha"], - ["workflowSha", workflowSha, "runs[0].dispatch.workflowSha"], - ])("rejects direct-main v2 evidence with a mismatched %s identity", (field, value, message) => { - const plan = preflight(); - const evidence = directMainV2Evidence(plan, "default"); - (evidence.dispatch as Record)[field] = value; - - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); - }); - - it.each([ - ["repository", "other/NemoClaw", "runs[0].dispatch.repository"], - ["prNumber", 0, "runs[0].dispatch.prNumber"], - ["candidateRepository", "not-a-repository", "runs[0].dispatch.candidateRepository"], - ["candidateSha", "d".repeat(40), "runs[0].dispatch.candidateSha"], - ["baseSha", "short", "runs[0].dispatch.baseSha"], - ["workflowSha", "short", "runs[0].dispatch.workflowSha"], - ])("rejects v2 evidence with a mismatched %s", (field, value, message) => { - const plan = preflight(); - const evidence = runEvidence(plan, "default", { receiptVersion: 2 }); - (evidence.dispatch as Record)[field] = value; - - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); - }); - - it("rejects a v2 run whose head is the candidate instead of the trusted workflow", () => { - const plan = preflight(); - - expect(() => - buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { receiptVersion: 2, sha: candidateSha }), - ]), - ).toThrow("runs[0].run.head_sha must equal"); - }); - - it("rejects a missing activation path for a default E2E", () => { - expect(() => - preflight({ - candidatePathExists: (_sha, candidatePath) => - candidatePath !== "ci/protected-managed-image-multiarch-activation-v1.json", - }), - ).toThrow( - "candidate commit is missing required E2E activation path ci/protected-managed-image-multiarch-activation-v1.json for managed-image-multiarch-startup", - ); - }); - - it("rejects an in-progress workflow with successful execution jobs", () => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - (evidence.run as Record).status = "in_progress"; - - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( - 'runs[0].run.status must equal "completed"', - ); - }); - - it("rejects a failed workflow with successful execution jobs", () => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - (evidence.run as Record).conclusion = "failure"; - - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( - 'runs[0].run.conclusion must equal "success"', - ); - }); - - it("fails when the candidate commit cannot be inspected for activation paths", () => { - expect(() => - buildReleaseE2ePreflight({ - candidateSha: "0".repeat(40), - }), - ).toThrow("could not inspect release E2E activation path"); - }); - - it("keeps every static and dynamic matrix row as a distinct execution", () => { - const plan = preflight(); - const ids = plan.executions.map((execution) => execution.id); - - expect(ids.filter((id) => id.startsWith("mcp-bridge-dev["))).toHaveLength(3); - expect(ids.filter((id) => id.startsWith("hermes-gpu-startup["))).toHaveLength(3); - expect(ids.filter((id) => id.startsWith("openshell-gateway-upgrade["))).toHaveLength(5); - expect(ids).toContain("live[id=ubuntu-repo-cloud-openclaw]"); - expect(ids).toContain("shared-e2e[id=vllm-docker-storage]"); - expect(new Set(ids).size).toBe(ids.length); - }); - - it("accumulates successful evidence across rerun attempts", () => { - const plan = preflight(); - const firstDefaultRun = runEvidence(plan, "default"); - const laterFailure = runEvidence(plan, "default", { - attempt: 2, - conclusion: () => "failure", - only: (execution) => execution.id === "snapshot-commands", - }); - (firstDefaultRun.run as Record).run_attempt = 2; - (firstDefaultRun.jobs as { jobs: unknown[] }).jobs.push( - ...(laterFailure.jobs as { jobs: unknown[] }).jobs, - ); - const ledger = buildReleaseE2eLedger(plan, [firstDefaultRun]); - - expect(ledger.successfulCount).toBe(ledger.requiredCount); - expect(ledger.missingCount).toBe(0); - expect(ledger.entries.find((entry) => entry.id === "snapshot-commands")).toMatchObject({ - attempts: [ - { attempt: 2, conclusion: "failure" }, - { attempt: 1, conclusion: "success" }, - ], - successfulEvidence: { attempt: 1 }, - status: "successful", - }); - }); - - it("rejects evidence assembled from multiple workflow runs", () => { - const plan = preflight(); - expect(() => - buildReleaseE2eLedger(plan, [runEvidence(plan, "default"), runEvidence(plan, "default")]), - ).toThrow("release E2E evidence requires exactly one workflow run, received 2"); - }); - - it("requires the full run to include staging Brev Launchable", () => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - (evidence.dispatch as Record).includeStagingBrevLaunchable = false; - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( - "runs[0].dispatch.includeStagingBrevLaunchable must equal true", - ); - }); - - it.each([ - ["allowJetsonRunnerQueue", "runs[0].dispatch.allowJetsonRunnerQueue must equal false"], - ["allowDgxSparkRunnerQueue", "runs[0].dispatch.allowDgxSparkRunnerQueue must equal false"], - ])("rejects release evidence that opts into %s", (field, message) => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - (evidence.dispatch as Record)[field] = true; - - expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); - }); - - it("reports a failed matrix row without collapsing its successful siblings", () => { - const plan = preflight(); - const failedId = 'hermes-gpu-startup[scenario="fallback",sandbox_name="e2e-hgpu-fallback"]'; - const ledger = buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { - conclusion: (execution) => (execution.id === failedId ? "failure" : "success"), - }), - ]); - - expect(ledger.missingCount).toBe(1); - expect(ledger.entries.find((entry) => entry.id === failedId)).toMatchObject({ - status: "missing", - attempts: [{ conclusion: "failure" }], - }); - expect( - ledger.entries.find( - (entry) => - entry.id === - 'hermes-gpu-startup[scenario="compatibility-only",sandbox_name="e2e-hgpu-compat"]', - ), - ).toMatchObject({ status: "successful" }); - }); - - it("does not count an in-progress execution as successful", () => { - const plan = preflight(); - const pendingId = "snapshot-commands"; - const ledger = buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { - status: (execution) => (execution.id === pendingId ? "in_progress" : "completed"), - }), - ]); - - expect(ledger.missingCount).toBe(1); - expect(ledger.entries.find((entry) => entry.id === pendingId)).toMatchObject({ - attempts: [{ conclusion: "success", status: "in_progress" }], - status: "missing", - }); - }); - - it("does not treat a skipped execution as successful evidence", () => { - const plan = preflight(); - const skippedId = "snapshot-commands"; - const ledger = buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { - conclusion: (execution) => (execution.id === skippedId ? "skipped" : "success"), - }), - ]); - - expect(ledger.entries.find((entry) => entry.id === skippedId)).toMatchObject({ - attempts: [{ conclusion: "skipped", status: "completed" }], - status: "missing", - }); - }); - - it("ignores job evidence from another workflow run", () => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - const ignoredId = plan.executions.find((execution) => execution.group === "default")!.id; - const jobs = evidence.jobs as { jobs: Array> }; - jobs.jobs[0]!.run_id = 999; - - const ledger = buildReleaseE2eLedger(plan, [evidence]); - - expect(ledger.entries.find((entry) => entry.id === ignoredId)).toMatchObject({ - attempts: [], - status: "missing", - }); - }); - - it("ignores job evidence newer than the enclosing workflow run attempt", () => { - const plan = preflight(); - const evidence = runEvidence(plan, "default"); - const ignoredId = plan.executions.find((execution) => execution.group === "default")!.id; - const jobs = evidence.jobs as { jobs: Array> }; - jobs.jobs[0]!.run_attempt = 2; - - const ledger = buildReleaseE2eLedger(plan, [evidence]); - - expect(ledger.entries.find((entry) => entry.id === ignoredId)).toMatchObject({ - attempts: [], - status: "missing", - }); - }); - - it("rejects malformed job evidence", () => { - const plan = preflight(); - const malformed = runEvidence(plan, "default"); - const jobs = malformed.jobs as { jobs: Array> }; - delete jobs.jobs[0]!.name; - - expect(() => buildReleaseE2eLedger(plan, [malformed])).toThrow( - "runs[0].job.name must be a non-empty string", - ); - }); - - it("rejects a selective dispatch receipt that claims empty selectors", () => { - const plan = preflight(); - const selective = runEvidence(plan, "default"); - const dispatch = selective.dispatch as Record; - dispatch.jobs = "snapshot-commands"; - dispatch.emptySelectors = true; - - expect(() => buildReleaseE2eLedger(plan, [selective])).toThrow( - 'runs[0].dispatch.jobs must equal ""', - ); - }); - - it("rejects evidence from another candidate SHA", () => { - const plan = preflight(); - - expect(() => - buildReleaseE2eLedger(plan, [runEvidence(plan, "default", { sha: "b".repeat(40) })]), - ).toThrow("runs[0].run.head_sha must equal"); - }); -});