Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion .github/workflows/e2e-standard-profile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ on:
restore_cli:
required: true
type: boolean
cloudflared:
required: true
type: boolean
host_packages:
required: true
type: string
Expand All @@ -66,6 +69,9 @@ on:
compatible_api_key:
required: true
type: boolean
github_token:
required: true
type: boolean
shard:
required: true
type: string
Expand All @@ -84,6 +90,8 @@ on:
required: false
NVIDIA_INFERENCE_API_KEY:
required: false
BRAVE_API_KEY:
required: false

permissions:
contents: read
Expand Down Expand Up @@ -374,6 +382,27 @@ jobs:
with:
provenance-json: ${{ inputs.cli_artifact_provenance }}

- name: Install reviewed cloudflared
if: ${{ inputs.cloudflared }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"

- name: Add swap for Hermes image rebuild
if: ${{ inputs.host_preparation == 'rebuild-swap' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
Expand Down Expand Up @@ -493,7 +522,7 @@ jobs:
run: |
set -euo pipefail
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
-u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN \
-u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u BRAVE_API_KEY -u GITHUB_TOKEN \
-u COMPATIBLE_API_KEY bash scripts/install-openshell.sh

- name: Run catalogue E2E target
Expand All @@ -504,6 +533,8 @@ jobs:
NVIDIA_API_KEY: ${{ inputs.trusted_main && secrets.NVIDIA_API_KEY || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ inputs.trusted_main && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
COMPATIBLE_API_KEY: ${{ inputs.compatible_api_key && inputs.trusted_main && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
BRAVE_API_KEY: ${{ inputs.trusted_main && secrets.BRAVE_API_KEY || '' }}
GITHUB_TOKEN: ${{ inputs.github_token && inputs.trusted_main && github.token || '' }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
Expand Down
618 changes: 91 additions & 527 deletions .github/workflows/e2e.yaml

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion ci/source-architecture-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
},
"allowedCycles": [],
"maxRootFiles": {
"src/lib/onboard": 309,
"src/lib/onboard": 308,
"src/lib/actions": 19,
"src/lib/actions/sandbox": 183,
"src/lib/state": 38,
Expand Down
5 changes: 0 additions & 5 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -201,11 +201,6 @@
"test": "builds the policy boundary before semantic collection and CLI compilation",
"category": "compatibility"
},
{
"file": "test/e2e/support/e2e-workflow.test.ts",
"test": "rejects credential-backed provider smokes in the PR-safe inference-routing job",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "keeps every candidate on standard runners when $name (#7145)",
Expand Down
48 changes: 27 additions & 21 deletions scripts/checks/check-cloudflared-update.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,23 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# invalidState: the five reviewed E2E consumers drift to different cloudflared
# invalidState: the four reviewed E2E consumers drift to different cloudflared
# versions/digests, or their shared pin no longer matches the upstream asset.
# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all five
# sourceBoundary: Cloudflare owns the release asset; NemoClaw owns all four
# workflow pins and independently verifies the downloaded bytes.
# whyNotSourceFix: upstream cannot enforce which release NemoClaw workflows use.
# regressionTest: cloudflared-update-check-workflow.test.ts covers five-pin
# regressionTest: cloudflared-update-check-workflow.test.ts covers four-pin
# parity, asset URL identity, digest mismatch, and update instructions.
# removalCondition: remove this checker when the five consumers share one
# removalCondition: remove this checker when the four consumers share one
# machine-readable dependency manifest with equivalent live asset verification.

set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
E2E_WORKFLOW="${CLOUDFLARED_E2E_WORKFLOW:-${REPO_ROOT}/.github/workflows/e2e.yaml}"
PROFILE_WORKFLOW="${CLOUDFLARED_PROFILE_WORKFLOW:-${REPO_ROOT}/.github/workflows/e2e-standard-profile.yaml}"
PIN_SOURCES=("${E2E_WORKFLOW}" "${PROFILE_WORKFLOW}")
RELEASE_API_URL="${CLOUDFLARED_RELEASE_API_URL:-https://api.github.com/repos/cloudflare/cloudflared/releases/latest}"
DOWNLOAD_BASE_URL="${CLOUDFLARED_DOWNLOAD_BASE_URL:-https://github.com/cloudflare/cloudflared/releases/download}"
CURL_BIN="${CLOUDFLARED_CURL_BIN:-curl}"
Expand All @@ -30,38 +32,40 @@ fail() {
for tool in "${CURL_BIN}" jq "${SHA256SUM_BIN}"; do
command -v "${tool}" >/dev/null 2>&1 || fail "required tool is unavailable: ${tool}"
done
[[ -r "${E2E_WORKFLOW}" ]] || fail "cannot read pin source: ${E2E_WORKFLOW}"
for source in "${PIN_SOURCES[@]}"; do
[[ -r "${source}" ]] || fail "cannot read pin source: ${source}"
done

version_pins=()
while IFS= read -r pin || [[ -n "${pin}" ]]; do
version_pins+=("${pin}")
done < <(
sed -nE 's/^[[:space:]]*CLOUDFLARED_VERSION:[[:space:]]*"([^"]+)".*$/\1/p' \
"${E2E_WORKFLOW}"
"${PIN_SOURCES[@]}"
)

sha_pins=()
while IFS= read -r pin || [[ -n "${pin}" ]]; do
sha_pins+=("${pin}")
done < <(
sed -nE 's/^[[:space:]]*CLOUDFLARED_DEB_SHA256:[[:space:]]*"([0-9a-fA-F]+)".*$/\1/p' \
"${E2E_WORKFLOW}"
"${PIN_SOURCES[@]}"
)

[[ "${#version_pins[@]}" -eq 5 ]] \
|| fail "expected exactly five CLOUDFLARED_VERSION pins in ${E2E_WORKFLOW}; found ${#version_pins[@]}"
[[ "${#sha_pins[@]}" -eq 5 ]] \
|| fail "expected exactly five CLOUDFLARED_DEB_SHA256 pins in ${E2E_WORKFLOW}; found ${#sha_pins[@]}"
[[ "${#version_pins[@]}" -eq 4 ]] \
|| fail "expected exactly four CLOUDFLARED_VERSION pins; found ${#version_pins[@]}"
[[ "${#sha_pins[@]}" -eq 4 ]] \
|| fail "expected exactly four CLOUDFLARED_DEB_SHA256 pins; found ${#sha_pins[@]}"

pinned_version="${version_pins[0]}"
pinned_sha="$(printf '%s' "${sha_pins[0]}" | tr '[:upper:]' '[:lower:]')"
for pin in "${version_pins[@]}"; do
[[ "${pin}" == "${pinned_version}" ]] \
|| fail "CLOUDFLARED_VERSION pins diverge in ${E2E_WORKFLOW}: ${version_pins[*]}"
|| fail "CLOUDFLARED_VERSION pins diverge: ${version_pins[*]}"
done
for pin in "${sha_pins[@]}"; do
[[ "$(printf '%s' "${pin}" | tr '[:upper:]' '[:lower:]')" == "${pinned_sha}" ]] \
|| fail "CLOUDFLARED_DEB_SHA256 pins diverge in ${E2E_WORKFLOW}: ${sha_pins[*]}"
|| fail "CLOUDFLARED_DEB_SHA256 pins diverge: ${sha_pins[*]}"
done
[[ "${pinned_version}" =~ ^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$ ]] \
|| fail "invalid pinned cloudflared version: ${pinned_version}"
Expand Down Expand Up @@ -113,21 +117,23 @@ expected_asset_url="${DOWNLOAD_BASE_URL%/}/${latest_version}/cloudflared-linux-a
latest_sha="$("${SHA256SUM_BIN}" "${cloudflared_deb}" | awk '{print tolower($1)}')"
[[ "${latest_sha}" =~ ^[0-9a-f]{64}$ ]] || fail "could not compute the latest asset SHA256"

version_lines="$(grep -n 'CLOUDFLARED_VERSION:' "${E2E_WORKFLOW}" | cut -d: -f1 | paste -sd, -)"
sha_lines="$(grep -n 'CLOUDFLARED_DEB_SHA256:' "${E2E_WORKFLOW}" | cut -d: -f1 | paste -sd, -)"
workflow_display="${E2E_WORKFLOW#"${REPO_ROOT}/"}"

print_update_instructions() {
printf '%s\n' \
'cloudflared update required.' \
"Pinned version: ${pinned_version}" \
"Pinned linux-amd64.deb SHA256: ${pinned_sha}" \
"Latest version: ${latest_version}" \
"Latest linux-amd64.deb SHA256: ${latest_sha}" \
'Update locations:' \
" ${workflow_display} CLOUDFLARED_VERSION lines: ${version_lines}" \
" ${workflow_display} CLOUDFLARED_DEB_SHA256 lines: ${sha_lines}" \
'Set all five version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2
'Update locations:' >&2
for source in "${PIN_SOURCES[@]}"; do
workflow_display="${source#"${REPO_ROOT}/"}"
version_lines="$(grep -n 'CLOUDFLARED_VERSION:' "${source}" | cut -d: -f1 | paste -sd, -)"
sha_lines="$(grep -n 'CLOUDFLARED_DEB_SHA256:' "${source}" | cut -d: -f1 | paste -sd, -)"
printf ' %s CLOUDFLARED_VERSION lines: %s\n' "${workflow_display}" "${version_lines}" >&2
printf ' %s CLOUDFLARED_DEB_SHA256 lines: %s\n' "${workflow_display}" "${sha_lines}" >&2
done
printf '%s\n' \
'Set all four version/SHA256 pairs to the latest reviewed values, then rerun this check.' >&2
}

if [[ "${latest_version}" != "${pinned_version}" ]]; then
Expand Down
1 change: 0 additions & 1 deletion scripts/checks/verify-openshell-e2e-qualification.mts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ const SENSITIVE_EXACT_PATHS = new Set([
"src/lib/inference/serving/managed-runtime-receipts.ts",
"src/lib/onboard/gateway-host-runtime.ts",
"test/openshell-e2e-qualification.test.ts",
"tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts",
]);

const SENSITIVE_PREFIXES = [
Expand Down
48 changes: 27 additions & 21 deletions test/cloudflared-update-check-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import { describe, expect, it } from "vitest";
import { readYaml, type WorkflowStep } from "./helpers/e2e-workflow-contract";

const ROOT = path.resolve(import.meta.dirname, "..");
const E2E_WORKFLOW = path.join(ROOT, ".github", "workflows", "e2e.yaml");
const CHECK_SCRIPT = path.join(ROOT, "scripts", "checks", "check-cloudflared-update.sh");
const FULL_SHA_ACTION = /@[0-9a-f]{40}$/iu;

Expand All @@ -31,16 +30,10 @@ type CloudflaredUpdateWorkflow = {
>;
};

function pinValues(source: string, name: string): string[] {
return [...source.matchAll(new RegExp(`^\\s*${name}:\\s*"([^"]+)"`, "gmu"))].map(
(match) => match[1],
);
}

function writePinFixture(file: string, version: string, sha256: string): void {
function writePinFixture(file: string, version: string, sha256: string, count: number): void {
fs.writeFileSync(
file,
["one", "two", "three", "four", "five"]
Array.from({ length: count }, (_, index) => `consumer-${index + 1}`)
.map(
(job) =>
` ${job}:\n env:\n CLOUDFLARED_VERSION: "${version}"\n CLOUDFLARED_DEB_SHA256: "${sha256}"`,
Expand All @@ -55,6 +48,7 @@ function runFixtureCheck(
) {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-update-"));
const workflowPath = path.join(tempDir, "e2e.yaml");
const profileWorkflowPath = path.join(tempDir, "e2e-standard-profile.yaml");
const releasePath = path.join(tempDir, "release.json");
const assetPath = path.join(tempDir, "cloudflared-linux-amd64.deb");
const curlPath = path.join(tempDir, "curl");
Expand All @@ -66,7 +60,8 @@ function runFixtureCheck(
const downloadBase = "https://downloads.example.invalid/cloudflared";
const assetUrl = `${downloadBase}/${options.latestVersion}/cloudflared-linux-amd64.deb`;

writePinFixture(workflowPath, options.pinnedVersion, pinnedSha);
writePinFixture(workflowPath, options.pinnedVersion, pinnedSha, 3);
writePinFixture(profileWorkflowPath, options.pinnedVersion, pinnedSha, 1);
fs.writeFileSync(assetPath, asset);
fs.writeFileSync(
releasePath,
Expand Down Expand Up @@ -109,6 +104,7 @@ esac
CLOUDFLARED_CURL_BIN: curlPath,
CLOUDFLARED_DOWNLOAD_BASE_URL: downloadBase,
CLOUDFLARED_E2E_WORKFLOW: workflowPath,
CLOUDFLARED_PROFILE_WORKFLOW: profileWorkflowPath,
CLOUDFLARED_RELEASE_API_URL: apiUrl,
FAKE_API_URL: apiUrl,
FAKE_ASSET: assetPath,
Expand All @@ -126,20 +122,29 @@ describe("cloudflared update-check workflow contract", () => {
const workflow = readYaml<CloudflaredUpdateWorkflow>(
".github/workflows/cloudflared-update-check.yaml",
);
const e2e = fs.readFileSync(E2E_WORKFLOW, "utf8");
const configuredCheckCommand =
workflow.jobs?.["check-cloudflared"]?.steps?.find((step) => typeof step.run === "string")
?.run ?? "";

it("extracts exactly five identical reviewed version and SHA256 pins", () => {
const versions = pinValues(e2e, "CLOUDFLARED_VERSION");
const hashes = pinValues(e2e, "CLOUDFLARED_DEB_SHA256");
expect(versions).toHaveLength(5);
expect(hashes).toHaveLength(5);
expect(new Set(versions).size).toBe(1);
expect(new Set(hashes).size).toBe(1);
expect(versions[0]).toMatch(/^[0-9]{4}\.[0-9]{1,2}\.[0-9]+$/u);
expect(hashes[0]).toMatch(/^[0-9a-f]{64}$/u);
it("validates all current workflow pins before querying upstream", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-pins-"));
const curlPath = path.join(tempDir, "curl");
fs.writeFileSync(
curlPath,
"#!/usr/bin/env bash\nprintf 'fixture curl reached\\n' >&2\nexit 77\n",
{ mode: 0o755 },
);
try {
const result = spawnSync("bash", [CHECK_SCRIPT], {
cwd: ROOT,
encoding: "utf8",
env: { ...process.env, CLOUDFLARED_CURL_BIN: curlPath },
});
expect(result.status).toBe(77);
expect(result.stderr).toContain("fixture curl reached");
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});

it("queries the upstream latest release and verifies its exact linux-amd64 asset", () => {
Expand Down Expand Up @@ -185,7 +190,8 @@ describe("cloudflared update-check workflow contract", () => {
);
expect(fixture.result.stderr).toContain("CLOUDFLARED_VERSION lines:");
expect(fixture.result.stderr).toContain("CLOUDFLARED_DEB_SHA256 lines:");
expect(fixture.result.stderr).toContain("Set all five version/SHA256 pairs");
expect(fixture.result.stderr).toContain("e2e-standard-profile.yaml CLOUDFLARED_VERSION");
expect(fixture.result.stderr).toContain("Set all four version/SHA256 pairs");
} finally {
fs.rmSync(fixture.tempDir, { recursive: true, force: true });
}
Expand Down
35 changes: 20 additions & 15 deletions test/e2e-recommendations.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -902,27 +902,31 @@ jobs:
confidence: "high",
},
metadata({
changedFiles: [".github/workflows/e2e.yaml", "test/e2e/live/token-rotation.test.ts"],
changedFiles: [
".github/workflows/e2e.yaml",
"test/e2e/live/managed-image-protected-runtime.test.ts",
],
}),
{
e2eWorkflowText: String.raw`
jobs:
token-rotation:
if: \${{ (inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',token-rotation,') }}
managed-image-protected-runtime:
if: \${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime') }}
steps:
- run: npx vitest run --project e2e-live test/e2e/live/token-rotation.test.ts
- run: npx vitest run --project e2e-live test/e2e/live/managed-image-protected-runtime.test.ts
`,
},
);

expect(normalized.required.map((item) => [item.selectorType, item.id])).toEqual([
["job", "cloud-onboard"],
["job", "managed-image-multiarch-startup"],
["job", "managed-image-protected-runtime"],
["job", "security-posture"],
["job", "token-rotation"],
]);
expect(normalized.required.find((item) => item.id === "token-rotation")).not.toHaveProperty(
"dispatchCommand",
);
expect(
normalized.required.find((item) => item.id === "managed-image-protected-runtime"),
).not.toHaveProperty("dispatchCommand");
expect(normalized.noTargetE2eReason).toBeNull();
});

Expand All @@ -931,32 +935,33 @@ jobs:
{
required: [
{
id: "token-rotation",
id: "managed-image-protected-runtime",
workflow: E2E_WORKFLOW,
selectorType: "job",
reason: "focused job covers the changed live test",
reason: "focused job covers the changed managed-image test",
},
],
optional: [],
noTargetE2eReason: null,
confidence: "high",
},
metadata({ changedFiles: ["test/e2e/live/token-rotation.test.ts"] }),
metadata({ changedFiles: ["test/e2e/live/managed-image-protected-runtime.test.ts"] }),
{
e2eWorkflowText: String.raw`
jobs:
token-rotation:
if: \${{ contains(format(',{0},', inputs.jobs), ',token-rotation,') }}
managed-image-protected-runtime:
if: \${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime') }}
steps:
- run: npx vitest run --project e2e-live test/e2e/live/token-rotation.test.ts
- run: npx vitest run --project e2e-live test/e2e/live/managed-image-protected-runtime.test.ts
`,
},
);

expect(normalized.required.map((item) => [item.selectorType, item.id])).toEqual([
["job", "cloud-onboard"],
["job", "managed-image-multiarch-startup"],
["job", "managed-image-protected-runtime"],
["job", "security-posture"],
["job", "token-rotation"],
]);
expect(normalized.required[0]).not.toHaveProperty("dispatchCommand");
});
Expand Down
Loading
Loading