Nightly Codex contract #78
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Codex contract | |
| on: | |
| schedule: | |
| - cron: '23 3 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: nightly-codex-contract | |
| cancel-in-progress: false | |
| jobs: | |
| real-codex-appserver: | |
| name: Real Codex app-server contract | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 (#59) | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (#59) | |
| with: | |
| python-version: '3.11.9' | |
| cache: pip | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 (#59) | |
| with: | |
| node-version: '20.19.4' | |
| cache: npm | |
| cache-dependency-path: desktop/package-lock.json | |
| - name: Install pinned test dependencies | |
| run: | | |
| python -m pip install --disable-pip-version-check \ | |
| pytest==8.4.1 \ | |
| pytest-asyncio==1.0.0 \ | |
| PyYAML==6.0.2 \ | |
| aiohttp==3.12.13 \ | |
| jsonschema==4.26.0 \ | |
| httpx==0.28.1 | |
| - name: Install pinned Codex CLI | |
| run: | | |
| npm install --global --ignore-scripts @openai/codex@0.144.3 | |
| command -v codex | |
| test "$(codex --version)" = "codex-cli 0.144.3" | |
| - name: Run real-binary app-server contract | |
| env: | |
| CODEX_BIN: codex | |
| run: | | |
| python -m pytest tests/test_appserver_real_binary.py -v -rs \ | |
| --junitxml=real-codex-results.xml | |
| - name: Reject an all-skipped contract run | |
| if: always() | |
| run: | | |
| python - <<'PY' | |
| import sys | |
| import xml.etree.ElementTree as ET | |
| path = "real-codex-results.xml" | |
| try: | |
| root = ET.parse(path).getroot() | |
| except (FileNotFoundError, ET.ParseError) as exc: | |
| raise SystemExit(f"real Codex contract produced no readable JUnit report: {exc}") | |
| suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) | |
| tests = sum(int(suite.get("tests", "0")) for suite in suites) | |
| skipped = sum(int(suite.get("skipped", "0")) for suite in suites) | |
| if tests == 0 or skipped == tests: | |
| print( | |
| f"real Codex contract did not execute: tests={tests}, skipped={skipped}. " | |
| "Verify the pinned codex binary is installed and discoverable on PATH.", | |
| file=sys.stderr, | |
| ) | |
| raise SystemExit(1) | |
| print(f"real Codex contract executed: tests={tests}, skipped={skipped}") | |
| PY | |
| real-kimcli-appserver: | |
| name: Real kimcli app-server contract | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 (#59) | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 (#59) | |
| with: | |
| python-version: '3.11.9' | |
| cache: pip | |
| - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 (#59) | |
| with: | |
| node-version: '20.19.4' | |
| cache: npm | |
| cache-dependency-path: desktop/package-lock.json | |
| - name: Install pinned test dependencies | |
| run: | | |
| python -m pip install --disable-pip-version-check \ | |
| pytest==8.4.1 \ | |
| pytest-asyncio==1.0.0 \ | |
| PyYAML==6.0.2 \ | |
| aiohttp==3.12.13 \ | |
| jsonschema==4.26.0 \ | |
| httpx==0.28.1 | |
| # kimcli = rebranded pinned codex-cli fork (github.com/AdamMagued/codex, | |
| # branch kim-brand-0.144.3). This step is the drift alarm: it FAILS | |
| # until the kimcli-v0.144.3 release is published on that repo, which is | |
| # intended — a red nightly here means the pinned release is missing, | |
| # not that Kim itself regressed. | |
| - name: Download pinned kimcli release (fails until kimcli-v0.144.3 is published) | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| base=kimcli-x86_64-unknown-linux-musl | |
| asset="${base}.tar.gz" | |
| sidecar="${base}.sha256" | |
| mkdir -p kimcli-dl | |
| gh release download kimcli-v0.144.3 \ | |
| --repo AdamMagued/codex \ | |
| --pattern "${asset}" \ | |
| --pattern "${sidecar}" \ | |
| --dir kimcli-dl | |
| cd kimcli-dl | |
| expected=$(awk '{print $1}' "${sidecar}") | |
| actual=$(sha256sum "$asset" | awk '{print $1}') | |
| if [ "$actual" != "$expected" ]; then | |
| echo "checksum mismatch for $asset: expected $expected, got $actual" >&2 | |
| exit 1 | |
| fi | |
| echo "checksum verified: $expected" | |
| mkdir -p out | |
| tar -xzf "$asset" -C out | |
| chmod +x out/kimcli | |
| test "$(./out/kimcli --version)" = "kimcli 0.144.3 (rebranded codex-cli 0.144.3)" | |
| - name: Run real-binary app-server contract (kimcli) | |
| env: | |
| CODEX_BIN: ${{ github.workspace }}/kimcli-dl/out/kimcli | |
| run: | | |
| python -m pytest tests/test_appserver_real_binary.py -v -rs \ | |
| --junitxml=real-kimcli-results.xml | |
| - name: Reject an all-skipped contract run | |
| if: always() | |
| run: | | |
| python - <<'PY' | |
| import sys | |
| import xml.etree.ElementTree as ET | |
| path = "real-kimcli-results.xml" | |
| try: | |
| root = ET.parse(path).getroot() | |
| except (FileNotFoundError, ET.ParseError) as exc: | |
| raise SystemExit(f"real kimcli contract produced no readable JUnit report: {exc}") | |
| suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) | |
| tests = sum(int(suite.get("tests", "0")) for suite in suites) | |
| skipped = sum(int(suite.get("skipped", "0")) for suite in suites) | |
| if tests == 0 or skipped == tests: | |
| print( | |
| f"real kimcli contract did not execute: tests={tests}, skipped={skipped}. " | |
| "Verify the pinned kimcli binary is installed and discoverable on PATH.", | |
| file=sys.stderr, | |
| ) | |
| raise SystemExit(1) | |
| print(f"real kimcli contract executed: tests={tests}, skipped={skipped}") | |
| PY |