diff --git a/.github/actions/setup-gnu-emacs/action.yml b/.github/actions/setup-gnu-emacs/action.yml index 79932b0b39..67498bf42f 100644 --- a/.github/actions/setup-gnu-emacs/action.yml +++ b/.github/actions/setup-gnu-emacs/action.yml @@ -1,34 +1,98 @@ name: 'Set up GNU Emacs oracle' description: >- - Install the pinned GNU Emacs that every GNU-vs-Neomacs comparison runs - against. Replaces the old apt emacs-nox (29.3 on ubuntu-24.04), which - version-skewed against the local pinned reference and the lisp tree: - 29.3's initial-scratch-message says "visit it with \[find-file]" while - Emacs 31 says "visit it with `\[find-file]'" — one row that sits on every - pair test's initial screen. The local pinned reference is the Emacs - 31.0.90 pretest built 2026-06-10; nix-emacs-ci publishes no pretests, so - this installs 31.1, the fixed release cut from the same emacs-31 branch - (verified to carry the same startup.el scratch message). The build is - minimal (no window-system, no native compilation), which matches the - pinned reference's no-native-comp configuration for TUI/batch oracles; - GUI-frame comparisons against GNU need an X-capable build, which no CI - profile has ever provided (emacs-nox included). + Install the GNU Emacs that every GNU-vs-Neomacs comparison runs against: + the parity reference pinned in parity-reference.toml, built from that exact + emacs-mirror commit. Nothing else will do. The lisp tree is synced to + emacs-31.0.90, and Emacs 31.0.91/31.1 removed the `display (min-width ...)` + padding from `mode-line-position` (emacs commit 388adcc570b), so against + 31.1 every TUI pair test disagrees on the mode-line row of every screen: + 889 of 942 tests failed on main. The earlier apt emacs-nox 29.3 disagreed + on the *scratch* buffer's first row instead. nix-emacs-ci publishes no + pretests, so the reference is compiled here from source and cached by + commit; a dedicated ci.yml job warms that cache before the shards that + spawn GNU start, and a cache hit costs one restore. The build is TTY/batch + only (no window system, no native compilation), matching the pinned + reference's configuration; GUI-frame comparisons against GNU need an + X-capable build, which no CI profile has ever provided. runs: using: 'composite' steps: - - name: Install GNU Emacs 31.1 - # master 2026-09-05. Self-installs Nix only when the runner has none - # (MELPA jobs install their own Nix first via install-nix-action). - uses: purcell/setup-emacs@34c6ded44899fd1bf74d2889558befd1750e61a7 + - name: Read the pinned GNU reference + id: pin + shell: bash + run: | + # The action names no version of its own: parity-reference.toml is + # the single record of which GNU the parity numbers are about, and a + # re-pin there (cargo run -p xtask -- pin-reference) moves CI with it. + set -euo pipefail + pin="$GITHUB_WORKSPACE/parity-reference.toml" + version="$(sed -n 's/^emacs_version = "\(.*\)"$/\1/p' "$pin")" + commit="$(sed -n 's/^mirror_commit = "\(.*\)"$/\1/p' "$pin")" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || [[ ! "$commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "parity-reference.toml does not pin emacs_version and mirror_commit" >&2 + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + # actions/cache restores as the runner user, and /opt is root's. + sudo mkdir -p /opt/gnu-emacs-oracle + sudo chown "$(id -u):$(id -g)" /opt/gnu-emacs-oracle + + - name: Restore the cached oracle build + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - version: 31.1 + path: /opt/gnu-emacs-oracle + # Keyed on the commit and on this file, so a re-pin or a configure + # change rebuilds instead of serving the previous oracle. + key: gnu-emacs-oracle-${{ runner.os }}-${{ steps.pin.outputs.commit }}-${{ hashFiles('.github/actions/setup-gnu-emacs/action.yml') }} + + - name: Build GNU Emacs from the pinned commit + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + EMACS_COMMIT: ${{ steps.pin.outputs.commit }} + EMACS_VERSION: ${{ steps.pin.outputs.version }} + run: | + set -euo pipefail + # Build tools only; the libraries come from scripts/ci/setup-linux.sh, + # which every caller runs first (its build package set carries + # gnutls, ncurses and libxml2 development headers). + sudo apt-get update + sudo apt-get install -y --no-install-recommends autoconf texinfo make gcc + src="$(mktemp -d)" + git -C "$src" init -q + # Fetch the commit itself, not the tag: the tag is documentation, the + # commit is the identity the parity numbers were measured against. + git -C "$src" fetch -q --depth 1 https://github.com/emacs-mirror/emacs.git "$EMACS_COMMIT" + git -C "$src" checkout -q FETCH_HEAD + cd "$src" + ./autogen.sh + ./configure --prefix=/opt/gnu-emacs-oracle \ + --without-x --without-pgtk --without-native-compilation \ + --with-gnutls=ifavailable --with-xml2=ifavailable \ + --with-tree-sitter=ifavailable --with-libgmp=ifavailable \ + --with-sound=no --without-dbus --without-gconf --without-gsettings \ + --without-libsystemd --without-selinux --without-gpm \ + --without-sqlite3 + make -j"$(nproc)" + make install + rm -rf "$src" - - name: Record and smoke the oracle build + - name: Put the oracle on PATH and prove its version shell: bash + env: + EMACS_VERSION: ${{ steps.pin.outputs.version }} run: | - # A divergence is only meaningful against a known GNU version: record - # it in the log, then fail at this environment seam (the check that - # used to live in setup-linux.sh) instead of inside an oracle test. - emacs --version | head -n 1 - emacs --batch --quick --eval '(kill-emacs 0)' + # A divergence is only meaningful against a known GNU version, so + # this fails at the environment seam, not inside an oracle test, and + # it checks the version rather than merely that an emacs runs. + set -euo pipefail + echo "/opt/gnu-emacs-oracle/bin" >> "$GITHUB_PATH" + got="$(/opt/gnu-emacs-oracle/bin/emacs --batch --quick --eval '(princ emacs-version)')" + if [[ "$got" != "$EMACS_VERSION" ]]; then + echo "GNU oracle reports emacs-version $got, parity-reference.toml pins $EMACS_VERSION" >&2 + exit 1 + fi + /opt/gnu-emacs-oracle/bin/emacs --version | head -n 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4276ba8509..824dbba2e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -471,6 +471,23 @@ jobs: with: suite: core + gnu-emacs-oracle: + name: GNU Emacs oracle (linux x86_64) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - *checkout + + # Warm the per-commit cache of the pinned GNU reference once, while the + # runtime and test archives build, so the tui shards and the MELPA and + # GUI parity jobs each restore one cached build instead of compiling + # Emacs twenty-odd times in parallel on a cache miss. + - name: Install Linux system dependencies + run: scripts/ci/setup-linux.sh oracle + + - name: Set up GNU Emacs oracle + uses: ./.github/actions/setup-gnu-emacs + neomacs-test-runtime: name: Packaged Neomacs Runtime (linux x86_64) runs-on: ubuntu-24.04 @@ -620,7 +637,7 @@ jobs: neomacs-tui-tests: if: github.event_name != 'schedule' - needs: [neomacs-test-runtime, neomacs-workspace-test-archive] + needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle] uses: ./.github/workflows/nextest-shards.yml with: suite: tui @@ -628,7 +645,7 @@ jobs: neomacs-melpa-tests: name: MELPA ${{ matrix.suite }} compatibility and GNU parity (linux x86_64) if: github.event_name != 'schedule' - needs: [neomacs-test-runtime, neomacs-workspace-test-archive] + needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle] runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: @@ -648,10 +665,10 @@ jobs: NEXTEST_FILTER: ${{ matrix.filter }} NEOMACS_BIN: ${{ github.workspace }}/target/release/neomacs NEOMACS_RUNTIME_ROOT: ${{ github.workspace }} - NEOMACS_MELPA_ORACLE_EMACS: emacs + NEOMACS_MELPA_ORACLE_EMACS: /opt/gnu-emacs-oracle/bin/emacs NEOMACS_GUI_TEST_BACKEND: x11 NEOMACS_GUI_TEST_BINARY: ${{ github.workspace }}/target/release/neomacs - NEOMACS_GUI_TEST_GNU_EMACS: emacs + NEOMACS_GUI_TEST_GNU_EMACS: /opt/gnu-emacs-oracle/bin/emacs steps: - *checkout - *prepare_workspace_tmp @@ -758,7 +775,7 @@ jobs: neomacs-prefix-face-tui-parity: name: GNU prefix face TUI parity (linux x86_64) if: github.event_name != 'schedule' - needs: [neomacs-test-runtime, neomacs-workspace-test-archive] + needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle] runs-on: ubuntu-24.04 timeout-minutes: 30 env: @@ -805,7 +822,7 @@ jobs: neomacs-gui-tests: name: real GUI tests (linux x86_64, X11) if: github.event_name != 'schedule' - needs: [neomacs-test-runtime, neomacs-workspace-test-archive] + needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle] runs-on: ubuntu-24.04 timeout-minutes: 60 env: @@ -814,7 +831,7 @@ jobs: TMPDIR: ${{ github.workspace }}/tmp NEOMACS_GUI_TEST_BACKEND: x11 NEOMACS_GUI_TEST_BINARY: ${{ github.workspace }}/target/release/neomacs - NEOMACS_GUI_TEST_GNU_EMACS: emacs + NEOMACS_GUI_TEST_GNU_EMACS: /opt/gnu-emacs-oracle/bin/emacs steps: - *checkout - *prepare_workspace_tmp @@ -859,7 +876,7 @@ jobs: neomacs-melpa-live-canary: name: MELPA live ecosystem canary (linux x86_64) if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - needs: [neomacs-test-runtime, neomacs-workspace-test-archive] + needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle] runs-on: ubuntu-24.04 timeout-minutes: 30 env: @@ -868,7 +885,7 @@ jobs: TMPDIR: ${{ github.workspace }}/tmp NEOMACS_BIN: ${{ github.workspace }}/target/release/neomacs NEOMACS_RUNTIME_ROOT: ${{ github.workspace }} - NEOMACS_MELPA_ORACLE_EMACS: emacs + NEOMACS_MELPA_ORACLE_EMACS: /opt/gnu-emacs-oracle/bin/emacs steps: - *checkout - *prepare_workspace_tmp diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f1f9baa703..550c6512af 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -20,7 +20,7 @@ jobs: RUST_BACKTRACE: 1 TMPDIR: ${{ github.workspace }}/tmp NEOMACS_TUI_RECORD: on - NEOMACS_MELPA_ORACLE_EMACS: emacs + NEOMACS_MELPA_ORACLE_EMACS: /opt/gnu-emacs-oracle/bin/emacs steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -31,10 +31,8 @@ jobs: - name: Install system dependencies (apt) run: scripts/ci/setup-linux.sh ecosystem - # Before the tui tests, which spawn the GNU oracle. This provisions a - # Nix of its own; the later install-nix-action in this job must reuse - # it (this workflow is manual-dispatch only, so a conflict fails loudly - # in a low-traffic job rather than silently skipping the oracle). + # Before the tui tests, which spawn the GNU oracle: the pinned + # parity reference, restored from the per-commit cache or compiled. - name: Set up GNU Emacs oracle uses: ./.github/actions/setup-gnu-emacs diff --git a/crates/xtask/src/main_test.rs b/crates/xtask/src/main_test.rs index a5df84fe88..996d77bfea 100644 --- a/crates/xtask/src/main_test.rs +++ b/crates/xtask/src/main_test.rs @@ -700,13 +700,31 @@ fn linux_ci_setup_profiles_expose_capabilities_and_reject_unknown_profiles() { ); let oracle = packages("oracle"); - for package in ["liblcms2-dev", "emacs-nox", "libfaketime"] { + for package in ["liblcms2-dev", "libfaketime"] { assert!(oracle.lines().any(|candidate| candidate == package)); } + // GNU Emacs is never an apt package: the apt emacs-nox (29.3 on + // ubuntu-24.04) version-skews against the Emacs 31 reference the lisp + // tree and parity-reference.toml track. Every GNU-vs-Neomacs comparison + // runs the build that .github/actions/setup-gnu-emacs installs. + for profile in [ + "build", + "build-no-gstreamer", + "oracle", + "ecosystem", + "release", + ] { + assert!( + !packages(profile) + .lines() + .any(|package| package == "emacs-nox"), + "profile {profile} must not install the apt emacs-nox" + ); + } + let ecosystem = packages("ecosystem"); for package in [ - "emacs-nox", "gnupg", "xvfb", "xauth", @@ -1148,7 +1166,9 @@ fn ci_uses_one_typed_sharded_nextest_workflow_for_core_oracle_and_tui() { assert!(oracle.contains("uses: ./.github/workflows/nextest-shards.yml")); assert!(oracle.contains("suite: oracle")); let tui = github_workflow_job(workflow, "neomacs-tui-tests"); - assert!(tui.contains("needs: [neomacs-test-runtime, neomacs-workspace-test-archive]")); + assert!(tui.contains( + "needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle]" + )); assert!(tui.contains("uses: ./.github/workflows/nextest-shards.yml")); assert!(tui.contains("suite: tui")); } @@ -1188,13 +1208,16 @@ fn ci_runs_offline_melpa_parity_from_shared_artifacts() { )); let job = github_workflow_job(workflow, "neomacs-melpa-tests"); - assert!(job.contains("needs: [neomacs-test-runtime, neomacs-workspace-test-archive]")); + assert!(job.contains( + "needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle]" + )); assert!(!job.contains("if: ${{ false }}")); assert!(job.contains("name: neomacs-test-runtime-linux-x86_64")); assert!(job.contains("tar xzf neomacs-test-runtime-linux-x86_64.tar.gz")); assert!(job.contains("name: neomacs-workspace-tests-nextest-archive-linux-x86_64")); assert!(job.contains("NEOMACS_BIN: ${{ github.workspace }}/target/release/neomacs")); - assert!(job.contains("NEOMACS_MELPA_ORACLE_EMACS: /usr/bin/emacs")); + assert!(job.contains(&format!("NEOMACS_MELPA_ORACLE_EMACS: {GNU_ORACLE_EMACS}"))); + assert!(job.contains("uses: ./.github/actions/setup-gnu-emacs")); assert!(job.contains("run: scripts/ci/setup-linux.sh ecosystem")); for suite in ["batch", "tui", "gui"] { assert!(job.contains(&format!("suite: {suite}"))); @@ -1232,12 +1255,85 @@ fn ci_executes_display_stack_and_real_gui_tests_from_shared_artifacts() { assert!(display.contains("protocol)|package(neomacs-display-runtime)")); let gui = github_workflow_job(workflow, "neomacs-gui-tests"); - assert!(gui.contains("needs: [neomacs-test-runtime, neomacs-workspace-test-archive]")); + assert!(gui.contains( + "needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle]" + )); assert!(gui.contains("NEOMACS_GUI_TEST_BACKEND: x11")); - assert!(gui.contains("NEOMACS_GUI_TEST_GNU_EMACS: /usr/bin/emacs")); + assert!(gui.contains(&format!("NEOMACS_GUI_TEST_GNU_EMACS: {GNU_ORACLE_EMACS}"))); + assert!(gui.contains("uses: ./.github/actions/setup-gnu-emacs")); assert!(gui.contains("package(neomacs-gui-tests)")); } +/// Where `.github/actions/setup-gnu-emacs` installs the GNU oracle. The +/// path carries no version on purpose: the version is read from +/// parity-reference.toml, so a re-pin changes one file. +const GNU_ORACLE_EMACS: &str = "/opt/gnu-emacs-oracle/bin/emacs"; + +/// The GNU oracle CI runs against is the parity reference the lisp tree and +/// every published parity number track, and nothing else. Emacs 31.1 (and +/// 31.0.91) removed the `display (min-width ...)` padding from +/// `mode-line-position` (emacs commit 388adcc570b), so against any newer +/// GNU every TUI pair test disagrees on the mode-line row; the old apt +/// emacs-nox 29.3 disagreed on the scratch buffer's first row. The action +/// therefore takes the tag and commit from parity-reference.toml at run +/// time instead of naming a version of its own, and CI warms one cached +/// build in a dedicated job before the shards that spawn GNU start. +#[test] +fn gnu_oracle_action_builds_the_pinned_parity_reference() { + let action = include_str!(concat!( + env!("CARGO_WORKSPACE_DIR"), + "/.github/actions/setup-gnu-emacs/action.yml" + )); + let reference = include_str!(concat!( + env!("CARGO_WORKSPACE_DIR"), + "/parity-reference.toml" + )); + let pinned = |key: &str| -> String { + reference + .lines() + .find_map(|line| line.strip_prefix(&format!("{key} = "))) + .unwrap_or_else(|| panic!("parity-reference.toml must pin {key}")) + .trim_matches('"') + .to_string() + }; + assert_eq!(pinned("emacs_version"), "31.0.90"); + assert_eq!(pinned("mirror_commit").len(), 40); + + // The action derives both identities from the pin file rather than + // repeating them, so a re-pin cannot leave CI on a stale oracle. + assert!(action.contains("parity-reference.toml")); + assert!(action.contains("emacs_version")); + assert!(action.contains("mirror_commit")); + assert!(!action.contains("purcell/setup-emacs")); + assert!(!action.contains("apt-get install -y --no-install-recommends emacs")); + for stale in ["31.1", "31.0.91", "29.3"] { + assert!( + !action.contains(&format!("version: {stale}")), + "the action must not name a GNU version of its own ({stale})" + ); + } + assert!(action.contains("/opt/gnu-emacs-oracle")); + assert!(action.contains("--without-native-compilation")); + assert!(action.contains("--without-x")); + assert!(action.contains("actions/cache@")); + // The smoke step must prove the version, not just that an emacs runs. + assert!(action.contains("emacs-version")); + + let workflow = include_str!(concat!( + env!("CARGO_WORKSPACE_DIR"), + "/.github/workflows/ci.yml" + )); + let warm = github_workflow_job(workflow, "gnu-emacs-oracle"); + assert!(warm.contains("uses: ./.github/actions/setup-gnu-emacs")); + for job in ["neomacs-tui-tests", "neomacs-prefix-face-tui-parity"] { + let spawns_gnu = github_workflow_job(workflow, job); + assert!( + spawns_gnu.contains("gnu-emacs-oracle]"), + "{job} spawns GNU and must wait for the warmed oracle cache" + ); + } +} + #[test] fn ci_runs_live_melpa_only_as_an_explicit_canary() { let workflow = include_str!(concat!( @@ -1247,7 +1343,9 @@ fn ci_runs_live_melpa_only_as_an_explicit_canary() { let job = github_workflow_job(workflow, "neomacs-melpa-live-canary"); assert!(workflow.contains("schedule:")); - assert!(job.contains("needs: [neomacs-test-runtime, neomacs-workspace-test-archive]")); + assert!(job.contains( + "needs: [neomacs-test-runtime, neomacs-workspace-test-archive, gnu-emacs-oracle]" + )); assert!(job.contains("github.event_name == 'schedule'")); assert!(job.contains("github.event_name == 'workflow_dispatch'")); assert!(job.contains("- *download_test_runtime")); diff --git a/docs/building.md b/docs/building.md index f659276a89..b2422e0c99 100644 --- a/docs/building.md +++ b/docs/building.md @@ -48,6 +48,16 @@ cargo nextest run -p neomacs-tui-tests --release --no-fail-fast The TUI harness uses `target/release/neomacs` by default, regardless of the Cargo test profile. Set `NEOMACS_TUI_NEOMACS_BIN` to use a different binary. +The GNU side of every pair comparison is whichever `emacs` is first on `PATH`, +and it must be the parity reference pinned in `parity-reference.toml` +(Emacs 31.0.90, the revision the Lisp tree is synced to). Newer releases +change what the shared Lisp renders: Emacs 31.0.91 and 31.1 dropped the +`min-width` padding from `mode-line-position`, so against them almost every +TUI pair test fails on the mode-line row. CI compiles that exact commit in +`.github/actions/setup-gnu-emacs`; locally, build the `emacs-31.0.90` tag +without native compilation and put its `bin` first on `PATH` for the TUI +and GUI suites. + Set `NEOMACS_TUI_RECORD=on` to write an asciicast v3 recording for every `TuiSession`. Recording is disabled by default. Core parity tests are grouped by Rust test name and package parity tests by package scenario: