diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ab168010648..7db3d989ebc 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -301,6 +301,199 @@ jobs: # See the comment above about running all tests but still failing if one # of them does; it also applies here. + # Validate the repo against GHC's JavaScript backend, using the + # `javascript-unknown-ghcjs` cross toolchain from the ghcup `cross` release + # channel. See `cabal.validate-js.project` for how the build is set up, and + # `Cabal/jsbits/js-backend-polyfills.js` for the RTS primitives we provide + # ourselves. + # + # Known limitations of this job (all caused by the 9.12.2 cross bindist and + # the GHC JavaScript backend, not by Cabal itself): + # * `Cabal-tests:test:no-thunks-test` is not run: it needs the + # `unpackClosure#` primop, which the JavaScript backend does not provide. + # * `Cabal-tests:test:rpmvercmp` is not run: its reference implementation + # is a C FFI routine which cannot be bridged to the JavaScript RTS. + # * `cabal-install` unit tests are not run yet: they fail to load due to a + # static-pointers codegen bug in the JavaScript backend (used by + # `hackage-security`). + # * `cabal-testsuite` is not run yet: it executes test scripts in a GHCi + # session, and the cross bindist is "not built for interactive use". + validate-javascript: + name: Validate javascript ghc-${{ matrix.ghc }} + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + # Versions available in the ghcup `cross` release channel. + ghc: ["9.12.2"] + env: + # ghcup toolchain id of the JavaScript cross compiler (used with + # `ghcup install ghc`). + JS_GHC_TOOLCHAIN: javascript-unknown-ghcjs-${{ matrix.ghc }} + # Executable names of the cross compiler and its ghc-pkg. + JS_GHC: javascript-unknown-ghcjs-ghc-${{ matrix.ghc }} + JS_GHC_PKG: javascript-unknown-ghcjs-ghc-pkg-${{ matrix.ghc }} + # The emscripten version required by the cross bindist (see the ghcup + # pre-install message). + EMSDK_VERSION: "3.1.74" + # Isolate the store etc. of the JavaScript build from the host one. + CABAL_DIR: /home/runner/.cabal-js + steps: + - uses: actions/checkout@v7 + + # See https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#hackage-revisions + - name: Add manually supplied allow-newer + if: github.event_name == 'workflow_dispatch' && github.event.inputs.allow-newer != '' + run: | + echo "allow-newer: ${{ github.event.inputs.allow-newer }}" >> cabal.validate-js.project + + - name: Add manually supplied constraints + if: github.event_name == 'workflow_dispatch' && github.event.inputs.constraints != '' + run: | + echo "constraints: ${{ github.event.inputs.constraints }}" >> cabal.validate-js.project + + - uses: haskell-actions/setup@v2 + id: setup-haskell + with: + # Host toolchain; only used to drive `cabal`, all compilation is + # done with the cross compiler installed below. + ghc-version: ${{ env.GHC_FOR_RELEASE }} + cabal-version: latest + + - name: Cache emsdk, JS store and build directory + uses: actions/cache@v6 + with: + path: | + ~/.local/emsdk + ~/.cabal-js + dist-newstyle-validate-js + key: javascript-${{ env.JS_GHC }}-${{ env.EMSDK_VERSION }}-${{ hashFiles('cabal.validate-js.project') }}-${{ github.sha }} + restore-keys: | + javascript-${{ env.JS_GHC }}-${{ env.EMSDK_VERSION }}-${{ hashFiles('cabal.validate-js.project') }}- + javascript-${{ env.JS_GHC }}-${{ env.EMSDK_VERSION }}- + + - name: Install emsdk + run: | + if [ ! -x "$HOME/.local/emsdk/upstream/emscripten/emcc" ]; then + rm -rf "$HOME/.local/emsdk" + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$HOME/.local/emsdk" + "$HOME/.local/emsdk/emsdk" install "$EMSDK_VERSION" + "$HOME/.local/emsdk/emsdk" activate "$EMSDK_VERSION" + fi + source "$HOME/.local/emsdk/emsdk_env.sh" + emcc --version | head -1 + + - name: Install the GHC JavaScript cross toolchain + run: | + ghcup config add-release-channel cross + source "$HOME/.local/emsdk/emsdk_env.sh" + if ! command -v "$JS_GHC" > /dev/null 2>&1; then + # `emconfigure` makes the bindist's configure script use emcc. + emconfigure ghcup install ghc "$JS_GHC_TOOLCHAIN" + fi + "$JS_GHC" --version + + # See https://github.com/haskell/cabal/pull/8546 + - name: Work around git problem + run: git config --global protocol.file.allow always + + - name: Update the Hackage index (JS store) + run: | + source "$HOME/.local/emsdk/emsdk_env.sh" + cabal update + + # NB: use explicit component targets: `tests: True` is set in the + # project file, so a bare package target would also build every test + # suite of that package. + - name: Build + run: | + source "$HOME/.local/emsdk/emsdk_env.sh" + cabal build \ + --builddir=dist-newstyle-validate-js \ + --project-file=cabal.validate-js.project \ + --with-compiler="$JS_GHC" \ + --with-hc-pkg="$JS_GHC_PKG" \ + Cabal Cabal-hooks cabal-testsuite \ + cabal-install-solver cabal-install-solver:test:unit-tests \ + cabal-install:exe:cabal \ + Cabal-tests:test:unit-tests \ + Cabal-tests:test:check-tests \ + Cabal-tests:test:parser-tests + + # The test binaries are Node.js scripts, and expect to be run from the + # directory of their package (they use relative data file paths). + - name: Run lib tests + run: | + source "$HOME/.local/emsdk/emsdk_env.sh" + rc=0 + for t in unit-tests check-tests parser-tests; do + echo "::group::Cabal-tests:test:$t" + BIN=$(cabal list-bin \ + --builddir=dist-newstyle-validate-js \ + --project-file=cabal.validate-js.project \ + --with-compiler="$JS_GHC" \ + --with-hc-pkg="$JS_GHC_PKG" \ + "Cabal-tests:test:$t") + # Only the unit-tests support `--with-ghc` (see `libTests` in + # cabal-validate/src/Main.hs). + if [ "$t" = "unit-tests" ]; then + EXTRA_ARGS=(--with-ghc="$JS_GHC") + else + EXTRA_ARGS=() + fi + ( cd Cabal-tests && "$BIN" --hide-successes "${EXTRA_ARGS[@]}" ) || rc=1 + echo "::endgroup::" + done + exit $rc + + - name: Run solver tests + run: | + source "$HOME/.local/emsdk/emsdk_env.sh" + BIN=$(cabal list-bin \ + --builddir=dist-newstyle-validate-js \ + --project-file=cabal.validate-js.project \ + --with-compiler="$JS_GHC" \ + --with-hc-pkg="$JS_GHC_PKG" \ + cabal-install-solver:test:unit-tests) + ( cd cabal-install-solver && "$BIN" --hide-successes ) + + # Smoke test the `cabal` executable that we just built for the + # JavaScript target: use it to build a project with the cross compiler + # and run the resulting executable. + - name: Smoke test the JavaScript cabal + run: | + source "$HOME/.local/emsdk/emsdk_env.sh" + REPO_CABAL=$(cabal list-bin \ + --builddir=dist-newstyle-validate-js \ + --project-file=cabal.validate-js.project \ + --with-compiler="$JS_GHC" \ + --with-hc-pkg="$JS_GHC_PKG" \ + cabal-install:exe:cabal) + SMOKE_DIR=$(mktemp -d) + cd "$SMOKE_DIR" + cat > smoke.cabal <<'EOF' + cabal-version: 3.0 + name: smoke + version: 0 + build-type: Simple + + executable smoke + main-is: Main.hs + build-depends: base ^>=4.21 + default-language: GHC2021 + EOF + printf 'module Main where\nmain :: IO ()\nmain = putStrLn "js smoke ok"\n' > Main.hs + echo "packages: ." > cabal.project + "$REPO_CABAL" build \ + --builddir=dist-smoke \ + --with-compiler="$JS_GHC" \ + --with-hc-pkg="$JS_GHC_PKG" + EXE=$(find dist-smoke -name smoke -type f -perm -u+x | head -1) + OUT=$("$EXE") + echo "smoke output: $OUT" + [ "$OUT" = "js smoke ok" ] + build-alpine: name: Build statically linked using alpine runs-on: ubuntu-latest @@ -442,7 +635,7 @@ jobs: contents: write # IMPORTANT! Any job added to the workflow should be added here too - needs: [validate, validate-old-ghcs, build-alpine, dogfooding] + needs: [validate, validate-old-ghcs, validate-javascript, build-alpine, dogfooding] steps: - uses: actions/download-artifact@v8 @@ -470,7 +663,7 @@ jobs: contents: write # IMPORTANT! Any job added to the workflow should be added here too - needs: [validate, validate-old-ghcs, build-alpine, dogfooding] + needs: [validate, validate-old-ghcs, validate-javascript, build-alpine, dogfooding] steps: - uses: actions/download-artifact@v8 @@ -505,7 +698,7 @@ jobs: name: Validate post job runs-on: ubuntu-latest # IMPORTANT! Any job added to the workflow should be added here too - needs: [validate, validate-old-ghcs, build-alpine, dogfooding] + needs: [validate, validate-old-ghcs, validate-javascript, build-alpine, dogfooding] steps: - run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc34539b437..311863a40a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -167,6 +167,58 @@ For these test executables, `-p` which applies a regex filter to the test names. When running `cabal-install` test suites, one need only use `cabal test` or `cabal run ` in order to test locally. +### Testing with GHC's JavaScript backend + +CI runs a `validate-javascript` job that builds and tests the repo against +GHC's JavaScript backend, using the `javascript-unknown-ghcjs` cross +toolchain from the [ghcup `cross` release channel](https://www.haskell.org/ghcup/guide/cross/). +To reproduce it locally (Linux, x86_64): + +1. Install [emscripten](https://emscripten.org/docs/getting_started/downloads.html) + at the version required by the cross bindist (see the ghcup pre-install + message; 3.1.74 for `9.12.2`): + + ```sh + git clone https://github.com/emscripten-core/emsdk.git ~/.local/emsdk + ~/.local/emsdk/emsdk install 3.1.74 && ~/.local/emsdk/emsdk activate 3.1.74 + ``` + +2. Install the cross compiler (with emsdk on the `PATH`, and `emconfigure` + so the bindist's `configure` uses `emcc`): + + ```sh + ghcup config add-release-channel cross + source ~/.local/emsdk/emsdk_env.sh + emconfigure ghcup install ghc javascript-unknown-ghcjs-9.12.2 + ``` + +3. Build and test using `cabal.validate-js.project` (isolating the store in a + separate `CABAL_DIR` is recommended): + + ```sh + export CABAL_DIR=~/.cabal-js + source ~/.local/emsdk/emsdk_env.sh + cabal update + cabal build --builddir=dist-newstyle-validate-js \ + --project-file=cabal.validate-js.project \ + --with-compiler=javascript-unknown-ghcjs-ghc-9.12.2 \ + --with-hc-pkg=javascript-unknown-ghcjs-ghc-pkg-9.12.2 \ + Cabal cabal-install:exe:cabal Cabal-tests:test:unit-tests + ``` + + The produced executables are Node.js scripts and can be run directly + (they require `node` on the `PATH`). + +Known limitations on the JavaScript target (as of the `9.12.2` cross +bindist): `cabal-testsuite` cannot run (it needs a GHCi session and the +cross bindist is not built for interactive use), `cabal-install`'s unit tests +fail to load (a static-pointers codegen bug in the backend, triggered by +`hackage-security`), and the `no-thunks-test` and `rpmvercmp` suites of +`Cabal-tests` are not runnable (`unpackClosure#` primop and a C FFI routine +are missing on the target). See the `validate-javascript` job in +[.github/workflows/validate.yml](.github/workflows/validate.yml) for the +current list of steps that CI runs. + ## Running other checks locally Various other checks done by CI can be run locally to make sure your code doesn't diff --git a/Cabal/Cabal.cabal b/Cabal/Cabal.cabal index 1383588445d..b00db95e236 100644 --- a/Cabal/Cabal.cabal +++ b/Cabal/Cabal.cabal @@ -24,6 +24,9 @@ build-type: Simple extra-doc-files: README.md ChangeLog.md +extra-source-files: + jsbits/js-backend-polyfills.js + source-repository head type: git location: https://github.com/haskell/cabal/ @@ -40,6 +43,10 @@ library default-extensions: NoImportQualifiedPost hs-source-dirs: src + if arch(javascript) + -- See jsbits/js-backend-polyfills.js for why this is needed. + js-sources: jsbits/js-backend-polyfills.js + build-depends: , Cabal-syntax ^>= 3.19 , array >= 0.4.0.1 && < 0.6 diff --git a/Cabal/jsbits/js-backend-polyfills.js b/Cabal/jsbits/js-backend-polyfills.js new file mode 100644 index 00000000000..c9bb5c0661e --- /dev/null +++ b/Cabal/jsbits/js-backend-polyfills.js @@ -0,0 +1,103 @@ +// Polyfills for missing JavaScript RTS primitives in GHC 9.12 bindists. +// +// The `directory` boot library shipped with GHC 9.12.2 calls `fchmodat` (via +// `System.Directory.Internal.Posix.setModeAt`, used for example by +// `removePathForcibly` and `copyPermissions`), but the JavaScript runtime in +// the 9.12.2 bindists does not define the `h$fchmodat` primitive that the +// generated FFI stub expects, so such calls fail at runtime with: +// +// ReferenceError: h$fchmodat is not defined +// +// Likewise, the `clock` package's `System.Clock` FFI stubs reference +// `h$CLOCK_*` constants that the runtime does not define. The generated code +// calls them as zero-argument functions, and the runtime's `h$clock_gettime` +// implementation ignores the clock id, so the numeric values below, which +// mirror Linux, are only used for completeness. +// +// These definitions follow the same conventions as the runtime's `h$openat` +// (in particular `h$calculate_at` for dirfd-relative paths). The `flags` +// argument of `fchmodat` (e.g. `AT_SYMLINK_NOFOLLOW`) is ignored, as node's +// `chmodSync` has no symlink-safe equivalent on Linux. +// +// This file can be removed once the GHC JavaScript bindist in use ships a +// runtime with a native `h$fchmodat` and the `h$CLOCK_*` constants. + +function h$fchmodat(dirfd, path, path_off, mode, flags) { + if (h$isNode()) { + try { + var p = h$calculate_at(dirfd, path, path_off); + h$fs.chmodSync(p, mode); + return 0; + } catch (err) { + h$setErrno(err); + return -1; + } + } else { + return h$unsupported(-1); + } +} + +// Finally, `GHC.Conc.getNumProcessors` FFI stub references +// `h$getNumberOfProcessors`, which the runtime does not define either. + +function h$getNumberOfProcessors() { + if (h$isNode()) { + return h$os.cpus().length; + } + return 1; +} + +// The `cryptohash-sha256` package implements the hashing via C FFI routines +// (`hs_cryptohash_sha256_*`). Those are compiled with Emscripten, but the +// FFI stubs expect `h$`-prefixed JavaScript functions, so they do not +// resolve. Implement them using node's crypto module instead. The context +// pointer is used as an opaque key to a hash object. + +var h$sha256contexts = new Map(); +var h$crypto = require("crypto"); + +function h$hs_cryptohash_sha256_init(ctx, ctx_off) { + h$sha256contexts.set(ctx, h$crypto.createHash("sha256")); + return 0; +} + +function h$hs_cryptohash_sha256_update(ctx, ctx_off, data, data_off, len) { + var hash = h$sha256contexts.get(ctx); + hash.update(data.u8.subarray(data_off, data_off + len)); + return 0; +} + +function h$hs_cryptohash_sha256_finalize(ctx, ctx_off, out, out_off) { + var hash = h$sha256contexts.get(ctx); + h$sha256contexts.delete(ctx); + var digest = hash.digest(); + for (var i = 0; i < digest.length; i++) { + out.u8[out_off + i] = digest[i]; + } + return 0; +} + +function h$CLOCK_REALTIME() { + return 0; +} +function h$CLOCK_MONOTONIC() { + return 1; +} +function h$CLOCK_PROCESS_CPUTIME_ID() { + return 2; +} +function h$CLOCK_THREAD_CPUTIME_ID() { + return 3; +} +function h$CLOCK_MONOTONIC_RAW() { + return 4; +} +function h$CLOCK_REALTIME_COARSE() { + return 5; +} +function h$CLOCK_MONOTONIC_COARSE() { + return 6; +} +function h$CLOCK_BOOTTIME() { + return 7; +} diff --git a/cabal-testsuite/src/Test/Cabal/Server.hs b/cabal-testsuite/src/Test/Cabal/Server.hs index 51ee8bb909d..4973833c52b 100644 --- a/cabal-testsuite/src/Test/Cabal/Server.hs +++ b/cabal-testsuite/src/Test/Cabal/Server.hs @@ -270,6 +270,11 @@ initServer s0 = do pid <- withProcessHandle (serverProcessHandle s0) $ \case #if mingw32_HOST_OS OpenHandle x -> fmap show (Win32.getProcessId x) +#elif defined(javascript_HOST_ARCH) + -- PHANDLE has no `Show` instance on the JavaScript backend. + -- The process id is only used for logging, so a + -- placeholder is fine. + OpenHandle _ -> return "" #else OpenHandle x -> return (show x) #endif @@ -316,7 +321,13 @@ stopServer s = do -- will actually die, and then hClose will fail because -- the ":quit" command was buffered up but never got -- flushed. + -- + -- NB: `interruptProcessGroupOf` is unsupported by the JavaScript + -- backend's RTS (it throws "operation unsupported on this platform"); + -- there the ":quit" command and `terminateProcess` suffice. +#if !defined(javascript_HOST_ARCH) interruptProcessGroupOf (serverProcessHandle s) +#endif log ServerMeta s "Waiting..." -- Close input BEFORE waiting, close output AFTER waiting. diff --git a/cabal.validate-js.project b/cabal.validate-js.project new file mode 100644 index 00000000000..e7fdeb862c5 --- /dev/null +++ b/cabal.validate-js.project @@ -0,0 +1,78 @@ +-- Project file used by CI (and locally) to validate the repo against GHC's +-- JavaScript backend (target `javascript-unknown-ghcjs`). +-- +-- Usage: +-- export CABAL_DIR=~/.cabal-js (optional but recommended: isolates the store) +-- cabal build --builddir=dist-newstyle-validate-js --project-file=cabal.validate-js.project +-- --with-compiler=javascript-unknown-ghcjs-ghc-9.12.2 +-- --with-hc-pkg=javascript-unknown-ghcjs-ghc-pkg-9.12.2 +-- +-- See .github/workflows/validate.yml (job `validate-javascript`). + +import: project-cabal/ghc-options.config +import: project-cabal/ghc-latest.config +import: project-cabal/pkgs.config +import: project-cabal/constraints.config + +tests: True +write-ghc-environment-files: never + +-- Unlike `cabal.validate.project` we do not enable `-Werror` here: the goal is +-- JavaScript backend compatibility, and warnings specific to the JS target +-- would otherwise break CI spuriously. + +-- Unlike `cabal.validate.project` we do not enable the `git-rev` flags: it +-- would pull in the `githash` package from Hackage, an unnecessary dependency +-- for these builds. + +-- `network` is needed by cabal-install (via HTTP) and cabal-testsuite (via +-- network-wait). The released network-3.2.9.0 fails to build for the +-- JavaScript target because `Network.Socket.Shutdown` uses `GHC.Event`, which +-- is a stub on the JS RTS (see GHC #24108-ish / base-4.21). The fix (dropping +-- `GHC.Event` in favour of `forkIO`/`threadDelay`) is in network master but +-- has not been released yet, so we pin an unreleased commit. +source-repository-package + type: git + location: https://github.com/haskell/network.git + subdir: . + tag: 923f7d5996cb49d9b3e6ba05573b226db57c1f49 + -- The git checkout does not contain the `configure` script (it is generated + -- before uploading to Hackage), so we have to (re)generate it here. + -- Requires `autoconf` on the PATH. + post-checkout-command: autoreconf -i + +-- `splitmix` selects its seed initialisation via `impl(ghcjs)` / +-- `impl(ghc)`, but GHC's JavaScript backend reports itself as `ghc`, so the +-- C-based initialiser is chosen; its `ccall splitmix_init` becomes an +-- unresolvable `h$splitmix_init` on the JavaScript target. We patch the +-- source to (a) take the JavaScript branch and (b) use a function-expression +-- inline FFI (the `$r`-style snippet is GHCJS-only and a syntax error on +-- GHC's JavaScript backend). +source-repository-package + type: git + location: https://github.com/haskellari/splitmix.git + tag: 66ca2edbe935582cb66fd1fd36d1cd06366d9779 + -- The command is executed token-wise (no shell), so all expressions below + -- are deliberately whitespace-free. It (a) flips `impl(ghcjs)` to + -- `arch(javascript)` so the JS branch is taken on GHC's JavaScript backend, + -- and (b) rewrites the GHCJS-style inline FFI snippet into an arrow + -- function expression, the form GHC's JavaScript backend expects. + post-checkout-command: sed -i -e s@impl(ghcjs)@arch(javascript)@ -e s@\([^"]*\)"[^"]*Math.floor(Math.random()\*0x100000000);[^"]*"@\1"()=>Math.floor(Math.random()*0x100000000)"@ src/System/Random/SplitMix/Init.hs + +-- The `ghc-options` below force the JavaScript branch of +-- `System.Random.SplitMix.Init` (normally guarded by `__GHCJS__`, which GHC's +-- JavaScript backend does not define) and enable the JavaScript FFI. +package splitmix + ghc-options: -XJavaScriptFFI -DSPLITMIX_INIT_GHCJS=1 -D__GHCJS__=1 + +-- The C FFI routines of `cryptohash-sha256` do not resolve on the JavaScript +-- target (see Cabal/jsbits/js-backend-polyfills.js for the JS implementation +-- shim we provide). + +-- `resolv` (pulled in by the native-dns flag) fails to build for the +-- JavaScript target and DNS resolution is not needed for the tests. +package cabal-install + flags: -native-dns + +package cabal-install-solver + flags: +debug-expensive-assertions diff --git a/changelog.d/12294.md b/changelog.d/12294.md new file mode 100644 index 00000000000..39aa82b3b0b --- /dev/null +++ b/changelog.d/12294.md @@ -0,0 +1,18 @@ +--- +synopsis: Add a CI job validating against GHC's JavaScript backend +packages: [Cabal] +prs: 12294 +issues: 12293 +--- + +The `directory` and `clock` boot libraries shipped with GHC 9.12's JavaScript +backend reference RTS primitives (`h$fchmodat`, `h$CLOCK_*`, +`h$getNumberOfProcessors`) and `cryptohash-sha256` uses C FFI routines whose +`h$`-prefixed stubs the JavaScript runtime does not define. Calls into these +failed at runtime with `ReferenceError: ... is not defined`. + +The `Cabal` library now ships these primitives as JavaScript `js-sources` +under `arch(javascript)` (`jsbits/js-backend-polyfills.js`), implementing +them on top of the primitives the runtime does provide. This can be removed +once the GHC JavaScript runtime in use implements the missing primitives +natively.