scheduler: run duppy on a pool of domains, retire its monad, and stop rescanning the event loop

.github/docker/debian-asan.dockerfile

@@ -5,7 +5,7 @@ FROM $BASE_IMAGE AS ocaml
 
 MAINTAINER The Savonet Team <contact@liquidsoap.info>
 
-ARG OCAML_VERSION=5.4.0
+ARG OCAML_VERSION=5.5.0
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV ASAN_OPTIONS="detect_leaks=0:detect_stack_use_after_return=0:detect_container_overflow=0:protect_shadow_gap=0:verify_asan_link_order=0"

.github/docker/website.dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.4.0
+FROM ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.5.0
 
 MAINTAINER The Savonet Team <contact@liquidsoap.info>
 

.github/docker/win32-base.dockerfile

@@ -1,59 +1,111 @@
-ARG OCAML_VERSION=5.4.1
-FROM ghcr.io/ocaml-cross/windows-x64-base:${OCAML_VERSION} AS builder
+ARG DEBIAN_VERSION=bookworm
 
-# Install missing build-time prerequisites for MXE packages.
-# python3-packaging is required by glib (a build dep of curl and others).
+FROM debian:${DEBIAN_VERSION}-slim AS builder
+
+ARG OCAML_VERSION=5.5.0
+
+# MXE revision. This is pinned rather than tracked: MXE decides a package is
+# stale by comparing its install stamp against the mtime of the recipe, so a
+# moving tree rebuilds the toolchain and every C library under it, and the
+# library versions we ship change without anything in this repo changing.
+ARG MXE_GIT_TAG=71cb7a3c56c0fde5d6aa806ad7623240e9128d57
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ENV MXE_DIR=/usr/src/mxe
+ENV CROSS_TRIPLE=x86_64-w64-mingw32.static
+
+# MXE's own requirements, from its docs/index.html. wine runs the built
+# executable, xvfb gives the initial wineboot a display, zip bundles the
+# release and gosu drops the CI step to the opam user.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3-packaging && \
-    apt-get clean
-
-# Update MXE to get the latest package recipes (e.g. ffmpeg 7.1.1).
-RUN cd /usr/src/mxe && git fetch origin && git reset --hard origin/master
-
-# Build MXE packages required by the Windows opam packages.
-# Each package is a separate layer to maximise Docker cache reuse.
-# ffmpeg is last because it is the slowest build and the most stable,
-# so its cache is only invalidated when strictly necessary.
-
-RUN cd /usr/src/mxe && make openssl
-RUN cd /usr/src/mxe && make curl
-RUN cd /usr/src/mxe && make libsrt
-RUN cd /usr/src/mxe && make jack
-RUN cd /usr/src/mxe && make portaudio
-RUN cd /usr/src/mxe && make libsamplerate
-RUN cd /usr/src/mxe && make dlfcn-win32
-RUN cd /usr/src/mxe && make libao
-RUN cd /usr/src/mxe && make ogg
-RUN cd /usr/src/mxe && make flac
-RUN cd /usr/src/mxe && make vorbis
-RUN cd /usr/src/mxe && make opus
-RUN cd /usr/src/mxe && make faad2
-RUN cd /usr/src/mxe && make lame
-RUN cd /usr/src/mxe && make libmad
-RUN cd /usr/src/mxe && make fdk-aac
-RUN cd /usr/src/mxe && make ffmpeg
-
-# Install xvfb to provide a virtual display for the initial Wine setup.
-RUN apt-get update && apt-get install -y --no-install-recommends xvfb xauth && apt-get clean
-
-# Install the latest opam release, answering prompts non-interactively.
+    apt-get install --no-install-recommends --yes \
+      autoconf automake autopoint bash bison bzip2 ca-certificates flex \
+      g++ g++-multilib gettext git gperf intltool libc6-dev-i386 \
+      libclang-dev libgdk-pixbuf-2.0-dev libgl-dev libltdl-dev libpcre2-dev \
+      libssl-dev libtool-bin libxml-parser-perl lzip make openssl p7zip-full \
+      patch perl python3 python3-mako python3-packaging python3-pkg-resources \
+      python3-setuptools python-is-python3 ruby sed sqlite3 unzip wget \
+      xz-utils \
+      curl gosu wine xvfb xauth zip && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/mxe/mxe.git "${MXE_DIR}" && \
+    git -C "${MXE_DIR}" checkout "${MXE_GIT_TAG}"
+
+# settings.mk is gitignored, so it is ours to own. The default target is
+# narrowed so that a bare `make` cannot start building all of MXE.
+RUN printf '%s\n' \
+      "MXE_TARGETS := ${CROSS_TRIPLE}" \
+      'MXE_USE_CCACHE :=' \
+      'LOCAL_PKG_LIST := cc' \
+      '.DEFAULT local-pkg-list:' \
+      'local-pkg-list: $(LOCAL_PKG_LIST)' \
+      > "${MXE_DIR}/settings.mk"
+
+# The cross toolchain, on its own layer: it is the slowest thing here and it
+# only moves when MXE_GIT_TAG does.
+RUN cd "${MXE_DIR}" && make cc
+
+# C libraries needed by the Windows opam packages. Each is a separate layer so
+# that adding one does not rebuild the others; ffmpeg is last because it is the
+# slowest and pulls the largest dependency chain.
+RUN cd "${MXE_DIR}" && make openssl
+RUN cd "${MXE_DIR}" && make curl
+RUN cd "${MXE_DIR}" && make libsrt
+RUN cd "${MXE_DIR}" && make jack
+RUN cd "${MXE_DIR}" && make portaudio
+RUN cd "${MXE_DIR}" && make libsamplerate
+RUN cd "${MXE_DIR}" && make dlfcn-win32
+RUN cd "${MXE_DIR}" && make libao
+RUN cd "${MXE_DIR}" && make ogg
+RUN cd "${MXE_DIR}" && make flac
+RUN cd "${MXE_DIR}" && make vorbis
+RUN cd "${MXE_DIR}" && make opus
+RUN cd "${MXE_DIR}" && make faad2
+RUN cd "${MXE_DIR}" && make lame
+RUN cd "${MXE_DIR}" && make libmad
+RUN cd "${MXE_DIR}" && make fdk-aac
+RUN cd "${MXE_DIR}" && make ffmpeg
+
+# The cross environment the final image also exports. The opam packages are
+# built in this stage, and without it their configure scripts find no
+# x86_64-w64-mingw32.static tools and quietly build for the host. MXE's bin
+# directory is appended, never prepended, so it cannot shadow the host tools
+# opam itself needs.
+ENV PATH="${PATH}:/usr/src/mxe/usr/bin"
+ENV WINEARCH=win64
+ENV CMAKE_TOOLCHAIN_FILE="/usr/src/mxe/usr/x86_64-w64-mingw32.static/share/cmake/mxe-conf.cmake"
+# PKG_CONFIG_PATH_default_windows is this repo's own dune-context convention;
+# an opam package's configure script only reads the real one.
+ENV PKG_CONFIG_PATH="/usr/src/mxe/usr/x86_64-w64-mingw32.static/lib/pkgconfig/"
+ENV TOOLPREF64="/usr/src/mxe/usr/bin/x86_64-w64-mingw32.static-"
+ENV PKG_CONFIG_PATH_default_windows="/usr/src/mxe/usr/x86_64-w64-mingw32.static/lib/pkgconfig/"
+ENV PKG_CONFIG_default_windows="/usr/src/mxe/usr/bin/x86_64-w64-mingw32.static-pkg-config"
+
 RUN printf "\ny\n" | bash -c "sh <(curl -fsSL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh)"
 
-# Pre-install opam dependencies so the CI build step only compiles liquidsoap.
+RUN useradd -g staff --create-home opam
 
 USER opam
 
 # Local opam overlay: camomile-embedded is not in opam-cross-windows.
-COPY .github/opam /tmp/opam-overlay
+COPY --chown=opam .github/opam /home/opam/opam-overlay
 
 RUN xvfb-run -a wineboot --init
 
+RUN opam init --auto-setup --disable-sandboxing --compiler="${OCAML_VERSION}" && \
+    opam repository add windows https://github.com/ocaml-cross/opam-cross-windows.git && \
+    opam repository add archive git+https://github.com/ocaml/opam-repository-archive && \
+    opam repository add liquidsoap-devel /home/opam/opam-overlay
+
+RUN eval $(opam env) && \
+    opam install -y ocaml-windows && \
+    opam clean
+
+# Pre-install opam dependencies so the CI build step only compiles liquidsoap.
 RUN eval $(opam env) && \
-    opam repository set-url windows https://github.com/ocaml-cross/opam-cross-windows.git && \
-    opam repo add archive git+https://github.com/ocaml/opam-repository-archive && \
-    opam repo add liquidsoap-devel /tmp/opam-overlay && \
-    opam update && \
-    opam install --deps-only -y /tmp/opam-overlay/liquidsoap-windows.opam && \
+    opam install --deps-only -y /home/opam/opam-overlay/liquidsoap-windows.opam && \
     opam clean
 
 FROM scratch

.github/opam/liquidsoap-windows.opam

@@ -16,7 +16,7 @@ homepage: "https://github.com/savonet/liquidsoap"
 bug-reports: "https://github.com/savonet/liquidsoap/issues"
 depends: [
   "dune" {>= "3.23"}
-  "ocaml-windows" {>= "5.4"}
+  "ocaml-windows" {>= "5.5"}
   "re" {>= "1.11.0" & < "1.14.0"}
   "re-windows" {>= "1.11.0" & < "1.14.0"}
   "curl-windows"

.github/scripts/build-win32.sh

@@ -20,19 +20,18 @@ echo "::group::Build liquidsoap-windows"
 
 eval "$(opam env)"
 
-# The CI image lacks the local opam overlay and predates crunch 4.1.0, which
-# camomile-embedded needs to build. Refreshing the windows repo there forces a
-# cross recompile that fails, hence default only; delete both lines once the
-# image is rebuilt.
-opam repo add liquidsoap-devel "${BASE_DIR}/.github/opam"
-opam update --repositories default
-
-# Diagnostic: surface the full dune error, which opam elides from the
-# combined install output.
-opam install -y -v camomile-embedded-windows
+# The image ships the overlay it was built against. Point it at this checkout
+# so a branch changing .github/opam is built with its own packages.
+opam repository set-url liquidsoap-devel "${BASE_DIR}/.github/opam"
+opam update liquidsoap-devel
 
 opam install -y --deps-only .github/opam/liquidsoap-windows.opam
 
+# The image sets PKG_CONFIG_PATH so that the opam packages configure against
+# mxe. Building liquidsoap goes through the dune context instead, and leaving
+# it set enables optional modules whose transitive libraries are not linked.
+unset PKG_CONFIG_PATH
+
 export LIQUIDSOAP_BUILD_VERSION="${TAG}${VERSION}"
 export LIQUIDSOAP_BUILD_TARGET=standalone
 export LIQUIDSOAP_SYS_CONFIG=mingw

.github/scripts/check-mutexed-lazy.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Stdlib Lazy is unsafe to force from several threads or domains, which
+# liquidsoap does. Lazy.Mutexed is the safe equivalent.
+
+set -e
+
+[ $# -eq 0 ] && exit 0
+
+if matches=$(grep -nE '\bLazy\.(force|t|from_val|from_fun|is_val)\b' "$@"); then
+  echo "Use Lazy.Mutexed rather than Lazy:"
+  echo "${matches}"
+  exit 1
+fi
+
+exit 0

.github/workflows/build-asan.yml

@@ -6,13 +6,13 @@ on:
       ocaml_version:
         description: "OCaml version"
         required: false
-        default: "5.4.0"
+        default: "5.5.0"
         type: string
   workflow_call:
     inputs:
       ocaml_version:
         required: false
-        default: "5.4.0"
+        default: "5.5.0"
         type: string
     secrets:
       DEPOT_TOKEN:
@@ -22,12 +22,12 @@ jobs:
   build:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-asan-${{ inputs.ocaml_version || '5.4.0' }}
+      image: ghcr.io/savonet/liquidsoap:ci-asan-${{ inputs.ocaml_version || '5.5.0' }}
       options: --user root --privileged -v ${{ github.workspace }}/${{ github.run_number }}:/tmp/artifacts
     env:
       HOME: /home/opam
       LIQUIDSOAP_INSTALL_NO_OPTIONAL_FAIL: "true"
-      OCAML_VERSION: ${{ inputs.ocaml_version || '5.4.0' }}
+      OCAML_VERSION: ${{ inputs.ocaml_version || '5.5.0' }}
       BRANCH: ${{ github.head_ref || github.ref_name }}
       ASAN_OPTIONS: "detect_leaks=0:detect_stack_use_after_return=0:detect_container_overflow=0:protect_shadow_gap=0:verify_asan_link_order=0"
       SKIP_SDL: "true"
@@ -52,7 +52,7 @@ jobs:
         id: build_deb
         env:
           GITHUB_SHA: ${{ github.sha }}
-          DOCKER_TAG: debian-testing-asan-ocaml${{ inputs.ocaml_version || '5.4.0' }}
+          DOCKER_TAG: debian-testing-asan-ocaml${{ inputs.ocaml_version || '5.5.0' }}
           BRANCH: ${{ github.head_ref || github.ref_name }}
           LIQ_TMP_DIR: /tmp/artifacts
           PLATFORM: amd64

.github/workflows/build-no-depopts.yml

@@ -14,7 +14,7 @@ jobs:
   build_no_depopts:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.4.0@sha256:1521b53591f846a76affe11cddc69402de41936523339c3fafa8c248fa396f5d
+      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.5.0@sha256:5c9b006dc14a8b81316da0bfa6938780c83797eb7d21f25f1b231bbca5d43d08
       options: --user opam
     env:
       HOME: /home/opam

.github/workflows/build-opam.yml

@@ -10,8 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         ocaml-compiler:
-          - 5.4.x
-          - 5.x
+          - 5.5.x
     steps:
       - name: Checkout latest code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -33,18 +32,18 @@ jobs:
         run: |
           opam install --cli=2.1 --confirm-level=unsafe-yes .
       - name: Install ocamlformat
-        if: matrix.ocaml-compiler == '5.4.x'
+        if: matrix.ocaml-compiler == '5.5.x'
         run: |
           opam install ocamlformat=0.28.1
       - name: Set PY env variable.
-        if: matrix.ocaml-compiler == '5.4.x'
+        if: matrix.ocaml-compiler == '5.5.x'
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
       - name: Restore pre-commit cache
-        if: matrix.ocaml-compiler == '5.4.x'
+        if: matrix.ocaml-compiler == '5.5.x'
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Run pre-commit
-        if: matrix.ocaml-compiler == '5.4.x'
+        if: matrix.ocaml-compiler == '5.5.x'
         uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

.github/workflows/build-win32.yml

@@ -23,7 +23,7 @@ jobs:
   build_win32:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-v2-win32-x64-5.4.1@sha256:b9dac6d22a7ec55e251ebc32d0bf4fd2c2e75e981dc35ea799bae45e1b105b17
+      image: ghcr.io/savonet/liquidsoap:ci-v2-win32-x64-5.5.0
       options: --user root -v ${{ github.workspace }}/${{ github.run_number }}:/tmp/${{ github.run_number }}
     env:
       IS_SNAPSHOT: ${{ inputs.is_snapshot == 'true' }}

.github/workflows/doc.yml

@@ -22,7 +22,7 @@ jobs:
   update_doc:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.4.0@sha256:1521b53591f846a76affe11cddc69402de41936523339c3fafa8c248fa396f5d
+      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.5.0@sha256:5c9b006dc14a8b81316da0bfa6938780c83797eb7d21f25f1b231bbca5d43d08
       options: --user root -v ${{ github.workspace }}/${{ github.run_number }}:/tmp/${{ github.run_number }}
     env:
       HOME: /home/opam

.github/workflows/docker-ci-asan.yml

@@ -6,7 +6,7 @@ on:
       ocaml_versions:
         description: "OCaml versions (JSON array)"
         required: false
-        default: '["5.4.0"]'
+        default: '["5.5.0"]'
         type: string
       push:
         description: "Push images to registry"

.github/workflows/docker-ci-win32.yml

@@ -6,7 +6,7 @@ on:
       ocaml_versions:
         description: "OCaml versions (JSON array)"
         required: false
-        default: '["5.4.1"]'
+        default: '["5.5.0"]'
         type: string
       no_cache:
         description: "Disable Docker layer cache"
@@ -28,6 +28,12 @@ jobs:
         ocaml_version: ${{ fromJSON(inputs.ocaml_versions) }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up Depot CLI
         uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2
       - name: Build and push

.github/workflows/js.yml

@@ -14,7 +14,7 @@ jobs:
   build_js:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.4.0@sha256:1521b53591f846a76affe11cddc69402de41936523339c3fafa8c248fa396f5d
+      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.5.0@sha256:5c9b006dc14a8b81316da0bfa6938780c83797eb7d21f25f1b231bbca5d43d08
       options: --user root
     env:
       HOME: /home/opam

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
   run_tests:
     runs-on: ubuntu-24.04
     container:
-      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.4.0@sha256:1521b53591f846a76affe11cddc69402de41936523339c3fafa8c248fa396f5d
+      image: ghcr.io/savonet/liquidsoap:ci-v2-debian_trixie-5.5.0@sha256:5c9b006dc14a8b81316da0bfa6938780c83797eb7d21f25f1b231bbca5d43d08
       options: --user root --privileged --ulimit core=-1 --security-opt seccomp=unconfined -v ${{ github.workspace }}/${{ github.run_number }}:/tmp/${{ github.run_number }}
     strategy:
       fail-fast: false

.pre-commit-config.yaml

@@ -66,6 +66,12 @@ repos:
         entry: koalaman/shellcheck --color=always
         types: [shell]
 
+      - id: mutexed-lazy
+        name: mutexed-lazy
+        language: system
+        entry: .github/scripts/check-mutexed-lazy.sh
+        files: ^src/(core|lang|js|bin)/.*\.mli?$
+
       - id: dunefmt
         name: dunefmt
         language: system