BUG: Fix missing bf16 Metal kernels on Apple Silicon

patches/llama.cpp/0001-metal-pin-msl-language-version.patch

@@ -0,0 +1,67 @@
+diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
+index 80e47f2c2..c7831c3a2 100644
+--- a/ggml/src/ggml-metal/ggml-metal-device.m
++++ b/ggml/src/ggml-metal/ggml-metal-device.m
+@@ -103,6 +103,44 @@ struct ggml_metal_library {
+     NSLock * lock;
+ };
+
++// Pin the Metal Shading Language (MSL) version used for runtime shader compilation.
++//
++// When the language version is not set explicitly, the Metal compiler derives a
++// default from the LC_BUILD_VERSION of the host executable (e.g. the python
++// interpreter that loaded the library). Host executables built against old SDKs
++// get a default older than MSL 3.1, in which case the shaders silently drop the
++// bf16 kernels through the __METAL_VERSION__ < 310 guard in ggml-metal.metal,
++// while the device still reports bfloat support. This results in runtime
++// failures such as:
++//
++//   ggml_metal_library_compile_pipeline: ... "Function kernel_mul_mv_ext_bf16_f32_r1_2
++//   was not found in the library"
++//
++// ref: https://github.com/ggml-org/llama.cpp/issues/21381
++// ref: https://github.com/hybridgroup/yzma/issues/226
++static void ggml_metal_compile_options_set_language_version(ggml_metal_device_t dev, MTLCompileOptions * options) {
++    if (ggml_metal_device_get_props(dev)->has_bfloat) {
++        // bfloat requires MSL 3.1+
++        if (@available(macOS 14.0, iOS 17.0, *)) {
++            if (options.languageVersion < MTLLanguageVersion3_1) {
++                options.languageVersion = MTLLanguageVersion3_1;
++            }
++        }
++    }
++
++    if (ggml_metal_device_get_props(dev)->has_tensor) {
++        // the tensor API requires MSL 4.0+
++        // note: MTLLanguageVersion4_0 requires the macOS 26 SDK, so use the raw
++        //       value (major << 16 | minor) to stay compatible with older Xcode
++        if (@available(macOS 26.0, iOS 26.0, *)) {
++            const MTLLanguageVersion msl_4_0 = (MTLLanguageVersion) (4 << 16);
++            if (options.languageVersion < msl_4_0) {
++                options.languageVersion = msl_4_0;
++            }
++        }
++    }
++}
++
+ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
+     id<MTLLibrary> library = nil;
+     id<MTLDevice> device = ggml_metal_device_get_obj(dev);
+@@ -228,6 +266,8 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
+                 MTLCompileOptions * options = [MTLCompileOptions new];
+                 options.preprocessorMacros = prep;
+
++                ggml_metal_compile_options_set_language_version(dev, options);
++
+                 //[options setFastMathEnabled:false];
+
+                 library = [device newLibraryWithSource:src options:options error:&error];
+@@ -285,6 +325,8 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev
+         MTLCompileOptions * options = [MTLCompileOptions new];
+         options.preprocessorMacros = prep;
+
++        ggml_metal_compile_options_set_language_version(dev, options);
++
+         library = [device newLibraryWithSource:src options:options error:&error];
+         if (error) {
+             if (verbose) {

scripts/build.py

@@ -15,6 +15,7 @@
 ROOT = Path(__file__).resolve().parents[1]
 PROJECT = ROOT / "thirdparty" / "llama.cpp"
 PREFIX = ROOT / "src" / "llama.cpp"
+PATCH_DIR = ROOT / "patches" / "llama.cpp"
 
 
 def log(message: str) -> None:
@@ -43,6 +44,69 @@ def split_cmake_args(value: str) -> list[str]:
     return parts
 
 
+def llamacpp_patches() -> list[Path]:
+    """Local hotfix patches applied to the vendored llama.cpp at build time.
+
+    The submodule checkout itself is never modified permanently: patches from
+    patches/llama.cpp/*.patch are applied before building and reverted right
+    after, so the tree stays clean for submodule bumps. Once a patch lands
+    upstream, delete it (and bump the submodule) -- a patch that no longer
+    applies fails the build loudly instead of being silently skipped.
+    """
+    if not PATCH_DIR.is_dir():
+        return []
+    return sorted(PATCH_DIR.glob("*.patch"))
+
+
+def apply_llamacpp_patches(patches: list[Path]) -> list[Path]:
+    """Apply patches to the llama.cpp checkout; return the ones applied now.
+
+    Patches that are already present in the working tree (e.g. left over from
+    an interrupted build) are skipped and not returned, so they are not
+    reverted either.
+    """
+    applied: list[Path] = []
+    for patch in patches:
+        already_applied = (
+            subprocess.run(
+                ["git", "apply", "--reverse", "--check", str(patch)],
+                cwd=PROJECT,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            ).returncode
+            == 0
+        )
+        if already_applied:
+            log(f"patch already applied, skipping: {patch.name}")
+            continue
+        run(["git", "apply", str(patch)], cwd=PROJECT)
+        log(f"applied patch: {patch.name}")
+        applied.append(patch)
+    return applied
+
+
+def revert_llamacpp_patches(patches: list[Path]) -> None:
+    for patch in reversed(patches):
+        run(["git", "apply", "--reverse", str(patch)], cwd=PROJECT)
+        log(f"reverted patch: {patch.name}")
+        # git apply --reverse restores the original content but with a fresh
+        # mtime that is still *older* than the objects just compiled from the
+        # patched sources. Bump the mtime of every file the patch touches so
+        # the next build recompiles them if the patch set has changed.
+        for line in subprocess.run(
+            ["git", "apply", "--numstat", str(patch)],
+            cwd=PROJECT,
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout.splitlines():
+            parts = line.split("\t")
+            if len(parts) == 3:
+                touched = PROJECT / parts[2]
+                if touched.exists():
+                    os.utime(touched)
+
+
 def hip_compiler() -> str:
     """Return the path to the HIP C++ compiler (clang).
 
@@ -198,21 +262,25 @@ def build_llamacpp() -> None:
     log("Running CMake with arguments: " + " ".join(cmake_args))
     log("Building targets: " + " ".join(targets))
 
-    run(["cmake", "..", *cmake_args], cwd=build_dir)
-    run(
-        [
-            "cmake",
-            "--build",
-            ".",
-            "--config",
-            "Release",
-            "--parallel",
-            nproc,
-            "--target",
-            *targets,
-        ],
-        cwd=build_dir,
-    )
+    applied = apply_llamacpp_patches(llamacpp_patches())
+    try:
+        run(["cmake", "..", *cmake_args], cwd=build_dir)
+        run(
+            [
+                "cmake",
+                "--build",
+                ".",
+                "--config",
+                "Release",
+                "--parallel",
+                nproc,
+                "--target",
+                *targets,
+            ],
+            cwd=build_dir,
+        )
+    finally:
+        revert_llamacpp_patches(applied)
 
     shutil.rmtree(PREFIX, ignore_errors=True)
     run([sys.executable, str(ROOT / "scripts" / "copy_libs.py")], cwd=ROOT)

tests/test_build_patches.py

@@ -0,0 +1,129 @@
+"""Unit tests for the build-time llama.cpp patch machinery in scripts/build.py.
+
+The wheel build applies hotfix patches from patches/llama.cpp/*.patch to the
+vendored submodule before the CMake build and reverts them right after, so
+the submodule working tree stays pristine. These tests exercise that logic
+against a throwaway git repository instead of the real submodule.
+"""
+
+import importlib.util
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).parent.parent
+
+spec = importlib.util.spec_from_file_location(
+    "xllamacpp_scripts_build", ROOT / "scripts" / "build.py"
+)
+build = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(build)
+
+
+def run_git(repo: Path, *args: str) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["git", *args], cwd=repo, check=True, capture_output=True, text=True
+    )
+
+
+@pytest.fixture
+def fake_llamacpp(tmp_path, monkeypatch):
+    """A throwaway git repo standing in for the llama.cpp submodule.
+
+    The patch file is authored the same way as the real one: edit, git diff,
+    revert.
+    """
+    proj = tmp_path / "llama.cpp"
+    proj.mkdir()
+    run_git(proj, "init")
+    target = proj / "ggml-metal-device.m"
+    target.write_text("line one\nline two\n")
+    run_git(proj, "add", ".")
+    run_git(
+        proj,
+        "-c",
+        "user.email=test@example.com",
+        "-c",
+        "user.name=test",
+        "commit",
+        "-m",
+        "init",
+    )
+
+    target.write_text("line one\nline two patched\n")
+    patch_dir = tmp_path / "patches" / "llama.cpp"
+    patch_dir.mkdir(parents=True)
+    patch_file = patch_dir / "0001-test.patch"
+    patch_file.write_text(run_git(proj, "diff").stdout)
+    run_git(proj, "checkout", "--", ".")
+
+    monkeypatch.setattr(build, "PROJECT", proj)
+    monkeypatch.setattr(build, "PATCH_DIR", patch_dir)
+    return proj, target, patch_file
+
+
+def git_is_clean(proj: Path) -> bool:
+    return run_git(proj, "status", "--porcelain").stdout == ""
+
+
+def test_no_patch_dir(tmp_path, monkeypatch):
+    monkeypatch.setattr(build, "PATCH_DIR", tmp_path / "does-not-exist")
+    assert build.llamacpp_patches() == []
+
+
+def test_apply_then_revert_leaves_tree_clean(fake_llamacpp):
+    proj, target, patch_file = fake_llamacpp
+
+    patches = build.llamacpp_patches()
+    assert patches == [patch_file]
+
+    applied = build.apply_llamacpp_patches(patches)
+    assert applied == [patch_file]
+    assert target.read_text() == "line one\nline two patched\n"
+
+    build.revert_llamacpp_patches(applied)
+    assert target.read_text() == "line one\nline two\n"
+    assert git_is_clean(proj)
+
+
+def test_apply_is_idempotent(fake_llamacpp):
+    proj, target, patch_file = fake_llamacpp
+
+    applied_first = build.apply_llamacpp_patches(build.llamacpp_patches())
+    assert applied_first == [patch_file]
+
+    # second build against an already-patched tree: skipped, not reverted
+    applied_second = build.apply_llamacpp_patches(build.llamacpp_patches())
+    assert applied_second == []
+    assert target.read_text() == "line one\nline two patched\n"
+
+    build.revert_llamacpp_patches(applied_first)
+    assert git_is_clean(proj)
+
+
+def test_revert_only_what_was_applied(fake_llamacpp):
+    proj, target, patch_file = fake_llamacpp
+
+    build.apply_llamacpp_patches(build.llamacpp_patches())
+    # simulate an interrupted build: patch left applied, next build skips it
+    applied = build.apply_llamacpp_patches(build.llamacpp_patches())
+    build.revert_llamacpp_patches(applied)
+    # nothing was applied by this run, so nothing is reverted either
+    assert target.read_text() == "line one\nline two patched\n"
+
+
+def test_inapplicable_patch_fails_loudly(fake_llamacpp, tmp_path):
+    proj, target, patch_file = fake_llamacpp
+    patch_file.write_text(
+        "diff --git a/ggml-metal-device.m b/ggml-metal-device.m\n"
+        "--- a/ggml-metal-device.m\n"
+        "+++ b/ggml-metal-device.m\n"
+        "@@ -1,2 +1,2 @@\n"
+        "-this content does not exist\n"
+        "-neither does this\n"
+        "+garbage\n"
+        "+patch\n"
+    )
+    with pytest.raises(subprocess.CalledProcessError):
+        build.apply_llamacpp_patches(build.llamacpp_patches())