diff --git a/docs/notes/2.34.x.md b/docs/notes/2.34.x.md index 9bb55a3d5f6..bc0b20e80f9 100644 --- a/docs/notes/2.34.x.md +++ b/docs/notes/2.34.x.md @@ -48,6 +48,8 @@ Fixed a bug that caused `generate-lockfiles --sync` not to have its intended eff Interpreter constraints can now select a specific CPython ABI, to distinguish the free-threaded (no-GIL) build from the standard build. Qualify the interpreter type using either a PEP 508 extra, `CPython[free-threaded]` or `CPython[gil]`, or the equivalent `CPython+t` / `CPython-t` spellings. Both spellings are adopted from Pex. For example, `interpreter_constraints=["CPython[free-threaded]==3.14.*"]` matches only free-threaded 3.14 interpreters, while `CPython==3.14.*` continues to match either ABI. +Pant's integration with mypy requires a single sqlite `.db` to atomically manage. Starting in [version 2.0](https://mypy-lang.blogspot.com/2026/05/mypy-20-relased.html) mypy defaults to sharding across multiple db files. When using versions of mypy >=2 Pants will no unconditionally set `--sqlite-num-shards=1` so cacheing continues to work. No user action or configuration change is needed. + The default version of [Pex](https://github.com/pex-tool/pex) used by the Python backend has been upgraded to [`v2.97.3`](https://github.com/pex-tool/pex/releases/tag/v2.97.3). Of particular note for Pants users: - [Fix concurrent use of artifact downloads](https://github.com/pex-tool/pex/pull/3207). diff --git a/src/python/pants/backend/python/typecheck/mypy/rules.py b/src/python/pants/backend/python/typecheck/mypy/rules.py index 42644e905d7..482d4753d60 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules.py @@ -4,6 +4,7 @@ from __future__ import annotations import dataclasses +import logging from collections.abc import Iterable from dataclasses import dataclass from hashlib import sha256 @@ -65,7 +66,9 @@ from pants.option.global_options import GlobalOptions from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet, OrderedSet -from pants.util.strutil import pluralize, shell_quote +from pants.util.strutil import pluralize, shell_quote, softwrap + +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -89,6 +92,12 @@ class MyPyRequest(CheckRequest): tool_name = MyPy.options_scope +def _user_supplied_sqlite_num_shards(args: Iterable[str]) -> bool: + return any( + arg == "--sqlite-num-shards" or arg.startswith("--sqlite-num-shards=") for arg in args + ) + + def _get_cache_args( mypy_version: packaging.version.Version, python_version: str | None, @@ -100,7 +109,7 @@ def _get_cache_args( and python_version is not None and cache_mode == MyPyCacheMode.sqlite ): - return ( + args: tuple[str, ...] = ( # Skip mtime checks because we don't propagate mtime when materializing the # sandbox, so the mtime checks will always fail otherwise. "--skip-cache-mtime-check", @@ -109,6 +118,12 @@ def _get_cache_args( "--cache-dir", cache_dir, ) + if mypy_version >= packaging.version.Version("2.0"): + # mypy >= 2.0 shards the sqlite cache into cache.{i}.db files (16 + # by default, ). The + # copy-back scheme here depends on there being exactly one file. + args += ("--sqlite-num-shards=1",) + return args else: return ("--cache-dir=/dev/null",) @@ -131,7 +146,18 @@ async def _generate_argv( mypy_pex_info = await determine_venv_pex_resolve_info(pex) mypy_info = mypy_pex_info.find("mypy") assert mypy_info is not None - args.extend(_get_cache_args(mypy_info.version, python_version, mypy.cache_mode, cache_dir)) + cache_args = _get_cache_args(mypy_info.version, python_version, mypy.cache_mode, cache_dir) + if "--sqlite-num-shards=1" in cache_args and _user_supplied_sqlite_num_shards(mypy.args): + logger.warning( + softwrap( + """ + `--sqlite-num-shards` set in `[mypy].args`, but Pants manages + mypy's sqlite cache as a single file and will use + `--sqlite-num-shards=1` instead. + """ + ) + ) + args.extend(cache_args) args.append(f"@{file_list_path}") return tuple(args) diff --git a/src/python/pants/backend/python/typecheck/mypy/rules_test.py b/src/python/pants/backend/python/typecheck/mypy/rules_test.py index e845cf9c9fb..2f4129291f1 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules_test.py @@ -5,23 +5,41 @@ import packaging.version -from pants.backend.python.typecheck.mypy.rules import _get_cache_args, determine_python_files +from pants.backend.python.typecheck.mypy.rules import ( + _get_cache_args, + _user_supplied_sqlite_num_shards, + determine_python_files, +) from pants.backend.python.typecheck.mypy.subsystem import MyPyCacheMode def test_get_cache_args() -> None: modern_mypy = packaging.version.Version("1.0") old_mypy = packaging.version.Version("0.600") + sharded_mypy = packaging.version.Version("2.0") args = _get_cache_args(modern_mypy, "3.12", MyPyCacheMode.sqlite, "/cache") assert "--sqlite-cache" in args assert "--skip-cache-mtime-check" in args assert "--cache-dir" in args assert "/cache" in args + assert "--sqlite-num-shards=1" not in args + + args = _get_cache_args(sharded_mypy, "3.12", MyPyCacheMode.sqlite, "/cache") + assert "--sqlite-cache" in args + assert "--sqlite-num-shards=1" in args + + args = _get_cache_args( + packaging.version.Version("2.3.0"), "3.12", MyPyCacheMode.sqlite, "/cache" + ) + assert "--sqlite-num-shards=1" in args args = _get_cache_args(modern_mypy, "3.12", MyPyCacheMode.none, "/cache") assert args == ("--cache-dir=/dev/null",) + args = _get_cache_args(sharded_mypy, "3.12", MyPyCacheMode.none, "/cache") + assert args == ("--cache-dir=/dev/null",) + args = _get_cache_args(old_mypy, "3.12", MyPyCacheMode.sqlite, "/cache") assert args == ("--cache-dir=/dev/null",) @@ -29,6 +47,15 @@ def test_get_cache_args() -> None: assert args == ("--cache-dir=/dev/null",) +def test_user_supplied_sqlite_num_shards() -> None: + assert not _user_supplied_sqlite_num_shards([]) + assert not _user_supplied_sqlite_num_shards(["--strict", "--pretty"]) + assert not _user_supplied_sqlite_num_shards(["--sqlite-num-shards-like"]) + assert _user_supplied_sqlite_num_shards(["--sqlite-num-shards=8"]) + assert _user_supplied_sqlite_num_shards(["--sqlite-num-shards", "8"]) + assert _user_supplied_sqlite_num_shards(["--strict", "--sqlite-num-shards=16"]) + + def test_determine_python_files() -> None: assert determine_python_files([]) == () assert determine_python_files(["f.py"]) == ("f.py",)