diff --git a/bin/generate_schema.py b/bin/generate_schema.py index ad8304969..e98f41b8b 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -40,6 +40,28 @@ description: cibuildwheel's settings. type: object properties: + inherit: + type: object + additionalProperties: false + properties: + audit-command: {"$ref": "#/$defs/inherit"} + audit-requires: {"$ref": "#/$defs/inherit"} + before-all: {"$ref": "#/$defs/inherit"} + before-build: {"$ref": "#/$defs/inherit"} + xbuild-tools: {"$ref": "#/$defs/inherit"} + xbuild-files: {"$ref": "#/$defs/inherit"} + before-test: {"$ref": "#/$defs/inherit"} + config-settings: {"$ref": "#/$defs/inherit"} + container-engine: {"$ref": "#/$defs/inherit"} + environment: {"$ref": "#/$defs/inherit"} + environment-pass: {"$ref": "#/$defs/inherit"} + repair-wheel-command: {"$ref": "#/$defs/inherit"} + test-command: {"$ref": "#/$defs/inherit"} + test-extras: {"$ref": "#/$defs/inherit"} + test-sources: {"$ref": "#/$defs/inherit"} + test-requires: {"$ref": "#/$defs/inherit"} + test-environment: {"$ref": "#/$defs/inherit"} + test-runtime: {"$ref": "#/$defs/inherit"} audit-command: description: Execute a shell command to audit each wheel after it is repaired. Use {wheel} for each wheel path, or {abi3_wheel} to only audit abi3 wheels. type: string_array @@ -315,28 +337,6 @@ additionalProperties: false properties: select: {} - inherit: - type: object - additionalProperties: false - properties: - audit-command: {"$ref": "#/$defs/inherit"} - audit-requires: {"$ref": "#/$defs/inherit"} - before-all: {"$ref": "#/$defs/inherit"} - before-build: {"$ref": "#/$defs/inherit"} - xbuild-tools: {"$ref": "#/$defs/inherit"} - xbuild-files: {"$ref": "#/$defs/inherit"} - before-test: {"$ref": "#/$defs/inherit"} - config-settings: {"$ref": "#/$defs/inherit"} - container-engine: {"$ref": "#/$defs/inherit"} - environment: {"$ref": "#/$defs/inherit"} - environment-pass: {"$ref": "#/$defs/inherit"} - repair-wheel-command: {"$ref": "#/$defs/inherit"} - test-command: {"$ref": "#/$defs/inherit"} - test-extras: {"$ref": "#/$defs/inherit"} - test-sources: {"$ref": "#/$defs/inherit"} - test-requires: {"$ref": "#/$defs/inherit"} - test-environment: {"$ref": "#/$defs/inherit"} - test-runtime: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 8dd21a1a6..afac21d06 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -50,7 +50,13 @@ from cibuildwheel.selector import BuildSelector, EnableGroup, TestSelector, selector_matches from cibuildwheel.typing import PLATFORMS, PlatformName from cibuildwheel.util import resources -from cibuildwheel.util.helpers import format_safe, parse_key_value_string, strtobool, unwrap +from cibuildwheel.util.helpers import ( + format_safe, + parse_arbitrary_key_value_string, + parse_key_value_string, + strtobool, + unwrap, +) from cibuildwheel.util.packaging import DependencyConstraints TYPE_CHECKING = False @@ -436,6 +442,28 @@ def _stringify_setting( return setting +def parse_inherit(config: str | dict[str, str] | None) -> dict[str, InheritRule]: + inherit_dict: dict[str, str] + + if config is None: + return {} + + if isinstance(config, str): + parsed = parse_arbitrary_key_value_string(config, default_value="append") + inherit_dict = {k: "".join(v) for k, v in parsed.items()} + elif isinstance(config, dict): + inherit_dict = config + else: + msg = "'inherit' must be a string or a table" + raise OptionsReaderError(msg) + + if not all(v in {"none", "append", "prepend"} for v in inherit_dict.values()): + msg = "'inherit' must contain only {'none', 'append', 'prepend'} values" + raise OptionsReaderError(msg) + + return {k: InheritRule[v.upper()] for k, v in inherit_dict.items()} + + class OptionsReader: """ Gets options from the environment, config or defaults, optionally scoped @@ -484,45 +512,18 @@ def __init__( self._validate_platform_option(option_name) self.config_options = config_options + self.config_options_inherit = parse_inherit(config_options.get("inherit")) self.config_platform_options = config_platform_options + self.config_platform_options_inherit = parse_inherit(config_platform_options.get("inherit")) - self.overrides: list[Override] = [] self.current_identifier: str | None = None - config_overrides = self.config_options.get("overrides") - - if config_overrides is not None: - if not isinstance(config_overrides, list): - msg = "'tool.cibuildwheel.overrides' must be a list" - raise OptionsReaderError(msg) - - for config_override in config_overrides: - select = config_override.pop("select", None) - - if not select: - msg = "'select' must be set in an override" - raise OptionsReaderError(msg) - - if isinstance(select, list): - select = " ".join(select) - - inherit = config_override.pop("inherit", {}) - if not isinstance(inherit, dict) or not all( - i in {"none", "append", "prepend"} for i in inherit.values() - ): - msg = "'inherit' must be a dict containing only {'none', 'append', 'prepend'} values" - raise OptionsReaderError(msg) - - inherit_enum = {k: InheritRule[v.upper()] for k, v in inherit.items()} - - self.overrides.append(Override(select, config_override, inherit_enum)) - def _validate_global_option(self, name: str) -> None: """ Raises an error if an option with this name is not allowed in the [tool.cibuildwheel] section of a config file. """ - allowed_option_names = self.default_options.keys() | PLATFORMS | {"overrides"} + allowed_option_names = self.default_options.keys() | PLATFORMS | {"inherit", "overrides"} if name not in allowed_option_names: msg = f"Option {name!r} not supported in a config file." @@ -541,7 +542,9 @@ def _validate_platform_option(self, name: str) -> None: msg = f"{name!r} is not allowed in {disallowed_platform_options}" raise OptionsReaderError(msg) - allowed_option_names = self.default_options.keys() | self.default_platform_options.keys() + allowed_option_names = ( + self.default_options.keys() | self.default_platform_options.keys() | {"inherit"} + ) if name not in allowed_option_names: msg = f"Option {name!r} not supported in the {self.platform!r} section" @@ -562,6 +565,56 @@ def _load_file(self, filename: Path) -> tuple[dict[str, Any], dict[str, Any]]: return global_options, platform_options + @functools.cached_property + def overrides(self) -> list[Override]: + config_overrides = self.config_options.get("overrides") + overrides: list[Override] = [] + + if config_overrides is not None: + if not isinstance(config_overrides, list): + msg = "'tool.cibuildwheel.overrides' must be a list" + raise OptionsReaderError(msg) + + for config_override in config_overrides: + select = config_override.pop("select", None) + + if not select: + msg = "'select' must be set in an override" + raise OptionsReaderError(msg) + + if isinstance(select, list): + select = " ".join(select) + + inherit = config_override.pop("inherit", {}) + + overrides.append(Override(select, config_override, parse_inherit(inherit))) + + return overrides + + @functools.cached_property + def env_inherit(self) -> dict[str, InheritRule]: + env_inherit_str = self.env.get("CIBW_INHERIT", "") + try: + return parse_inherit(env_inherit_str) + except OptionsReaderError as e: + msg = f"Failed to parse CIBW_INHERIT environment variable. {e}" + raise errors.ConfigurationError(msg) from e + + @functools.cached_property + def env_platform_inherit(self) -> dict[str, InheritRule]: + env_inherit = self.env_inherit + + # find the rules which have -{platform} on the end of their key, + # remove the platform suffix from the key and return the resulting + # rule. + platform_suffix = f"-{self.platform}" + + return { + key.removesuffix(platform_suffix): value + for key, value in env_inherit.items() + if key.endswith(platform_suffix) + } + @property def active_config_overrides(self) -> list[Override]: if self.current_identifier is None: @@ -585,7 +638,7 @@ def get( env_plat: bool = True, option_format: OptionFormat | None = None, ignore_empty: bool = False, - env_rule: InheritRule = InheritRule.NONE, + default_env_rule: InheritRule = InheritRule.NONE, ) -> str: """ Get and return the value for the named option from environment, @@ -609,16 +662,37 @@ def get( # get the option from the default, then the config file, then finally the environment. # platform-specific options are preferred, if they're allowed. return _resolve_cascade( - (self.default_options.get(name), InheritRule.NONE), - (self.default_platform_options.get(name), InheritRule.NONE), - (self.config_options.get(name), InheritRule.NONE), - (self.config_platform_options.get(name), InheritRule.NONE), + ( + self.default_options.get(name), + InheritRule.NONE, + ), + ( + self.default_platform_options.get(name), + InheritRule.NONE, + ), + ( + self.config_options.get(name), + self.config_options_inherit.get(name, InheritRule.NONE), + ), + ( + self.config_platform_options.get(name), + self.config_platform_options_inherit.get(name, InheritRule.NONE), + ), *[ - (o.options.get(name), o.inherit.get(name, InheritRule.NONE)) + ( + o.options.get(name), + o.inherit.get(name, InheritRule.NONE), + ) for o in self.active_config_overrides ], - (self.env.get(envvar), env_rule), - (self.env.get(plat_envvar) if env_plat else None, env_rule), + ( + self.env.get(envvar), + self.env_inherit.get(name, default_env_rule), + ), + ( + self.env.get(plat_envvar) if env_plat else None, + self.env_platform_inherit.get(name, default_env_rule), + ), ignore_empty=ignore_empty, option_format=option_format, ) @@ -691,7 +765,10 @@ def globals(self) -> GlobalOptions: allow_empty = args.allow_empty or strtobool(self.env.get("CIBW_ALLOW_EMPTY", "0")) enable_groups = self.reader.get( - "enable", env_plat=False, option_format=ListFormat(sep=" "), env_rule=InheritRule.APPEND + "enable", + env_plat=False, + option_format=ListFormat(sep=" "), + default_env_rule=InheritRule.APPEND, ) try: enable = { @@ -797,12 +874,11 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: if xbuild_tools == ["\u0000"]: xbuild_tools = None - xbuild_files = parse_key_value_string( + xbuild_files = parse_arbitrary_key_value_string( self.reader.get( "xbuild-files", option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ), - kw_arg_names=["*"], ) test_sources = shlex.split( diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 5795aa2bc..7bce36d36 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -26,6 +26,67 @@ "description": "cibuildwheel's settings.", "type": "object", "properties": { + "inherit": { + "type": "object", + "additionalProperties": false, + "properties": { + "audit-command": { + "$ref": "#/$defs/inherit" + }, + "audit-requires": { + "$ref": "#/$defs/inherit" + }, + "before-all": { + "$ref": "#/$defs/inherit" + }, + "before-build": { + "$ref": "#/$defs/inherit" + }, + "xbuild-tools": { + "$ref": "#/$defs/inherit" + }, + "xbuild-files": { + "$ref": "#/$defs/inherit" + }, + "before-test": { + "$ref": "#/$defs/inherit" + }, + "config-settings": { + "$ref": "#/$defs/inherit" + }, + "container-engine": { + "$ref": "#/$defs/inherit" + }, + "environment": { + "$ref": "#/$defs/inherit" + }, + "environment-pass": { + "$ref": "#/$defs/inherit" + }, + "repair-wheel-command": { + "$ref": "#/$defs/inherit" + }, + "test-command": { + "$ref": "#/$defs/inherit" + }, + "test-extras": { + "$ref": "#/$defs/inherit" + }, + "test-sources": { + "$ref": "#/$defs/inherit" + }, + "test-requires": { + "$ref": "#/$defs/inherit" + }, + "test-environment": { + "$ref": "#/$defs/inherit" + }, + "test-runtime": { + "$ref": "#/$defs/inherit" + } + }, + "title": "CIBW_INHERIT" + }, "audit-command": { "description": "Execute a shell command to audit each wheel after it is repaired. Use {wheel} for each wheel path, or {abi3_wheel} to only audit abi3 wheels.", "oneOf": [ @@ -689,64 +750,7 @@ ] }, "inherit": { - "type": "object", - "additionalProperties": false, - "properties": { - "audit-command": { - "$ref": "#/$defs/inherit" - }, - "audit-requires": { - "$ref": "#/$defs/inherit" - }, - "before-all": { - "$ref": "#/$defs/inherit" - }, - "before-build": { - "$ref": "#/$defs/inherit" - }, - "xbuild-tools": { - "$ref": "#/$defs/inherit" - }, - "xbuild-files": { - "$ref": "#/$defs/inherit" - }, - "before-test": { - "$ref": "#/$defs/inherit" - }, - "config-settings": { - "$ref": "#/$defs/inherit" - }, - "container-engine": { - "$ref": "#/$defs/inherit" - }, - "environment": { - "$ref": "#/$defs/inherit" - }, - "environment-pass": { - "$ref": "#/$defs/inherit" - }, - "repair-wheel-command": { - "$ref": "#/$defs/inherit" - }, - "test-command": { - "$ref": "#/$defs/inherit" - }, - "test-extras": { - "$ref": "#/$defs/inherit" - }, - "test-sources": { - "$ref": "#/$defs/inherit" - }, - "test-requires": { - "$ref": "#/$defs/inherit" - }, - "test-environment": { - "$ref": "#/$defs/inherit" - }, - "test-runtime": { - "$ref": "#/$defs/inherit" - } - } + "$ref": "#/properties/inherit" }, "audit-command": { "$ref": "#/properties/audit-command" @@ -875,6 +879,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, @@ -1014,6 +1021,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, @@ -1099,6 +1109,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, @@ -1184,6 +1197,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, @@ -1256,6 +1272,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, @@ -1341,6 +1360,9 @@ "type": "object", "additionalProperties": false, "properties": { + "inherit": { + "$ref": "#/properties/inherit" + }, "audit-command": { "$ref": "#/properties/audit-command" }, diff --git a/cibuildwheel/util/helpers.py b/cibuildwheel/util/helpers.py index d6ed17971..1121c7244 100644 --- a/cibuildwheel/util/helpers.py +++ b/cibuildwheel/util/helpers.py @@ -111,7 +111,7 @@ def parse_key_value_string( if kw_arg_names is None: kw_arg_names = [] - all_field_names = None if ("*" in kw_arg_names) else [*positional_arg_names, *kw_arg_names] + all_field_names = [*positional_arg_names, *kw_arg_names] shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";") shlexer.commenters = "" @@ -128,7 +128,7 @@ def parse_key_value_string( # check to see if the option name is specified field_name, sep, first_value = field[0].partition(":") if sep: - if (all_field_names is not None) and (field_name not in all_field_names): + if field_name not in all_field_names: msg = f"Failed to parse {key_value_string!r}. Unknown field name {field_name!r}" raise ValueError(msg) @@ -147,6 +147,51 @@ def parse_key_value_string( return dict(result) +def parse_arbitrary_key_value_string( + key_value_string: str, default_value: str | None = None +) -> dict[str, list[str]]: + """ + Parses a string like + + "before-build; before-test: append; after-test: prepend" + or + "package1: some/header.h some/library.a; package2: other/header.h" + + There are no restrictions on the keys than can be set. + + No positional arguments are allowed. Words without a colon attached are + interpreted as keys. Keys without a value will be assigned the + default_value if provided, otherwise throw an error. + """ + shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";") + shlexer.commenters = "" + shlexer.whitespace_split = True + parts = list(shlexer) + # parts now looks like + # ['before-build', ';', 'before-test:', 'append', ';', 'after-test:', 'prepend'] + + # split by semicolon + result: defaultdict[str, list[str]] = defaultdict(list) + fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k] + for field in fields: + # check to see if the option name is specified + field_name, sep, first_value = field[0].partition(":") + if sep: + # the colon was present, so the first value is the value after the colon + values = ([first_value] if first_value else []) + field[1:] + result[field_name] += values + else: + # no colon, so it's a key (or set of keys) without values + if default_value is None: + msg = f"Failed to parse {key_value_string!r}. No value specified for {field_name!r}. Expected ':' followed by a value." + raise ValueError(msg) + + for key in field: + result[key].append(default_value) + + return dict(result) + + @dataclasses.dataclass(order=True) class FlexibleVersion: version_parts: tuple[int, ...] = dataclasses.field(init=False, repr=False) diff --git a/docs/configuration.md b/docs/configuration.md index 552772eb2..6a3576383 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -66,10 +66,11 @@ placed in `[tool.cibuildwheel]` and are lower case, with dashes, following common [TOML](https://toml.io) practice. Anything placed in subsections named after a platform will only affect those platforms. Platform-specific values replace the corresponding global value for that platform; table options -are not merged key by key. Lists can be used instead of strings for items that -are naturally a list. Multiline strings also work just like in the environment -variables. Environment variable overrides, such as `CIBW_TEST_COMMAND` and -`CIBW_TEST_COMMAND_LINUX`, will take precedence if defined. +are not merged key by key unless you configure [inheritance](#inherit). Lists can +be used instead of strings for items that are naturally a list. Multiline strings +also work just like in the environment variables. Environment variable overrides, +such as `CIBW_TEST_COMMAND` and `CIBW_TEST_COMMAND_LINUX`, take precedence if +defined. The example above using environment variables could have been written like this: @@ -113,9 +114,9 @@ trigger new containers, one per image. The ``output-dir``, ``build``, ``skip``, ``test_skip`` selectors, and architectures cannot be overridden. -You can specify a table of overrides in `inherit={}`, any list or table in this -list will inherit from previous overrides or the main configuration. The valid -options are `"none"` (the default), `"append"`, and `"prepend"`. +By default, values in an override replace values from the main configuration or +earlier overrides. You can instead [extend a list or table option](#inherit) by +setting an `inherit` rule for it. #### Examples: @@ -175,14 +176,50 @@ This example will provide the command `"pyproject-before && pyproject && pyproje on Python 3.11, and will have `environment = {FOO="BAZ", "PYTHON"="MONTY", "HAM"="EGGS"}`. -## Extending existing options {: #inherit } +## Option inheritance {: #inherit } -In the TOML configuration, you can choose how tables and lists are inherited. -By default, all values are overridden completely (`"none"`) but sometimes you'd -rather `"append"` or `"prepend"` to an existing list or table. You can do this -with the `inherit` table in overrides. For example, if you want to add an environment -variable for CPython 3.11, without `inherit` you'd have to repeat all the -original environment variables in the override. With `inherit`, it's just: +As cibuildwheel reads its configuration, each layer normally replaces the value +from the previous layer. The layers, from lowest to highest precedence, are: + +1. cibuildwheel's defaults +2. `[tool.cibuildwheel]` +3. `[tool.cibuildwheel.]` +4. matching `[[tool.cibuildwheel.overrides]]` entries, in order +5. `CIBW_