feat: add CPython 3.10 pre-release support (#675)

* Add CPython 3.10 support

* Fix logger displaying CPython 3.1 instead of 3.10

* Fix tests failing with CPython 3.10

* Use pytest instead of nose

* feat: --pre flag

Apply suggestions from code review

Co-authored-by: Matthieu Darbois <[email protected]>

* fix: update Python update script to process beta versions

* refactor: prerelease-pythons

* Use `strtobool` to parse `CIBW_PRERELEASE_PYTHONS` env var

* Update python version filtering for universal2 & arm64

* Filter out CPython 3.10 and above for `test_manylinuxXXXX_only[manylinux1]` test

* Use CIBW_BUILD filtering rather than CIBW_SKIP for test_docker_images

* Use skip_patterns to filter out pre-releases

* Use `prerelease_pythons` instead of `pre`

* Reword `CIBW_PRERELEASE_PYTHONS` doc per review.

* Use `CIBW_PRERELEASE_PYTHONS: True` for usage example.

* Update `cibuildwheel --help` doc

* docs: add note on spec.filter

* Clean up the BuildSelector __repr__ by refactoring

* fix: remove platform variants for CIBW_PRERELEASE_PYTHONS

* docs: mention the flag

Co-authored-by: Henry Schreiner <[email protected]>
Co-authored-by: Joe Rickerby <[email protected]>

Author: 3 people

README.md

@@ -28,9 +28,11 @@ What does it do?
 | CPython 3.7     | ✅ | N/A | ✅  | ✅  | ✅ | ✅ | ✅ | ✅ | ✅ |
 | CPython 3.8     | ✅ | N/A¹ | ✅  | ✅  | ✅ | ✅ | ✅ | ✅ | ✅ |
 | CPython 3.9     | ✅ | ✅ | ✅  | ✅  | ✅ | ✅ | ✅ | ✅ | ✅ |
+| CPython 3.10²   | ✅ | ✅ | ✅  | ✅  | ✅ | ✅ | ✅ | ✅ | ✅ |
 | PyPy 3.7 v7.3   | ✅ | N/A | ✅  | N/A | ✅ | ✅  | ✅  | N/A | N/A |
 
 <sup>¹ CPython 3.8's final binary release has experimental Universal2 support, but does not support macOS 10.x, so this is not currently available.</sup><br>
+<sup>² Available as a prerelease under a [flag](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons)</sup><br>
 
 - Builds manylinux, macOS 10.9+, and Windows wheels for CPython and PyPy
 - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, and GitLab CI
@@ -108,6 +110,7 @@ Options
 |   | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  <br> [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  | Choose the Python versions to build |
 |   | [`CIBW_ARCHS`](https://cibuildwheel.readthedocs.io/en/stable/options/#archs)  | Change the architectures built on your machine by default |
 |   | [`CIBW_PROJECT_REQUIRES_PYTHON`](https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python)  | Manually set the Python compatibility of your project |
+|   | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons)  | Enable building with pre-release versions of Python |
 | **Build customization** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
 |   | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all)  | Execute a shell command on the build system before any wheels are built. |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |

bin/update_pythons.py

@@ -6,7 +6,7 @@
 import difflib
 import logging
 from pathlib import Path
-from typing import Any, Union
+from typing import Any, Iterable, Union, cast
 
 import click
 import requests
@@ -79,23 +79,26 @@ def __init__(self, arch_str: ArchStr) -> None:
         response.raise_for_status()
         cp_info = response.json()
 
-        versions = (Version(v) for v in cp_info["versions"])
-        self.versions = sorted(v for v in versions if not v.is_devrelease)
+        self.version_dict = {Version(v): v for v in cp_info["versions"]}
 
     def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None:
-        versions = sorted(v for v in self.versions if spec.contains(v))
-        if not all(v.is_prerelease for v in versions):
-            versions = [v for v in versions if not v.is_prerelease]
+
+        # Specifier.filter selects all non pre-releases that match the spec,
+        # unless there are only pre-releases, then it selects pre-releases
+        # instead (like pip)
+        unsorted_versions = cast(Iterable[Version], spec.filter(self.version_dict))
+        versions = sorted(unsorted_versions, reverse=True)
+
         log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}")
 
         if not versions:
             return None
 
-        version = versions[-1]
+        version = versions[0]
         identifier = f"cp{version.major}{version.minor}-{self.arch}"
         return ConfigWinCP(
             identifier=identifier,
-            version=str(version),
+            version=self.version_dict[version],
             arch=self.arch_str,
         )
 
@@ -190,21 +193,23 @@ def __init__(self) -> None:
             # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ")
             version = Version(release["name"][7:])
 
-            if not version.is_prerelease and not version.is_devrelease:
-                uri = int(release["resource_uri"].rstrip("/").split("/")[-1])
-                self.versions_dict[version] = uri
+            uri = int(release["resource_uri"].rstrip("/").split("/")[-1])
+            self.versions_dict[version] = uri
 
     def update_version_macos(
         self, identifier: str, version: Version, spec: Specifier
     ) -> ConfigMacOS | None:
-        sorted_versions = sorted(v for v in self.versions_dict if spec.contains(v))
+
+        # see note above on Specifier.filter
+        unsorted_versions = cast(Iterable[Version], spec.filter(self.versions_dict))
+        sorted_versions = sorted(unsorted_versions, reverse=True)
 
         if version <= Version("3.8.9999"):
             file_ident = "macosx10.9.pkg"
         else:
             file_ident = "macos11.pkg"
 
-        for new_version in reversed(sorted_versions):
+        for new_version in sorted_versions:
             # Find the first patch version that contains the requested file
             uri = self.versions_dict[new_version]
             response = requests.get(
@@ -270,7 +275,7 @@ def update_config(self, config: dict[str, str]) -> None:
 @click.command()
 @click.option("--force", is_flag=True)
 @click.option(
-    "--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False)
+    "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False)
 )
 def update_pythons(force: bool, level: str) -> None:
 

cibuildwheel/__main__.py

@@ -132,6 +132,12 @@ def main() -> None:
         help="Do not report an error code if the build does not match any wheels.",
     )
 
+    parser.add_argument(
+        "--prerelease-pythons",
+        action="store_true",
+        help="Enable pre-release Python versions if available.",
+    )
+
     args = parser.parse_args()
 
     detect_obsolete_options()
@@ -206,6 +212,9 @@ def main() -> None:
     build_verbosity_str = get_option_from_environment(
         "CIBW_BUILD_VERBOSITY", platform=platform, default=""
     )
+    prerelease_pythons = args.prerelease_pythons or cibuildwheel.util.strtobool(
+        os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
+    )
 
     package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
 
@@ -222,8 +231,12 @@ def main() -> None:
     ) or get_requires_python_str(package_dir)
     requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
 
+    # Hardcode pre-releases here, current: Python 3.10
     build_selector = BuildSelector(
-        build_config=build_config, skip_config=skip_config, requires_python=requires_python
+        build_config=build_config,
+        skip_config=skip_config,
+        requires_python=requires_python,
+        prerelease_pythons=prerelease_pythons,
     )
     test_selector = TestSelector(skip_config=test_skip)
 

cibuildwheel/logger.py

@@ -186,7 +186,7 @@ def build_description_from_identifier(identifier: str) -> str:
     build_description = ""
 
     python_interpreter = python_identifier[0:2]
-    python_version = python_identifier[2:4]
+    python_version = python_identifier[2:]
 
     if python_interpreter == "cp":
         build_description += "CPython"
@@ -195,7 +195,7 @@ def build_description_from_identifier(identifier: str) -> str:
     else:
         raise Exception("unknown python")
 
-    build_description += f" {python_version[0]}.{python_version[1]} "
+    build_description += f" {python_version[0]}.{python_version[1:]} "
 
     try:
         build_description += PLATFORM_IDENTIFIER_DESCIPTIONS[platform_identifier]

cibuildwheel/macos.py

@@ -270,7 +270,7 @@ def setup_python(
     config_is_arm64 = python_configuration.identifier.endswith("arm64")
     config_is_universal2 = python_configuration.identifier.endswith("universal2")
 
-    if python_configuration.version == "3.9":
+    if python_configuration.version not in {"3.6", "3.7", "3.8"}:
         if python_configuration.identifier.endswith("x86_64"):
             # even on the macos11.0 Python installer, on the x86_64 side it's
             # compatible back to 10.9.

cibuildwheel/resources/build-platforms.toml

@@ -4,23 +4,28 @@ python_configurations = [
   { identifier = "cp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
   { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-manylinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
   { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
   { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
   { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-manylinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" },
   { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" },
   { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" },
+  { identifier = "cp310-manylinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" },
   { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
   { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" },
 ]
@@ -33,6 +38,9 @@ python_configurations = [
   { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" },
   { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" },
   { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" },
+  { identifier = "cp310-macosx_x86_64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" },
+  { identifier = "cp310-macosx_arm64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" },
+  { identifier = "cp310-macosx_universal2", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" },
   { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-osx64.tar.bz2" },
 ]
 
@@ -46,5 +54,7 @@ python_configurations = [
   { identifier = "cp38-win_amd64", version = "3.8.10", arch = "64" },
   { identifier = "cp39-win32", version = "3.9.5", arch = "32" },
   { identifier = "cp39-win_amd64", version = "3.9.5", arch = "64" },
+  { identifier = "cp310-win32", version = "3.10.0-b1", arch = "32" },
+  { identifier = "cp310-win_amd64", version = "3.10.0-b1", arch = "64" },
   { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-win64.zip" },
 ]

cibuildwheel/util.py

@@ -58,15 +58,25 @@ class IdentifierSelector:
     This class holds a set of build/skip patterns. You call an instance with a
     build identifier, and it returns True if that identifier should be
     included. Only call this on valid identifiers, ones that have at least 2
-    numeric digits before the first dash.
+    numeric digits before the first dash. If a pre-release version X.Y is present,
+    you can filter it with prerelease="XY".
     """
 
+    # a pattern that skips prerelease versions, when include_prereleases is False.
+    PRERELEASE_SKIP = "cp310-*"
+
     def __init__(
-        self, *, build_config: str, skip_config: str, requires_python: Optional[SpecifierSet] = None
+        self,
+        *,
+        build_config: str,
+        skip_config: str,
+        requires_python: Optional[SpecifierSet] = None,
+        prerelease_pythons: bool = False,
     ):
         self.build_patterns = build_config.split()
         self.skip_patterns = skip_config.split()
         self.requires_python = requires_python
+        self.prerelease_pythons = prerelease_pythons
 
     def __call__(self, build_id: str) -> bool:
         # Filter build selectors by python_requires if set
@@ -81,17 +91,33 @@ def __call__(self, build_id: str) -> bool:
         build_patterns = itertools.chain.from_iterable(
             bracex.expand(p) for p in self.build_patterns
         )
-        skip_patterns = itertools.chain.from_iterable(bracex.expand(p) for p in self.skip_patterns)
+
+        unexpanded_skip_patterns = self.skip_patterns.copy()
+
+        if not self.prerelease_pythons:
+            # filter out the prerelease pythons, alongside the user-defined
+            # skip patterns
+            unexpanded_skip_patterns += BuildSelector.PRERELEASE_SKIP.split()
+
+        skip_patterns = itertools.chain.from_iterable(
+            bracex.expand(p) for p in unexpanded_skip_patterns
+        )
 
         build: bool = any(fnmatch.fnmatch(build_id, pat) for pat in build_patterns)
         skip: bool = any(fnmatch.fnmatch(build_id, pat) for pat in skip_patterns)
         return build and not skip
 
     def __repr__(self) -> str:
-        if not self.skip_patterns:
-            return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r})'
-        else:
-            return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})'
+        result = f'{self.__class__.__name__}(build_config={" ".join(self.build_patterns)!r}'
+
+        if self.skip_patterns:
+            result += f', skip_config={" ".join(self.skip_patterns)!r}'
+        if self.prerelease_pythons:
+            result += ", prerelease_pythons=True"
+
+        result += ")"
+
+        return result
 
 
 class BuildSelector(IdentifierSelector):

docs/options.md

@@ -299,6 +299,32 @@ the package is compatible with all versions of Python that it can build.
 CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6"
 ```
 
+###  `CIBW_PRERELEASE_PYTHONS` {: #prerelease-pythons}
+> Enable building with pre-release versions of Python
+
+During the beta period, when new versions of Python are being tested,
+cibuildwheel will often gain early support for beta releases. If you would
+like to test wheel building with these versions, you can enable this flag.
+
+!!! caution
+    This option is provided for testing purposes only. It is not
+    recommended to distribute wheels built when `CIBW_PRERELEASE_PYTHONS` is
+    set, such as uploading to PyPI.  Please _do not_ upload these wheels to
+    PyPI, as they are not guaranteed to work with the final Python release.
+    Once Python is ABI stable and enters the release candidate phase, that
+    version of Python will become available without this flag.
+
+Default: Off (0) if Python is available in beta phase. No effect otherwise.
+
+This option can also be set using the [command-line option](#command-line) `--prerelease-pythons`.
+
+#### Examples
+
+```yaml
+# Include latest Python beta
+CIBW_PRERELEASE_PYTHONS: True
+```
+
 ## Build customization
 
 ### `CIBW_ENVIRONMENT` {: #environment}
@@ -715,6 +741,7 @@ CIBW_BUILD_VERBOSITY: 1
 usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}]
                     [--archs ARCHS] [--output-dir OUTPUT_DIR]
                     [--print-build-identifiers] [--allow-empty]
+                    [--prerelease-pythons]
                     [package_dir]
 
 Build wheels for all the platforms.
@@ -741,15 +768,16 @@ optional arguments:
                         natively supported on this machine. Set this option to
                         build an architecture via emulation, for example,
                         using binfmt_misc and QEMU. Default: auto. Choices:
-                        auto, native, all, x86_64, i686, aarch64, ppc64le,
-                        s390x, x86, AMD64
+                        auto, auto64, auto32, native, all, x86_64, i686,
+                        aarch64, ppc64le, s390x, universal2, arm64, x86, AMD64
   --output-dir OUTPUT_DIR
                         Destination folder for the wheels.
   --print-build-identifiers
                         Print the build identifiers matched by the current
                         invocation and exit.
   --allow-empty         Do not report an error code if the build does not
                         match any wheels.
+  --prerelease-pythons  Enable pre-release Python versions if available.
 ```
 
 <style>

test/test_0_basic.py

@@ -53,5 +53,7 @@ def test_build_identifiers(tmp_path):
     # check that the number of expected wheels matches the number of build
     # identifiers
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
-    build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    build_identifiers = utils.cibuildwheel_get_build_identifiers(
+        project_dir, prerelease_pythons=True
+    )
     assert len(expected_wheels) == len(build_identifiers)

test/test_before_test.py

@@ -49,11 +49,11 @@ def test(tmp_path):
             # checked in setup.py
             "CIBW_BEFORE_TEST": """python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
             "CIBW_BEFORE_TEST_WINDOWS": """python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""",
-            "CIBW_TEST_REQUIRES": "nose",
+            "CIBW_TEST_REQUIRES": "pytest",
             # the 'false ||' bit is to ensure this command runs in a shell on
             # mac/linux.
-            "CIBW_TEST_COMMAND": "false || nosetests {project}/test",
-            "CIBW_TEST_COMMAND_WINDOWS": "nosetests {project}/test",
+            "CIBW_TEST_COMMAND": "false || pytest {project}/test",
+            "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test",
         },
     )