"""Check extension modules
The script checks shared and built-in extension modules. It verifies that the
modules have been built and that they can be imported successfully. Missing
modules and failed imports are reported to the user. Shared extension
files are renamed on failed import.
Module information is parsed from several sources:
- core modules hard-coded in Modules/config.c.in
- Windows-specific modules that are hard-coded in PC/config.c
- MODULE_{name}_STATE entries in Makefile (provided through sysconfig)
- Various makesetup files:
- $(srcdir)/Modules/Setup
- Modules/Setup.[local|bootstrap|stdlib] files, which are generated
from $(srcdir)/Modules/Setup.*.in files
See --help for more information
"""
import argparse
import collections
import enum
import logging
import os
import pathlib
import re
import sys
import sysconfig
import warnings
import _imp
from importlib._bootstrap import _load as bootstrap_load
from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec
from importlib.util import spec_from_file_location, spec_from_loader
from typing import Iterable
SRC_DIR = pathlib.Path(__file__).parent.parent.parent
# core modules, hard-coded in Modules/config.h.in
CORE_MODULES = {
"_ast",
"_imp",
"_string",
"_tokenize",
"_warnings",
"builtins",
"gc",
"marshal",
"sys",
}
# Windows-only modules
WINDOWS_MODULES = {
"_overlapped",
"_testconsole",
"_winapi",
"_wmi",
"msvcrt",
"nt",
"winreg",
"winsound",
}
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser(
prog="check_extension_modules",
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--verbose",
action="store_true",
help="Verbose, report builtin, shared, and unavailable modules",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable debug logging",
)
parser.add_argument(
"--strict",
action=argparse.BooleanOptionalAction,
help=(
"Strict check, fail when a module is missing or fails to import"
"(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)"
),
default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")),
)
parser.add_argument(
"--cross-compiling",
action=argparse.BooleanOptionalAction,
help=(
"Use cross-compiling checks "
"(default: no, unless env var _PYTHON_HOST_PLATFORM is set)."
),
default="_PYTHON_HOST_PLATFORM" in os.environ,
)
parser.add_argument(
"--list-module-names",
action="store_true",
help="Print a list of module names to stdout and exit",
)
class ModuleState(enum.Enum):
# Makefile state "yes"
BUILTIN = "builtin"
SHARED = "shared"
DISABLED = "disabled"
MISSING = "missing"
NA = "n/a"
# disabled by Setup / makesetup rule
DISABLED_SETUP = "disabled_setup"
def __bool__(self):
return self.value in {"builtin", "shared"}
ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
class ModuleChecker:
pybuilddir_txt = "pybuilddir.txt"
setup_files = (
# see end of configure.ac
"Modules/Setup.local",
"Modules/Setup.stdlib",
"Modules/Setup.bootstrap",
SRC_DIR / "Modules/Setup",
)
def __init__(self, cross_compiling: bool = False, strict: bool = False):
self.cross_compiling = cross_compiling
self.strict_extensions_build = strict
self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
self.platform = sysconfig.get_platform()
self.builddir = self.get_builddir()
self.modules = self.get_modules()
self.builtin_ok = []
self.shared_ok = []
self.failed_on_import = []
self.missing = []
self.disabled_configure = []
self.disabled_setup = []
self.notavailable = []
def check(self):
if not hasattr(_imp, 'create_dynamic'):
logger.warning(
('Dynamic extensions not supported '
'(HAVE_DYNAMIC_LOADING not defined)'),
)
for modinfo in self.modules:
logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo))
if modinfo.state == ModuleState.DISABLED:
self.disabled_configure.append(modinfo)
elif modinfo.state == ModuleState.DISABLED_SETUP:
self.disabled_setup.append(modinfo)
elif modinfo.state == ModuleState.MISSING:
self.missing.append(modinfo)
elif modinfo.state == ModuleState.NA:
self.notavailable.append(modinfo)
else:
try:
if self.cross_compiling:
self.check_module_cross(modinfo)
else:
self.check_module_import(modinfo)
except (ImportError, FileNotFoundError):
self.rename_module(modinfo)
self.failed_on_import.append(modinfo)
else:
if modinfo.state == ModuleState.BUILTIN:
self.builtin_ok.append(modinfo)
else:
assert modinfo.state == ModuleState.SHARED
self.shared_ok.append(modinfo)
def summary(self, *, verbose: bool = False):
longest = max([len(e.name) for e in self.modules], default=0)
def print_three_column(modinfos: list[ModuleInfo]):
names = [modinfo.name for modinfo in modinfos]
names.sort(key=str.lower)
# guarantee zip() doesn't drop anything
while len(names) % 3:
names.append("")
for l, m, r in zip(names[::3], names[1::3], names[2::3]):
print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r))
if verbose and self.builtin_ok:
print("The following *built-in* modules have been successfully built:")
print_three_column(self.builtin_ok)
print()
if verbose and self.shared_ok:
print("The following *shared* modules have been successfully built:")
print_three_column(self.shared_ok)
print()
if self.disabled_configure:
print("The following modules are *disabled* in configure script:")
print_three_column(self.disabled_configure)
print()
if self.disabled_setup:
print("The following modules are *disabled* in Modules/Setup files:")
print_three_column(self.disabled_setup)
print()
if verbose and self.notavailable:
print(
f"The following modules are not available on platform '{self.platform}':"
)
print_three_column(self.notavailable)
print()
if self.missing:
print("The necessary bits to build these optional modules were not found:")
print_three_column(self.missing)
print("To find the necessary bits, look in configure.ac and config.log.")
print()
if self.failed_on_import:
print(
"Following modules built successfully "
"but were removed because they could not be imported:"
)
print_three_column(self.failed_on_import)
print()
if any(
modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import
):
print("Could not build the ssl module!")
print("Python requires a OpenSSL 1.1.1 or newer")
if sysconfig.get_config_var("OPENSSL_LDFLAGS"):
print("Custom linker flags may require --with-openssl-rpath=auto")
print()
disabled = len(self.disabled_configure) + len(self.disabled_setup)
print(
f"Checked {len(self.modules)} modules ("
f"{len(self.builtin_ok)} built-in, "
f"{len(self.shared_ok)} shared, "
f"{len(self.notavailable)} n/a on {self.platform}, "
f"{disabled} disabled, "
f"{len(self.missing)} missing, "
f"{len(self.failed_on_import)} failed on import)"
)
def check_strict_build(self):
"""Fail if modules are missing and it's a strict build"""
if self.strict_extensions_build and (self.failed_on_import or self.missing):
raise RuntimeError("Failed to build some stdlib modules")
def list_module_names(self, *, all: bool = False) -> set:
names = {modinfo.name for modinfo in self.modules}
if all:
names.update(WINDOWS_MODULES)
return names
def get_builddir(self) -> pathlib.Path:
try:
with open(self.pybuilddir_txt, encoding="utf-8") as f:
builddir = f.read()
except FileNotFoundError:
logger.error("%s must be run from the top build directory", __file__)
raise
builddir = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir)
return builddir
def get_modules(self) -> list[ModuleInfo]:
"""Get module info from sysconfig and Modules/Setup* files"""
seen = set()
modules = []
# parsing order is important, first entry wins
for modinfo in self.get_core_modules():
modules.append(modinfo)
seen.add(modinfo.name)
for setup_file in self.setup_files:
for modinfo in self.parse_setup_file(setup_file):
if modinfo.name not in seen:
modules.append(modinfo)
seen.add(modinfo.name)
for modinfo in self.get_sysconfig_modules():
if modinfo.name not in seen:
modules.append(modinfo)
seen.add(modinfo.name)
logger.debug("Found %i modules in total", len(modules))
modules.sort()
return modules
def get_core_modules(self) -> Iterable[ModuleInfo]:
"""Get hard-coded core modules"""
for name in CORE_MODULES:
modinfo = ModuleInfo(name, ModuleState.BUILTIN)
logger.debug("Found core module %s", modinfo)
yield modinfo
def get_sysconfig_modules(self) -> Iterable[ModuleInfo]:
"""Get modules defined in Makefile through sysconfig
MODBUILT_NAMES: modules in *static* block
MODSHARED_NAMES: modules in *shared* block
MODDISABLED_NAMES: modules in *disabled* block
"""
moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split())
if self.cross_compiling:
modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split())
else:
modbuiltin = set(sys.builtin_module_names)
for key, value in sysconfig.get_config_vars().items():
if not key.startswith("MODULE_") or not key.endswith("_STATE"):
continue
if value not in {"yes", "disabled", "missing", "n/a"}:
raise ValueError(f"Unsupported value '{value}' for {key}")
modname = key[7:-6].lower()
if modname in moddisabled:
# Setup "*disabled*" rule
state = ModuleState.DISABLED_SETUP
elif value in {"disabled", "missing", "n/a"}:
state = ModuleState(value)
elif modname in modbuiltin:
assert value == "yes"
state = ModuleState.BUILTIN
else:
assert value == "yes"
state = ModuleState.SHARED
modinfo = ModuleInfo(modname, state)
logger.debug("Found %s in Makefile", modinfo)
yield modinfo
def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
"""Parse a Modules/Setup file"""
assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo
# default to static module
state = ModuleState.BUILTIN
logger.debug("Parsing Setup file %s", setup_file)
with open(setup_file, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or assign_var.match(line):
continue
match line.split():
case ["*shared*"]:
state = ModuleState.SHARED
case ["*static*"]:
state = ModuleState.BUILTIN
case ["*disabled*"]:
state = ModuleState.DISABLED
case ["*noconfig*"]:
state = None
case [*items]:
if state == ModuleState.DISABLED:
# *disabled* can disable multiple modules per line
for item in items:
modinfo = ModuleInfo(item, state)
logger.debug("Found %s in %s", modinfo, setup_file)
yield modinfo
elif state in {ModuleState.SHARED, ModuleState.BUILTIN}:
# *shared* and *static*, first item is the name of the module.
modinfo = ModuleInfo(items[0], state)
logger.debug("Found %s in %s", modinfo, setup_file)
yield modinfo
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
else:
raise ValueError(modinfo)
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path:
"""Get shared library location in build directory"""
if modinfo.state == ModuleState.SHARED:
return self.builddir / f"{modinfo.name}{self.ext_suffix}"
else:
return None
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
"""Check that the module file is present and not empty"""
if spec.loader is BuiltinImporter:
return
try:
st = os.stat(spec.origin)
except FileNotFoundError:
logger.error("%s (%s) is missing", modinfo.name, spec.origin)
raise
if not st.st_size:
raise ImportError(f"{spec.origin} is an empty file")
def check_module_import(self, modinfo: ModuleInfo):
"""Attempt to import module and report errors"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
try:
with warnings.catch_warnings():
# ignore deprecation warning from deprecated modules
warnings.simplefilter("ignore", DeprecationWarning)
bootstrap_load(spec)
except ImportError as e:
logger.error("%s failed to import: %s", modinfo.name, e)
raise
except Exception as e:
if not hasattr(_imp, 'create_dynamic'):
logger.warning("Dynamic extension '%s' ignored", modinfo.name)
return
logger.exception("Importing extension '%s' failed!", modinfo.name)
raise
def check_module_cross(self, modinfo: ModuleInfo):
"""Sanity check for cross compiling"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
def rename_module(self, modinfo: ModuleInfo) -> None:
"""Rename module file"""
if modinfo.state == ModuleState.BUILTIN:
logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name)
return
failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
builddir_path = self.get_location(modinfo)
if builddir_path.is_symlink():
symlink = builddir_path
module_path = builddir_path.resolve().relative_to(os.getcwd())
failed_path = module_path.parent / failed_name
else:
symlink = None
module_path = builddir_path
failed_path = self.builddir / failed_name
# remove old failed file
failed_path.unlink(missing_ok=True)
# remove symlink
if symlink is not None:
symlink.unlink(missing_ok=True)
# rename shared extension file
try:
module_path.rename(failed_path)
except FileNotFoundError:
logger.debug("Shared extension file '%s' does not exist.", module_path)
else:
logger.debug("Rename '%s' -> '%s'", module_path, failed_path)
def main():
args = parser.parse_args()
if args.debug:
args.verbose = True
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format="[%(levelname)s] %(message)s",
)
checker = ModuleChecker(
cross_compiling=args.cross_compiling,
strict=args.strict,
)
if args.list_module_names:
names = checker.list_module_names(all=True)
for name in sorted(names):
print(name)
else:
checker.check()
checker.summary(verbose=args.verbose)
try:
checker.check_strict_build()
except RuntimeError as e:
parser.exit(1, f"\nError: {e}\n")
if __name__ == "__main__":
main()
Codebase Browser
cpython