import logging
import sys
import textwrap
from c_common.scriptutil import (
VERBOSITY,
add_verbosity_cli,
add_traceback_cli,
add_commands_cli,
add_kind_filtering_cli,
add_files_cli,
add_progress_cli,
process_args_by_key,
configure_logger,
get_prog,
)
from c_parser.info import KIND
import c_parser.__main__ as c_parser
import c_analyzer.__main__ as c_analyzer
import c_analyzer as _c_analyzer
from c_analyzer.info import UNKNOWN
from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT
logger = logging.getLogger(__name__)
CHECK_EXPLANATION = textwrap.dedent('''
-------------------------
Non-constant global variables are generally not supported
in the CPython repo. We use a tool to analyze the C code
and report if any unsupported globals are found. The tool
may be run manually with:
./python Tools/c-analyzer/check-c-globals.py --format summary [FILE]
Occasionally the tool is unable to parse updated code.
If this happens then add the file to the "EXCLUDED" list
in Tools/c-analyzer/cpython/_parser.py and create a new
issue for fixing the tool (and CC ericsnowcurrently
on the issue).
If the tool reports an unsupported global variable and
it is actually const (and thus supported) then first try
fixing the declaration appropriately in the code. If that
doesn't work then add the variable to the "should be const"
section of Tools/c-analyzer/cpython/ignored.tsv.
If the tool otherwise reports an unsupported global variable
then first try to make it non-global, possibly adding to
PyInterpreterState (for core code) or module state (for
extension modules). In an emergency, you can add the
variable to Tools/c-analyzer/cpython/globals-to-fix.tsv
to get CI passing, but doing so should be avoided. If
this course it taken, be sure to create an issue for
eliminating the global (and CC ericsnowcurrently).
''')
def _resolve_filenames(filenames):
if filenames:
resolved = (_files.resolve_filename(f) for f in filenames)
else:
resolved = _files.iter_filenames()
return resolved
#######################################
# the formats
def fmt_summary(analysis):
# XXX Support sorting and grouping.
supported = []
unsupported = []
for item in analysis:
if item.supported:
supported.append(item)
else:
unsupported.append(item)
total = 0
def section(name, groupitems):
nonlocal total
items, render = c_analyzer.build_section(name, groupitems,
relroot=REPO_ROOT)
yield from render()
total += len(items)
yield ''
yield '===================='
yield 'supported'
yield '===================='
yield from section('types', supported)
yield from section('variables', supported)
yield ''
yield '===================='
yield 'unsupported'
yield '===================='
yield from section('types', unsupported)
yield from section('variables', unsupported)
yield ''
yield f'grand total: {total}'
#######################################
# the checks
CHECKS = dict(c_analyzer.CHECKS, **{
'globals': _analyzer.check_globals,
})
#######################################
# the commands
FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')
def _cli_parse(parser):
process_output = c_parser.add_output_cli(parser)
process_kind = add_kind_filtering_cli(parser)
process_preprocessor = c_parser.add_preprocessor_cli(
parser,
get_preprocessor=_parser.get_preprocessor,
)
process_files = add_files_cli(parser, **FILES_KWARGS)
return [
process_output,
process_kind,
process_preprocessor,
process_files,
]
def cmd_parse(filenames=None, **kwargs):
filenames = _resolve_filenames(filenames)
if 'get_file_preprocessor' not in kwargs:
kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
c_parser.cmd_parse(
filenames,
relroot=REPO_ROOT,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
def _cli_check(parser, **kwargs):
return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)
def cmd_check(filenames=None, **kwargs):
filenames = _resolve_filenames(filenames)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
try:
c_analyzer.cmd_check(
filenames,
relroot=REPO_ROOT,
_analyze=_analyzer.analyze,
_CHECKS=CHECKS,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
except SystemExit as exc:
num_failed = exc.args[0] if getattr(exc, 'args', None) else None
if isinstance(num_failed, int):
if num_failed > 0:
sys.stderr.flush()
print(CHECK_EXPLANATION, flush=True)
raise # re-raise
except Exception:
sys.stderr.flush()
print(CHECK_EXPLANATION, flush=True)
raise # re-raise
def cmd_analyze(filenames=None, **kwargs):
formats = dict(c_analyzer.FORMATS)
formats['summary'] = fmt_summary
filenames = _resolve_filenames(filenames)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
c_analyzer.cmd_analyze(
filenames,
relroot=REPO_ROOT,
_analyze=_analyzer.analyze,
formats=formats,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
def _cli_data(parser):
filenames = False
known = True
return c_analyzer._cli_data(parser, filenames, known)
def cmd_data(datacmd, **kwargs):
formats = dict(c_analyzer.FORMATS)
formats['summary'] = fmt_summary
filenames = (file
for file in _resolve_filenames(None)
if file not in _parser.EXCLUDED)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
if datacmd == 'show':
types = _analyzer.read_known()
results = []
for decl, info in types.items():
if info is UNKNOWN:
if decl.kind in (KIND.STRUCT, KIND.UNION):
extra = {'unsupported': ['type unknown'] * len(decl.members)}
else:
extra = {'unsupported': ['type unknown']}
info = (info, extra)
results.append((decl, info))
if decl.shortkey == 'struct _object':
tempinfo = info
known = _analyzer.Analysis.from_results(results)
analyze = None
elif datacmd == 'dump':
known = _analyzer.KNOWN_FILE
def analyze(files, **kwargs):
decls = []
for decl in _analyzer.iter_decls(files, **kwargs):
if not KIND.is_type_decl(decl.kind):
continue
if not decl.filename.endswith('.h'):
if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
continue
decls.append(decl)
results = _c_analyzer.analyze_decls(
decls,
known={},
analyze_resolved=_analyzer.analyze_resolved,
)
return _analyzer.Analysis.from_results(results)
else: # check
known = _analyzer.read_known()
def analyze(files, **kwargs):
return _analyzer.iter_decls(files, **kwargs)
extracolumns = None
c_analyzer.cmd_data(
datacmd,
filenames,
known,
_analyze=analyze,
formats=formats,
extracolumns=extracolumns,
relroot=REPO_ROOT,
**kwargs
)
def _cli_capi(parser):
parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
parser.add_argument(f'--public', dest='levels',
action='append_const', const='public')
parser.add_argument(f'--no-public', dest='levels',
action='append_const', const='no-public')
for level in _capi.LEVELS:
parser.add_argument(f'--{level}', dest='levels',
action='append_const', const=level)
def process_levels(args, *, argv=None):
levels = []
for raw in args.levels or ():
for level in raw.replace(',', ' ').strip().split():
if level == 'public':
levels.append('stable')
levels.append('cpython')
elif level == 'no-public':
levels.append('private')
levels.append('internal')
elif level in _capi.LEVELS:
levels.append(level)
else:
parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
args.levels = set(levels)
parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
for kind in _capi.KINDS:
parser.add_argument(f'--{kind}', dest='kinds',
action='append_const', const=kind)
def process_kinds(args, *, argv=None):
kinds = []
for raw in args.kinds or ():
for kind in raw.replace(',', ' ').strip().split():
if kind in _capi.KINDS:
kinds.append(kind)
else:
parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
args.kinds = set(kinds)
parser.add_argument('--group-by', dest='groupby',
choices=['level', 'kind'])
parser.add_argument('--format', default='table')
parser.add_argument('--summary', dest='format',
action='store_const', const='summary')
def process_format(args, *, argv=None):
orig = args.format
args.format = _capi.resolve_format(args.format)
if isinstance(args.format, str):
if args.format not in _capi._FORMATS:
parser.error(f'unsupported format {orig!r}')
parser.add_argument('--show-empty', dest='showempty', action='store_true')
parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
parser.set_defaults(showempty=None)
# XXX Add --sort-by, --sort and --no-sort.
parser.add_argument('--ignore', dest='ignored', action='append')
def process_ignored(args, *, argv=None):
ignored = []
for raw in args.ignored or ():
ignored.extend(raw.replace(',', ' ').strip().split())
args.ignored = ignored or None
parser.add_argument('filenames', nargs='*', metavar='FILENAME')
process_progress = add_progress_cli(parser)
return [
process_levels,
process_kinds,
process_format,
process_ignored,
process_progress,
]
def cmd_capi(filenames=None, *,
levels=None,
kinds=None,
groupby='kind',
format='table',
showempty=None,
ignored=None,
track_progress=None,
verbosity=VERBOSITY,
**kwargs
):
render = _capi.get_renderer(format)
filenames = _files.iter_header_files(filenames, levels=levels)
#filenames = (file for file, _ in main_for_filenames(filenames))
if track_progress:
filenames = track_progress(filenames)
items = _capi.iter_capi(filenames)
if levels:
items = (item for item in items if item.level in levels)
if kinds:
items = (item for item in items if item.kind in kinds)
filter = _capi.resolve_filter(ignored)
if filter:
items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))
lines = render(
items,
groupby=groupby,
showempty=showempty,
verbose=verbosity > VERBOSITY,
)
print()
for line in lines:
print(line)
def _cli_builtin_types(parser):
parser.add_argument('--format', dest='fmt', default='table')
# parser.add_argument('--summary', dest='format',
# action='store_const', const='summary')
def process_format(args, *, argv=None):
orig = args.fmt
args.fmt = _builtin_types.resolve_format(args.fmt)
if isinstance(args.fmt, str):
if args.fmt not in _builtin_types._FORMATS:
parser.error(f'unsupported format {orig!r}')
parser.add_argument('--include-modules', dest='showmodules',
action='store_true')
def process_modules(args, *, argv=None):
pass
return [
process_format,
process_modules,
]
def cmd_builtin_types(fmt, *,
showmodules=False,
verbosity=VERBOSITY,
):
render = _builtin_types.get_renderer(fmt)
types = _builtin_types.iter_builtin_types()
match = _builtin_types.resolve_matcher(showmodules)
if match:
types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg)))
lines = render(
types,
# verbose=verbosity > VERBOSITY,
)
print()
for line in lines:
print(line)
# We do not define any other cmd_*() handlers here,
# favoring those defined elsewhere.
COMMANDS = {
'check': (
'analyze and fail if the CPython source code has any problems',
[_cli_check],
cmd_check,
),
'analyze': (
'report on the state of the CPython source code',
[(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
cmd_analyze,
),
'parse': (
'parse the CPython source files',
[_cli_parse],
cmd_parse,
),
'data': (
'check/manage local data (e.g. known types, ignored vars, caches)',
[_cli_data],
cmd_data,
),
'capi': (
'inspect the C-API',
[_cli_capi],
cmd_capi,
),
'builtin-types': (
'show the builtin types',
[_cli_builtin_types],
cmd_builtin_types,
),
}
#######################################
# the script
def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
import argparse
parser = argparse.ArgumentParser(
prog=prog or get_prog(),
)
# if subset == 'check' or subset == ['check']:
# if checks is not None:
# commands = dict(COMMANDS)
# commands['check'] = list(commands['check'])
# cli = commands['check'][1][0]
# commands['check'][1][0] = (lambda p: cli(p, checks=checks))
processors = add_commands_cli(
parser,
commands=COMMANDS,
commonspecs=[
add_verbosity_cli,
add_traceback_cli,
],
subset=subset,
)
args = parser.parse_args(argv)
ns = vars(args)
cmd = ns.pop('cmd')
verbosity, traceback_cm = process_args_by_key(
args,
argv,
processors[cmd],
['verbosity', 'traceback_cm'],
)
if cmd != 'parse':
# "verbosity" is sent to the commands, so we put it back.
args.verbosity = verbosity
return cmd, ns, verbosity, traceback_cm
def main(cmd, cmd_kwargs):
try:
run_cmd = COMMANDS[cmd][-1]
except KeyError:
raise ValueError(f'unsupported cmd {cmd!r}')
run_cmd(**cmd_kwargs)
if __name__ == '__main__':
cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
configure_logger(verbosity)
with traceback_cm:
main(cmd, cmd_kwargs)
Codebase Browser
cpython