import os.path
from c_common.clsutil import classonly
from c_parser.info import (
KIND,
Declaration,
TypeDeclaration,
Member,
FIXED_TYPE,
)
from c_parser.match import (
is_pots,
is_funcptr,
)
from c_analyzer.match import (
is_system_type,
is_process_global,
is_fixed_type,
is_immutable,
)
import c_analyzer as _c_analyzer
import c_analyzer.info as _info
import c_analyzer.datafiles as _datafiles
from . import _parser, REPO_ROOT
_DATA_DIR = os.path.dirname(__file__)
KNOWN_FILE = os.path.join(_DATA_DIR, 'known.tsv')
IGNORED_FILE = os.path.join(_DATA_DIR, 'ignored.tsv')
NEED_FIX_FILE = os.path.join(_DATA_DIR, 'globals-to-fix.tsv')
KNOWN_IN_DOT_C = {
'struct _odictobject': False,
'PyTupleObject': False,
'struct _typeobject': False,
'struct _arena': True, # ???
'struct _frame': False,
'struct _ts': True, # ???
'struct PyCodeObject': False,
'struct _is': True, # ???
'PyWideStringList': True, # ???
# recursive
'struct _dictkeysobject': False,
}
# These are loaded from the respective .tsv files upon first use.
_KNOWN = {
# {(file, ID) | ID => info | bool}
#'PyWideStringList': True,
}
#_KNOWN = {(Struct(None, typeid.partition(' ')[-1], None)
# if typeid.startswith('struct ')
# else TypeDef(None, typeid, None)
# ): ([], {'unsupported': None if supported else True})
# for typeid, supported in _KNOWN_IN_DOT_C.items()}
_IGNORED = {
# {ID => reason}
}
# XXX We should be handling these through known.tsv.
_OTHER_SUPPORTED_TYPES = {
# Holds tuple of strings, which we statically initialize:
'_PyArg_Parser',
# Uses of these should be const, but we don't worry about it.
'PyModuleDef',
'PyModuleDef_Slot[]',
'PyType_Spec',
'PyType_Slot[]',
'PyMethodDef',
'PyMethodDef[]',
'PyMemberDef[]',
'PyGetSetDef[]',
'PyNumberMethods',
'PySequenceMethods',
'PyMappingMethods',
'PyAsyncMethods',
'PyBufferProcs',
'PyStructSequence_Field[]',
'PyStructSequence_Desc',
}
# XXX We should normalize all cases to a single name,
# e.g. "kwlist" (currently the most common).
_KWLIST_VARIANTS = [
('*', 'kwlist'),
('*', 'keywords'),
('*', 'kwargs'),
('Modules/_csv.c', 'dialect_kws'),
('Modules/_datetimemodule.c', 'date_kws'),
('Modules/_datetimemodule.c', 'datetime_kws'),
('Modules/_datetimemodule.c', 'time_kws'),
('Modules/_datetimemodule.c', 'timezone_kws'),
('Modules/_lzmamodule.c', 'optnames'),
('Modules/_lzmamodule.c', 'arg_names'),
('Modules/cjkcodecs/multibytecodec.c', 'incnewkwarglist'),
('Modules/cjkcodecs/multibytecodec.c', 'streamkwarglist'),
('Modules/socketmodule.c', 'kwnames'),
]
KINDS = frozenset((*KIND.TYPES, KIND.VARIABLE))
def read_known():
if not _KNOWN:
# Cache a copy the first time.
extracols = None # XXX
#extracols = ['unsupported']
known = _datafiles.read_known(KNOWN_FILE, extracols, REPO_ROOT)
# For now we ignore known.values() (i.e. "extra").
types, _ = _datafiles.analyze_known(
known,
analyze_resolved=analyze_resolved,
)
_KNOWN.update(types)
return _KNOWN.copy()
def write_known():
raise NotImplementedError
datafiles.write_known(decls, IGNORED_FILE, ['unsupported'], relroot=REPO_ROOT)
def read_ignored():
if not _IGNORED:
_IGNORED.update(_datafiles.read_ignored(IGNORED_FILE, relroot=REPO_ROOT))
_IGNORED.update(_datafiles.read_ignored(NEED_FIX_FILE, relroot=REPO_ROOT))
return dict(_IGNORED)
def write_ignored():
raise NotImplementedError
_datafiles.write_ignored(variables, IGNORED_FILE, relroot=REPO_ROOT)
def analyze(filenames, *,
skip_objects=False,
**kwargs
):
if skip_objects:
# XXX Set up a filter.
raise NotImplementedError
known = read_known()
decls = iter_decls(filenames)
results = _c_analyzer.analyze_decls(
decls,
known,
analyze_resolved=analyze_resolved,
)
analysis = Analysis.from_results(results)
return analysis
def iter_decls(filenames, **kwargs):
decls = _c_analyzer.iter_decls(
filenames,
# We ignore functions (and statements).
kinds=KINDS,
parse_files=_parser.parse_files,
**kwargs
)
for decl in decls:
if not decl.data:
# Ignore forward declarations.
continue
yield decl
def analyze_resolved(resolved, decl, types, knowntypes, extra=None):
if decl.kind not in KINDS:
# Skip it!
return None
typedeps = resolved
if typedeps is _info.UNKNOWN:
if decl.kind in (KIND.STRUCT, KIND.UNION):
typedeps = [typedeps] * len(decl.members)
else:
typedeps = [typedeps]
#assert isinstance(typedeps, (list, TypeDeclaration)), typedeps
if extra is None:
extra = {}
elif 'unsupported' in extra:
raise NotImplementedError((decl, extra))
unsupported = _check_unsupported(decl, typedeps, types, knowntypes)
extra['unsupported'] = unsupported
return typedeps, extra
def _check_unsupported(decl, typedeps, types, knowntypes):
if typedeps is None:
raise NotImplementedError(decl)
if decl.kind in (KIND.STRUCT, KIND.UNION):
return _check_members(decl, typedeps, types, knowntypes)
elif decl.kind is KIND.ENUM:
if typedeps:
raise NotImplementedError((decl, typedeps))
return None
else:
return _check_typedep(decl, typedeps, types, knowntypes)
def _check_members(decl, typedeps, types, knowntypes):
if isinstance(typedeps, TypeDeclaration):
raise NotImplementedError((decl, typedeps))
#members = decl.members or () # A forward decl has no members.
members = decl.members
if not members:
# A forward decl has no members, but that shouldn't surface here..
raise NotImplementedError(decl)
if len(members) != len(typedeps):
raise NotImplementedError((decl, typedeps))
unsupported = []
for member, typedecl in zip(members, typedeps):
checked = _check_typedep(member, typedecl, types, knowntypes)
unsupported.append(checked)
if any(None if v is FIXED_TYPE else v for v in unsupported):
return unsupported
elif FIXED_TYPE in unsupported:
return FIXED_TYPE
else:
return None
def _check_typedep(decl, typedecl, types, knowntypes):
if not isinstance(typedecl, TypeDeclaration):
if hasattr(type(typedecl), '__len__'):
if len(typedecl) == 1:
typedecl, = typedecl
if typedecl is None:
# XXX Fail?
return 'typespec (missing)'
elif typedecl is _info.UNKNOWN:
if _has_other_supported_type(decl):
return None
# XXX Is this right?
return 'typespec (unknown)'
elif not isinstance(typedecl, TypeDeclaration):
raise NotImplementedError((decl, typedecl))
if isinstance(decl, Member):
return _check_vartype(decl, typedecl, types, knowntypes)
elif not isinstance(decl, Declaration):
raise NotImplementedError(decl)
elif decl.kind is KIND.TYPEDEF:
return _check_vartype(decl, typedecl, types, knowntypes)
elif decl.kind is KIND.VARIABLE:
if not is_process_global(decl):
return None
if _is_kwlist(decl):
return None
if _has_other_supported_type(decl):
return None
checked = _check_vartype(decl, typedecl, types, knowntypes)
return 'mutable' if checked is FIXED_TYPE else checked
else:
raise NotImplementedError(decl)
def _is_kwlist(decl):
# keywords for PyArg_ParseTupleAndKeywords()
# "static char *name[]" -> "static const char * const name[]"
# XXX These should be made const.
for relpath, name in _KWLIST_VARIANTS:
if decl.name == name:
if relpath == '*':
break
assert os.path.isabs(decl.file.filename)
relpath = os.path.normpath(relpath)
if decl.file.filename.endswith(os.path.sep + relpath):
break
else:
return False
vartype = ''.join(str(decl.vartype).split())
return vartype == 'char*[]'
def _is_local_static_mutex(decl):
if not hasattr(decl, "vartype"):
return False
if not hasattr(decl, "parent") or decl.parent is None:
# We only want to allow local variables
return False
vartype = decl.vartype
return (vartype.typespec == 'PyMutex') and (decl.storage == 'static')
def _has_other_supported_type(decl):
if hasattr(decl, 'file') and decl.file.filename.endswith('.c.h'):
assert 'clinic' in decl.file.filename, (decl,)
if decl.name == '_kwtuple':
return True
if _is_local_static_mutex(decl):
# GH-127081: Local static mutexes are used to
# wrap libc functions that aren't thread safe
return True
vartype = str(decl.vartype).split()
if vartype[0] == 'struct':
vartype = vartype[1:]
vartype = ''.join(vartype)
return vartype in _OTHER_SUPPORTED_TYPES
def _check_vartype(decl, typedecl, types, knowntypes):
"""Return failure reason."""
checked = _check_typespec(decl, typedecl, types, knowntypes)
if checked:
return checked
if is_immutable(decl.vartype):
return None
if is_fixed_type(decl.vartype):
return FIXED_TYPE
return 'mutable'
def _check_typespec(decl, typedecl, types, knowntypes):
typespec = decl.vartype.typespec
if typedecl is not None:
found = types.get(typedecl)
if found is None:
found = knowntypes.get(typedecl)
if found is not None:
_, extra = found
if extra is None:
# XXX Under what circumstances does this happen?
extra = {}
unsupported = extra.get('unsupported')
if unsupported is FIXED_TYPE:
unsupported = None
return 'typespec' if unsupported else None
# Fall back to default known types.
if is_pots(typespec):
return None
elif is_system_type(typespec):
return None
elif is_funcptr(decl.vartype):
return None
return 'typespec'
class Analyzed(_info.Analyzed):
@classonly
def is_target(cls, raw):
if not super().is_target(raw):
return False
if raw.kind not in KINDS:
return False
return True
#@classonly
#def _parse_raw_result(cls, result, extra):
# typedecl, extra = super()._parse_raw_result(result, extra)
# if typedecl is None:
# return None, extra
# raise NotImplementedError
def __init__(self, item, typedecl=None, *, unsupported=None, **extra):
if 'unsupported' in extra:
raise NotImplementedError((item, typedecl, unsupported, extra))
if not unsupported:
unsupported = None
elif isinstance(unsupported, (str, TypeDeclaration)):
unsupported = (unsupported,)
elif unsupported is not FIXED_TYPE:
unsupported = tuple(unsupported)
self.unsupported = unsupported
extra['unsupported'] = self.unsupported # ...for __repr__(), etc.
if self.unsupported is None:
#self.supported = None
self.supported = True
elif self.unsupported is FIXED_TYPE:
if item.kind is KIND.VARIABLE:
raise NotImplementedError(item, typedecl, unsupported)
self.supported = True
else:
self.supported = not self.unsupported
super().__init__(item, typedecl, **extra)
def render(self, fmt='line', *, itemonly=False):
if fmt == 'raw':
yield repr(self)
return
rendered = super().render(fmt, itemonly=itemonly)
# XXX ???
#if itemonly:
# yield from rendered
supported = self.supported
if fmt in ('line', 'brief'):
rendered, = rendered
parts = [
'+' if supported else '-' if supported is False else '',
rendered,
]
yield '\t'.join(parts)
elif fmt == 'summary':
raise NotImplementedError(fmt)
elif fmt == 'full':
yield from rendered
if supported:
yield f'\tsupported:\t{supported}'
else:
raise NotImplementedError(fmt)
class Analysis(_info.Analysis):
_item_class = Analyzed
@classonly
def build_item(cls, info, result=None):
if not isinstance(info, Declaration) or info.kind not in KINDS:
raise NotImplementedError((info, result))
return super().build_item(info, result)
def check_globals(analysis):
# yield (data, failure)
ignored = read_ignored()
for item in analysis:
if item.kind != KIND.VARIABLE:
continue
if item.supported:
continue
if item.id in ignored:
continue
reason = item.unsupported
if not reason:
reason = '???'
elif not isinstance(reason, str):
if len(reason) == 1:
reason, = reason
reason = f'({reason})'
yield item, f'not supported {reason:20}\t{item.storage or ""} {item.vartype}'
Codebase Browser
cpython