#!/usr/bin/env python3
# A tool to parse ASTMatchers.h and update the documentation in
# ../LibASTMatchersReference.html automatically. Run from the
# directory in which this file is located to update the docs.
import collections
import re
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
CLASS_INDEX_PAGE_URL = "https://clang.llvm.org/doxygen/classes.html"
try:
CLASS_INDEX_PAGE = urlopen(CLASS_INDEX_PAGE_URL).read().decode("utf-8")
except Exception as e:
CLASS_INDEX_PAGE = None
print("Unable to get %s: %s" % (CLASS_INDEX_PAGE_URL, e))
MATCHERS_FILE = "../../include/clang/ASTMatchers/ASTMatchers.h"
# Each matcher is documented in one row of the form:
# result | name | argA
# The subsequent row contains the documentation and is hidden by default,
# becoming visible via javascript when the user clicks the matcher name.
TD_TEMPLATE = """
<tr><td>%(result)s</td><td class="name" onclick="toggle('%(id)s')"><a name="%(id)sAnchor">%(name)s</a></td><td>%(args)s</td></tr>
<tr><td colspan="4" class="doc" id="%(id)s"><pre>%(comment)s</pre></td></tr>
"""
# We categorize the matchers into these three categories in the reference:
node_matchers = {}
narrowing_matchers = {}
traversal_matchers = {}
# We output multiple rows per matcher if the matcher can be used on multiple
# node types. Thus, we need a new id per row to control the documentation
# pop-up. ids[name] keeps track of those ids.
ids = collections.defaultdict(int)
# Cache for doxygen urls we have already verified.
doxygen_probes = {}
def esc(text):
"""Escape any html in the given text."""
text = re.sub(r"&", "&", text)
text = re.sub(r"<", "<", text)
text = re.sub(r">", ">", text)
def link_if_exists(m):
"""Wrap a likely AST node name in a link to its clang docs.
We want to do this only if the page exists, in which case it will be
referenced from the class index page.
"""
name = m.group(1)
url = "https://clang.llvm.org/doxygen/classclang_1_1%s.html" % name
if url not in doxygen_probes:
search_str = 'href="classclang_1_1%s.html"' % name
if CLASS_INDEX_PAGE is not None:
doxygen_probes[url] = search_str in CLASS_INDEX_PAGE
else:
doxygen_probes[url] = True
if not doxygen_probes[url]:
print("Did not find %s in class index page" % name)
if doxygen_probes[url]:
return r'Matcher<<a href="%s">%s</a>>' % (url, name)
else:
return m.group(0)
text = re.sub(r"Matcher<([^\*&]+)>", link_if_exists, text)
return text
def extract_result_types(comment):
"""Extracts a list of result types from the given comment.
We allow annotations in the comment of the matcher to specify what
nodes a matcher can match on. Those comments have the form:
Usable as: Any Matcher | (Matcher<T1>[, Matcher<t2>[, ...]])
Returns ['*'] in case of 'Any Matcher', or ['T1', 'T2', ...].
Returns the empty list if no 'Usable as' specification could be
parsed.
"""
result_types = []
m = re.search(r"Usable as: Any Matcher[\s\n]*$", comment, re.S)
if m:
return ["*"]
while True:
m = re.match(r"^(.*)Matcher<([^>]+)>\s*,?[\s\n]*$", comment, re.S)
if not m:
if re.search(r"Usable as:\s*$", comment):
return result_types
else:
return None
result_types += [m.group(2)]
comment = m.group(1)
def strip_doxygen(comment):
"""Returns the given comment without \-escaped words."""
# If there is only a doxygen keyword in the line, delete the whole line.
comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)
# If there is a doxygen \see command, change the \see prefix into "See also:".
# FIXME: it would be better to turn this into a link to the target instead.
comment = re.sub(r"\\see", r"See also:", comment)
# Delete the doxygen command and the following whitespace.
comment = re.sub(r"\\[^\s]+\s+", r"", comment)
return comment
def unify_arguments(args):
"""Gets rid of anything the user doesn't care about in the argument list."""
args = re.sub(r"clang::ast_matchers::internal::", r"", args)
args = re.sub(r"ast_matchers::internal::", r"", args)
args = re.sub(r"internal::", r"", args)
args = re.sub(r"extern const\s+(.*)&", r"\1 ", args)
args = re.sub(r"&", r" ", args)
args = re.sub(r"(^|\s)M\d?(\s)", r"\1Matcher<*>\2", args)
args = re.sub(r"BindableMatcher", r"Matcher", args)
args = re.sub(r"const Matcher", r"Matcher", args)
return args
def unify_type(result_type):
"""Gets rid of anything the user doesn't care about in the type name."""
result_type = re.sub(
r"^internal::(Bindable)?Matcher<([a-zA-Z_][a-zA-Z0-9_]*)>$", r"\2", result_type
)
return result_type
def add_matcher(result_type, name, args, comment, is_dyncast=False):
"""Adds a matcher to one of our categories."""
if name == "id":
# FIXME: Figure out whether we want to support the 'id' matcher.
return
matcher_id = "%s%d" % (name, ids[name])
ids[name] += 1
args = unify_arguments(args)
result_type = unify_type(result_type)
docs_result_type = esc("Matcher<%s>" % result_type)
if name == "mapAnyOf":
args = "nodeMatcherFunction..."
docs_result_type = "<em>unspecified</em>"
matcher_html = TD_TEMPLATE % {
"result": docs_result_type,
"name": name,
"args": esc(args),
"comment": esc(strip_doxygen(comment)),
"id": matcher_id,
}
if is_dyncast:
dict = node_matchers
lookup = result_type + name
# Use a heuristic to figure out whether a matcher is a narrowing or
# traversal matcher. By default, matchers that take other matchers as
# arguments (and are not node matchers) do traversal. We specifically
# exclude known narrowing matchers that also take other matchers as
# arguments.
elif "Matcher<" not in args or name in [
"allOf",
"anyOf",
"anything",
"unless",
"mapAnyOf",
]:
dict = narrowing_matchers
lookup = result_type + name + esc(args)
else:
dict = traversal_matchers
lookup = result_type + name + esc(args)
if dict.get(lookup) is None or len(dict.get(lookup)) < len(matcher_html):
dict[lookup] = matcher_html
def act_on_decl(declaration, comment, allowed_types):
"""Parse the matcher out of the given declaration and comment.
If 'allowed_types' is set, it contains a list of node types the matcher
can match on, as extracted from the static type asserts in the matcher
definition.
"""
if declaration.strip():
if re.match(r"^\s?(#|namespace|using|template <typename NodeType> using|})", declaration):
return
# Node matchers are defined by writing:
# VariadicDynCastAllOfMatcher<ResultType, ArgumentType> name;
m = re.match(
r""".*Variadic(?:DynCast)?AllOfMatcher\s*<
\s*([^\s,]+)\s*(?:,
\s*([^\s>]+)\s*)?>
\s*([^\s;]+)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
result, inner, name = m.groups()
if not inner:
inner = result
add_matcher(
result, name, "Matcher<%s>..." % inner, comment, is_dyncast=True
)
return
# Special case of type matchers:
# AstTypeMatcher<ArgumentType> name
m = re.match(
r""".*AstTypeMatcher\s*<
\s*([^\s>]+)\s*>
\s*([^\s;]+)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
inner, name = m.groups()
add_matcher(
"Type", name, "Matcher<%s>..." % inner, comment, is_dyncast=True
)
# FIXME: re-enable once we have implemented casting on the TypeLoc
# hierarchy.
# add_matcher('TypeLoc', '%sLoc' % name, 'Matcher<%sLoc>...' % inner,
# comment, is_dyncast=True)
return
# Parse the various matcher definition macros.
m = re.match(
""".*AST_TYPE(LOC)?_TRAVERSE_MATCHER(?:_DECL)?\(
\s*([^\s,]+\s*),
\s*(?:[^\s,]+\s*),
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
\)\s*;\s*$""",
declaration,
flags=re.X,
)
if m:
loc, name, results = m.groups()[0:3]
result_types = [r.strip() for r in results.split(",")]
comment_result_types = extract_result_types(comment)
if comment_result_types and sorted(result_types) != sorted(
comment_result_types
):
raise Exception("Inconsistent documentation for: %s" % name)
for result_type in result_types:
add_matcher(result_type, name, "Matcher<Type>", comment)
# if loc:
# add_matcher('%sLoc' % result_type, '%sLoc' % name, 'Matcher<TypeLoc>',
# comment)
return
m = re.match(
r"""^\s*AST_POLYMORPHIC_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\)
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
p, n, name, results = m.groups()[0:4]
args = m.groups()[4:]
result_types = [r.strip() for r in results.split(",")]
if allowed_types and allowed_types != result_types:
raise Exception("Inconsistent documentation for: %s" % name)
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
m = re.match(
r"""^\s*AST_POLYMORPHIC_MATCHER_REGEX(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),
\s*([^\s,]+)\s*
(?:,\s*\d+\s*)?
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
name, results, arg_name = m.groups()[0:3]
result_types = [r.strip() for r in results.split(",")]
if allowed_types and allowed_types != result_types:
raise Exception("Inconsistent documentation for: %s" % name)
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
If the matcher is used in clang-query, RegexFlags parameter
should be passed as a quoted string. e.g: "NoFlags".
Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
"""
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
m = re.match(
r"""^\s*AST_MATCHER_FUNCTION(_P)?(.?)(?:_OVERLOAD)?\(
(?:\s*([^\s,]+)\s*,)?
\s*([^\s,]+)\s*
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{\s*$""",
declaration,
flags=re.X,
)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
add_matcher(result, name, args, comment)
return
m = re.match(
r"""^\s*AST_MATCHER(_P)?(.?)(?:_OVERLOAD)?\(
(?:\s*([^\s,]+)\s*,)?
\s*([^\s,]+)\s*
(?:,\s*([^,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*([^\s,]+)\s*
,\s*([^\s,]+)\s*)?
(?:,\s*\d+\s*)?
\)\s*{""",
declaration,
flags=re.X,
)
if m:
p, n, result, name = m.groups()[0:4]
args = m.groups()[4:]
if not result:
if not allowed_types:
raise Exception("Did not find allowed result types for: %s" % name)
result_types = allowed_types
else:
result_types = [result]
if n not in ["", "2"]:
raise Exception('Cannot parse "%s"' % declaration)
args = ", ".join(
"%s %s" % (args[i], args[i + 1])
for i in range(0, len(args), 2)
if args[i]
)
for result_type in result_types:
add_matcher(result_type, name, args, comment)
return
m = re.match(
r"""^\s*AST_MATCHER_REGEX(?:_OVERLOAD)?\(
\s*([^\s,]+)\s*,
\s*([^\s,]+)\s*,
\s*([^\s,]+)\s*
(?:,\s*\d+\s*)?
\)\s*{""",
declaration,
flags=re.X,
)
if m:
result, name, arg_name = m.groups()[0:3]
if not result:
if not allowed_types:
raise Exception("Did not find allowed result types for: %s" % name)
result_types = allowed_types
else:
result_types = [result]
arg = "StringRef %s, Regex::RegexFlags Flags = NoFlags" % arg_name
comment += """
If the matcher is used in clang-query, RegexFlags parameter
should be passed as a quoted string. e.g: "NoFlags".
Flags can be combined with '|' example \"IgnoreCase | BasicRegex\"
"""
for result_type in result_types:
add_matcher(result_type, name, arg, comment)
return
# Parse ArgumentAdapting matchers.
m = re.match(
r"""^.*ArgumentAdaptingMatcherFunc<.*>\s*
([a-zA-Z]*);$""",
declaration,
flags=re.X,
)
if m:
name = m.groups()[0]
add_matcher("*", name, "Matcher<*>", comment)
return
# Parse Variadic functions.
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*([^,]+),\s*([^,]+),\s*[^>]+>\s*
([a-zA-Z]*);$""",
declaration,
flags=re.X,
)
if m:
result, arg, name = m.groups()[:3]
add_matcher(result, name, "%s, ..., %s" % (arg, arg), comment)
return
m = re.match(
r"""^.*internal::VariadicFunction\s*<\s*
internal::PolymorphicMatcher<[\S\s]+
AST_POLYMORPHIC_SUPPORTED_TYPES\(([^)]*)\),\s*(.*);$""",
declaration,
flags=re.X,
)
if m:
results, trailing = m.groups()
trailing, name = trailing.rsplit(">", 1)
name = name.strip()
trailing, _ = trailing.rsplit(",", 1)
_, arg = trailing.rsplit(",", 1)
arg = arg.strip()
result_types = [r.strip() for r in results.split(",")]
for result_type in result_types:
add_matcher(result_type, name, "%s, ..., %s" % (arg, arg), comment)
return
# Parse Variadic operator matchers.
m = re.match(
r"""^.*VariadicOperatorMatcherFunc\s*<\s*([^,]+),\s*([^\s]+)\s*>\s*
([a-zA-Z]*);$""",
declaration,
flags=re.X,
)
if m:
min_args, max_args, name = m.groups()[:3]
if max_args == "1":
add_matcher("*", name, "Matcher<*>", comment)
return
elif max_args == "std::numeric_limits<unsigned>::max()":
add_matcher("*", name, "Matcher<*>, ..., Matcher<*>", comment)
return
m = re.match(
r"""^.*MapAnyOfMatcher<.*>\s*
([a-zA-Z]*);$""",
declaration,
flags=re.X,
)
if m:
name = m.groups()[0]
add_matcher("*", name, "Matcher<*>...Matcher<*>", comment)
return
# Parse free standing matcher functions, like:
# Matcher<ResultType> Name(Matcher<ArgumentType> InnerMatcher) {
m = re.match(
r"""^\s*(?:template\s+<\s*(?:class|typename)\s+(.+)\s*>\s+)?
(.*)\s+
([^\s\(]+)\s*\(
(.*)
\)\s*{""",
declaration,
re.X,
)
if m:
template_name, result, name, args = m.groups()
if template_name:
matcherTemplateArgs = re.findall(
r"Matcher<\s*(%s)\s*>" % template_name, args
)
templateArgs = re.findall(
r"(?:^|[\s,<])(%s)(?:$|[\s,>])" % template_name, args
)
if len(matcherTemplateArgs) < len(templateArgs):
# The template name is used naked, so don't replace with `*`` later on
template_name = None
else:
args = re.sub(
r"(^|[\s,<])%s($|[\s,>])" % template_name, r"\1*\2", args
)
args = ", ".join(p.strip() for p in args.split(","))
m = re.match(r"(?:^|.*\s+)internal::(?:Bindable)?Matcher<([^>]+)>$", result)
if m:
result_types = [m.group(1)]
if (
template_name
and len(result_types) == 1
and result_types[0] == template_name
):
result_types = ["*"]
else:
result_types = extract_result_types(comment)
if not result_types:
if not comment:
# Only overloads don't have their own doxygen comments; ignore those.
print('Ignoring "%s"' % name)
else:
print('Cannot determine result type for "%s"' % name)
else:
for result_type in result_types:
add_matcher(result_type, name, args, comment)
else:
print('*** Unparsable: "' + declaration + '" ***')
def sort_table(matcher_type, matcher_map):
"""Returns the sorted html table for the given row map."""
table = ""
for key in sorted(matcher_map.keys()):
table += matcher_map[key] + "\n"
return (
"<!-- START_%(type)s_MATCHERS -->\n"
+ "%(table)s"
+ "<!--END_%(type)s_MATCHERS -->"
) % {
"type": matcher_type,
"table": table,
}
# Parse the ast matchers.
# We alternate between two modes:
# body = True: We parse the definition of a matcher. We need
# to parse the full definition before adding a matcher, as the
# definition might contain static asserts that specify the result
# type.
# body = False: We parse the comments and declaration of the matcher.
comment = ""
declaration = ""
allowed_types = []
body = False
for line in open(MATCHERS_FILE).read().splitlines():
if body:
if line.strip() and line[0] == "}":
if declaration:
act_on_decl(declaration, comment, allowed_types)
comment = ""
declaration = ""
allowed_types = []
body = False
else:
m = re.search(r"is_base_of<([^,]+), NodeType>", line)
if m and m.group(1):
allowed_types += [m.group(1)]
continue
if line.strip() and line.lstrip()[0] == "/":
comment += re.sub(r"^/+\s?", "", line) + "\n"
else:
declaration += " " + line
if (
(not line.strip())
or line.rstrip()[-1] == ";"
or (line.rstrip()[-1] == "{" and line.rstrip()[-3:] != "= {")
):
if line.strip() and line.rstrip()[-1] == "{":
body = True
else:
act_on_decl(declaration, comment, allowed_types)
comment = ""
declaration = ""
allowed_types = []
node_matcher_table = sort_table("DECL", node_matchers)
narrowing_matcher_table = sort_table("NARROWING", narrowing_matchers)
traversal_matcher_table = sort_table("TRAVERSAL", traversal_matchers)
reference = open("../LibASTMatchersReference.html").read()
reference = re.sub(
r"<!-- START_DECL_MATCHERS.*END_DECL_MATCHERS -->",
node_matcher_table,
reference,
flags=re.S,
)
reference = re.sub(
r"<!-- START_NARROWING_MATCHERS.*END_NARROWING_MATCHERS -->",
narrowing_matcher_table,
reference,
flags=re.S,
)
reference = re.sub(
r"<!-- START_TRAVERSAL_MATCHERS.*END_TRAVERSAL_MATCHERS -->",
traversal_matcher_table,
reference,
flags=re.S,
)
with open("../LibASTMatchersReference.html", "w", newline="\n") as output:
output.write(reference)
Codebase Browser
llvm