cpython/Parser/pegen.c

#include <Python.h>
#include "pycore_ast.h"           // _PyAST_Validate(),
#include "pycore_pystate.h"       // _PyThreadState_GET()
#include "pycore_pyerrors.h"      // PyExc_IncompleteInputError
#include <errcode.h>

#include "lexer/lexer.h"
#include "tokenizer/tokenizer.h"
#include "pegen.h"

// Internal parser functions

asdl_stmt_seq*
_PyPegen_interactive_exit(Parser *p)
{ … }

Py_ssize_t
_PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset)
{ … }

Py_ssize_t
_PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset)
{ … }

Py_ssize_t
_PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
{ … }

// Here, mark is the start of the node, while p->mark is the end.
// If node==NULL, they should be the same.
int
_PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
{ … }

// Like _PyPegen_insert_memo(), but updates an existing node if found.
int
_PyPegen_update_memo(Parser *p, int mark, int type, void *node)
{ … }

static int
init_normalization(Parser *p)
{ … }

static int
growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { … }

static int
growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { … }

static void
growable_comment_array_deallocate(growable_comment_array *arr) { … }

static int
_get_keyword_or_name_type(Parser *p, struct token *new_token)
{ … }

static int
initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { … }

static int
_resize_tokens_array(Parser *p) { … }

int
_PyPegen_fill_token(Parser *p)
{ … }

#if defined(Py_DEBUG)
// Instrumentation to count the effectiveness of memoization.
// The array counts the number of tokens skipped by memoization,
// indexed by type.

#define NSTATISTICS …
#define memo_statistics …

#ifdef Py_GIL_DISABLED
#define MUTEX_LOCK …
#define MUTEX_UNLOCK …
#else
#define MUTEX_LOCK …
#define MUTEX_UNLOCK …
#endif

void
_PyPegen_clear_memo_statistics(void)
{
    MUTEX_LOCK();
    for (int i = 0; i < NSTATISTICS; i++) {
        memo_statistics[i] = 0;
    }
    MUTEX_UNLOCK();
}

PyObject *
_PyPegen_get_memo_statistics(void)
{
    PyObject *ret = PyList_New(NSTATISTICS);
    if (ret == NULL) {
        return NULL;
    }

    MUTEX_LOCK();
    for (int i = 0; i < NSTATISTICS; i++) {
        PyObject *value = PyLong_FromLong(memo_statistics[i]);
        if (value == NULL) {
            MUTEX_UNLOCK();
            Py_DECREF(ret);
            return NULL;
        }
        // PyList_SetItem borrows a reference to value.
        if (PyList_SetItem(ret, i, value) < 0) {
            MUTEX_UNLOCK();
            Py_DECREF(ret);
            return NULL;
        }
    }
    MUTEX_UNLOCK();
    return ret;
}
#endif

int  // bool
_PyPegen_is_memoized(Parser *p, int type, void *pres)
{ … }

int
_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
{ … }

int
_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
{ … }

int
_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
{ … }

int
_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
{ … }

Token *
_PyPegen_expect_token(Parser *p, int type)
{ … }

void*
_PyPegen_expect_forced_result(Parser *p, void* result, const char* expected) { … }

Token *
_PyPegen_expect_forced_token(Parser *p, int type, const char* expected) { … }

expr_ty
_PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
{ … }

Token *
_PyPegen_get_last_nonnwhitespace_token(Parser *p)
{ … }

PyObject *
_PyPegen_new_identifier(Parser *p, const char *n)
{ … }

static expr_ty
_PyPegen_name_from_token(Parser *p, Token* t)
{ … }

expr_ty
_PyPegen_name_token(Parser *p)
{ … }

void *
_PyPegen_string_token(Parser *p)
{ … }

expr_ty _PyPegen_soft_keyword_token(Parser *p) { … }

static PyObject *
parsenumber_raw(const char *s)
{ … }

static PyObject *
parsenumber(const char *s)
{ … }

expr_ty
_PyPegen_number_token(Parser *p)
{ … }

/* Check that the source for a single input statement really is a single
   statement by looking at what is left in the buffer after parsing.
   Trailing whitespace and comments are OK. */
static int // bool
bad_single_statement(Parser *p)
{ … }

static int
compute_parser_flags(PyCompilerFlags *flags)
{ … }

// Parser API

Parser *
_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
                    int feature_version, int *errcode, PyArena *arena)
{ … }

void
_PyPegen_Parser_Free(Parser *p)
{ … }

static void
reset_parser_state_for_error_pass(Parser *p)
{ … }

static inline int
_is_end_of_source(Parser *p) { … }

void *
_PyPegen_run_parser(Parser *p)
{ … }

mod_ty
_PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
                             const char *enc, const char *ps1, const char *ps2,
                             PyCompilerFlags *flags, int *errcode,
                             PyObject **interactive_src, PyArena *arena)
{ … }

mod_ty
_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
                       PyCompilerFlags *flags, PyArena *arena)
{ … }