cpython/Parser/lexer/state.c

#include "Python.h"
#include "pycore_pystate.h"
#include "pycore_token.h"
#include "errcode.h"

#include "state.h"

/* Never change this */
#define TABSIZE …

/* Create and initialize a new tok_state structure */
struct tok_state *
_PyTokenizer_tok_new(void)
{ … }

static void
free_fstring_expressions(struct tok_state *tok)
{ … }

/* Free a tok_state structure */
void
_PyTokenizer_Free(struct tok_state *tok)
{ … }

void
_PyToken_Free(struct token *token) { … }

void
_PyToken_Init(struct token *token) { … }

int
_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
                         int end_col_offset, const char *start, const char *end)
{ … }

int
_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
{ … }