cpython/Parser/tokenizer/tokenizer.h

#ifndef Py_TOKENIZER_H
#define Py_TOKENIZER_H

#include "Python.h"

struct tok_state *_PyTokenizer_FromString(const char *, int, int);
struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int);
struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
                                              const char *, const char *);

#define tok_dump …

#endif /* !Py_TOKENIZER_H */