cpython/Python/codecs.c

/* ------------------------------------------------------------------------

   Python Codec Registry and support functions

Written by Marc-Andre Lemburg ([email protected]).

Copyright (c) Corporation for National Research Initiatives.

   ------------------------------------------------------------------------ */

#include "Python.h"
#include "pycore_call.h"          // _PyObject_CallNoArgs()
#include "pycore_interp.h"        // PyInterpreterState.codec_search_path
#include "pycore_lock.h"          // PyMutex
#include "pycore_pyerrors.h"      // _PyErr_FormatNote()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI

static const char *codecs_builtin_error_handlers[] = …;

const char *Py_hexdigits = …;

/* --- Codec Registry ----------------------------------------------------- */

int PyCodec_Register(PyObject *search_function)
{ … }

int
PyCodec_Unregister(PyObject *search_function)
{ … }

extern int _Py_normalize_encoding(const char *, char *, size_t);

/* Convert a string to a normalized Python string(decoded from UTF-8): all characters are
   converted to lower case, spaces and hyphens are replaced with underscores. */

static
PyObject *normalizestring(const char *string)
{ … }

/* Lookup the given encoding and return a tuple providing the codec
   facilities.

   The encoding string is looked up converted to all lower-case
   characters. This makes encodings looked up through this mechanism
   effectively case-insensitive.

   If no codec is found, a LookupError is set and NULL returned.

   As side effect, this tries to load the encodings package, if not
   yet done. This is part of the lazy load strategy for the encodings
   package.

*/

PyObject *_PyCodec_Lookup(const char *encoding)
{ … }

/* Codec registry encoding check API. */

int PyCodec_KnownEncoding(const char *encoding)
{ … }

static
PyObject *args_tuple(PyObject *object,
                     const char *errors)
{ … }

/* Helper function to get a codec item */

static
PyObject *codec_getitem(const char *encoding, int index)
{ … }

/* Helper functions to create an incremental codec. */
static
PyObject *codec_makeincrementalcodec(PyObject *codec_info,
                                     const char *errors,
                                     const char *attrname)
{ … }

static
PyObject *codec_getincrementalcodec(const char *encoding,
                                    const char *errors,
                                    const char *attrname)
{ … }

/* Helper function to create a stream codec. */

static
PyObject *codec_getstreamcodec(const char *encoding,
                               PyObject *stream,
                               const char *errors,
                               const int index)
{ … }

/* Helpers to work with the result of _PyCodec_Lookup

 */
PyObject *_PyCodecInfo_GetIncrementalDecoder(PyObject *codec_info,
                                             const char *errors)
{ … }

PyObject *_PyCodecInfo_GetIncrementalEncoder(PyObject *codec_info,
                                             const char *errors)
{ … }


/* Convenience APIs to query the Codec registry.

   All APIs return a codec object with incremented refcount.

 */

PyObject *PyCodec_Encoder(const char *encoding)
{ … }

PyObject *PyCodec_Decoder(const char *encoding)
{ … }

PyObject *PyCodec_IncrementalEncoder(const char *encoding,
                                     const char *errors)
{ … }

PyObject *PyCodec_IncrementalDecoder(const char *encoding,
                                     const char *errors)
{ … }

PyObject *PyCodec_StreamReader(const char *encoding,
                               PyObject *stream,
                               const char *errors)
{ … }

PyObject *PyCodec_StreamWriter(const char *encoding,
                               PyObject *stream,
                               const char *errors)
{ … }

/* Encode an object (e.g. a Unicode object) using the given encoding
   and return the resulting encoded object (usually a Python string).

   errors is passed to the encoder factory as argument if non-NULL. */

static PyObject *
_PyCodec_EncodeInternal(PyObject *object,
                        PyObject *encoder,
                        const char *encoding,
                        const char *errors)
{ … }

/* Decode an object (usually a Python string) using the given encoding
   and return an equivalent object (e.g. a Unicode object).

   errors is passed to the decoder factory as argument if non-NULL. */

static PyObject *
_PyCodec_DecodeInternal(PyObject *object,
                        PyObject *decoder,
                        const char *encoding,
                        const char *errors)
{ … }

/* Generic encoding/decoding API */
PyObject *PyCodec_Encode(PyObject *object,
                         const char *encoding,
                         const char *errors)
{ … }

PyObject *PyCodec_Decode(PyObject *object,
                         const char *encoding,
                         const char *errors)
{ … }

/* Text encoding/decoding API */
PyObject * _PyCodec_LookupTextEncoding(const char *encoding,
                                       const char *alternate_command)
{ … }


static
PyObject *codec_getitem_checked(const char *encoding,
                                const char *alternate_command,
                                int index)
{ … }

static PyObject * _PyCodec_TextEncoder(const char *encoding)
{ … }

static PyObject * _PyCodec_TextDecoder(const char *encoding)
{ … }

PyObject *_PyCodec_EncodeText(PyObject *object,
                              const char *encoding,
                              const char *errors)
{ … }

PyObject *_PyCodec_DecodeText(PyObject *object,
                              const char *encoding,
                              const char *errors)
{ … }

/* Register the error handling callback function error under the name
   name. This function will be called by the codec when it encounters
   an unencodable characters/undecodable bytes and doesn't know the
   callback name, when name is specified as the error parameter
   in the call to the encode/decode function.
   Return 0 on success, -1 on error */
int PyCodec_RegisterError(const char *name, PyObject *error)
{ … }

int _PyCodec_UnregisterError(const char *name)
{ … }

/* Lookup the error handling callback function registered under the
   name error. As a special case NULL can be passed, in which case
   the error handling callback for strict encoding will be returned. */
PyObject *PyCodec_LookupError(const char *name)
{ … }

static void wrong_exception_type(PyObject *exc)
{ … }

PyObject *PyCodec_StrictErrors(PyObject *exc)
{ … }


PyObject *PyCodec_IgnoreErrors(PyObject *exc)
{ … }


PyObject *PyCodec_ReplaceErrors(PyObject *exc)
{ … }

PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
{ … }

PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
{ … }

PyObject *PyCodec_NameReplaceErrors(PyObject *exc)
{ … }

#define ENC_UNKNOWN …
#define ENC_UTF8 …
#define ENC_UTF16BE …
#define ENC_UTF16LE …
#define ENC_UTF32BE …
#define ENC_UTF32LE …

static int
get_standard_encoding(const char *encoding, int *bytelength)
{ … }

/* This handler is declared static until someone demonstrates
   a need to call it directly. */
static PyObject *
PyCodec_SurrogatePassErrors(PyObject *exc)
{ … }

static PyObject *
PyCodec_SurrogateEscapeErrors(PyObject *exc)
{ … }


static PyObject *strict_errors(PyObject *self, PyObject *exc)
{ … }


static PyObject *ignore_errors(PyObject *self, PyObject *exc)
{ … }


static PyObject *replace_errors(PyObject *self, PyObject *exc)
{ … }


static PyObject *xmlcharrefreplace_errors(PyObject *self, PyObject *exc)
{ … }


static PyObject *backslashreplace_errors(PyObject *self, PyObject *exc)
{ … }

static PyObject *namereplace_errors(PyObject *self, PyObject *exc)
{ … }

static PyObject *surrogatepass_errors(PyObject *self, PyObject *exc)
{ … }

static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc)
{ … }

PyStatus
_PyCodec_InitRegistry(PyInterpreterState *interp)
{ … }

void
_PyCodec_Fini(PyInterpreterState *interp)
{ … }