cpython/Python/pylifecycle.c

/* Python interpreter top-level routines, including init/exit */

#include "Python.h"

#include "pycore_call.h"          // _PyObject_CallMethod()
#include "pycore_ceval.h"         // _PyEval_FiniGIL()
#include "pycore_codecs.h"        // _PyCodec_Lookup()
#include "pycore_context.h"       // _PyContext_Init()
#include "pycore_dict.h"          // _PyDict_Fini()
#include "pycore_exceptions.h"    // _PyExc_InitTypes()
#include "pycore_fileutils.h"     // _Py_ResetForceASCII()
#include "pycore_freelist.h"      // _PyObject_ClearFreeLists()
#include "pycore_floatobject.h"   // _PyFloat_InitTypes()
#include "pycore_global_objects_fini_generated.h"  // "_PyStaticObjects_CheckRefcnt()
#include "pycore_import.h"        // _PyImport_BootstrapImp()
#include "pycore_initconfig.h"    // _PyStatus_OK()
#include "pycore_list.h"          // _PyList_Fini()
#include "pycore_long.h"          // _PyLong_InitTypes()
#include "pycore_object.h"        // _PyDebug_PrintTotalRefs()
#include "pycore_pathconfig.h"    // _PyPathConfig_UpdateGlobal()
#include "pycore_pyerrors.h"      // _PyErr_Occurred()
#include "pycore_pylifecycle.h"   // _PyErr_Print()
#include "pycore_pymem.h"         // _PyObject_DebugMallocStats()
#include "pycore_pystate.h"       // _PyThreadState_GET()
#include "pycore_runtime.h"       // _Py_ID()
#include "pycore_runtime_init.h"  // _PyRuntimeState_INIT
#include "pycore_setobject.h"     // _PySet_NextEntry()
#include "pycore_sliceobject.h"   // _PySlice_Fini()
#include "pycore_sysmodule.h"     // _PySys_ClearAuditHooks()
#include "pycore_traceback.h"     // _Py_DumpTracebackThreads()
#include "pycore_uniqueid.h"      // _PyObject_FinalizeUniqueIdPool()
#include "pycore_typeobject.h"    // _PyTypes_InitTypes()
#include "pycore_typevarobject.h" // _Py_clear_generic_types()
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
#include "pycore_weakref.h"       // _PyWeakref_GET_REF()
#include "pycore_obmalloc.h"      // _PyMem_init_obmalloc()

#include "opcode.h"

#include <locale.h>               // setlocale()
#include <stdlib.h>               // getenv()
#ifdef HAVE_UNISTD_H
#  include <unistd.h>             // isatty()
#endif

#if defined(__APPLE__)
#  include <mach-o/loader.h>
#endif

#ifdef HAVE_SIGNAL_H
#  include <signal.h>             // SIG_IGN
#endif

#ifdef HAVE_LANGINFO_H
#  include <langinfo.h>           // nl_langinfo(CODESET)
#endif

#ifdef HAVE_FCNTL_H
#  include <fcntl.h>              // F_GETFD
#endif

#ifdef MS_WINDOWS
#  undef BYTE
#endif

#define PUTS(fd, str) …


/* Forward declarations */
static PyStatus add_main_module(PyInterpreterState *interp);
static PyStatus init_import_site(void);
static PyStatus init_set_builtins_open(void);
static PyStatus init_sys_streams(PyThreadState *tstate);
#ifdef __ANDROID__
static PyStatus init_android_streams(PyThreadState *tstate);
#endif
static void wait_for_thread_shutdown(PyThreadState *tstate);
static void finalize_subinterpreters(void);
static void call_ll_exitfuncs(_PyRuntimeState *runtime);

/* The following places the `_PyRuntime` structure in a location that can be
 * found without any external information. This is meant to ease access to the
 * interpreter state for various runtime debugging tools, but is *not* an
 * officially supported feature */

/* Suppress deprecation warning for PyBytesObject.ob_shash */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS

#if defined(MS_WINDOWS)

#pragma section("PyRuntime", read, write)
__declspec(allocate("PyRuntime"))

#elif defined(__APPLE__)

__attribute__((
    section(SEG_DATA ",PyRuntime")
))

#endif

_PyRuntimeState _PyRuntime
#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__))
__attribute__ ((section (".PyRuntime")))
#endif
= …;
_Py_COMP_DIAG_POP

static int runtime_initialized = …;

PyStatus
_PyRuntime_Initialize(void)
{ … }

void
_PyRuntime_Finalize(void)
{ … }

int
Py_IsFinalizing(void)
{ … }

/* Hack to force loading of object files */
int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = …; /* Python/pystrcmp.o */


/* APIs to access the initialization flags
 *
 * Can be called prior to Py_Initialize.
 */

int
_Py_IsCoreInitialized(void)
{ … }

int
Py_IsInitialized(void)
{ … }


/* Helper functions to better handle the legacy C locale
 *
 * The legacy C locale assumes ASCII as the default text encoding, which
 * causes problems not only for the CPython runtime, but also other
 * components like GNU readline.
 *
 * Accordingly, when the CLI detects it, it attempts to coerce it to a
 * more capable UTF-8 based alternative as follows:
 *
 *     if (_Py_LegacyLocaleDetected()) {
 *         _Py_CoerceLegacyLocale();
 *     }
 *
 * See the documentation of the PYTHONCOERCECLOCALE setting for more details.
 *
 * Locale coercion also impacts the default error handler for the standard
 * streams: while the usual default is "strict", the default for the legacy
 * C locale and for any of the coercion target locales is "surrogateescape".
 */

int
_Py_LegacyLocaleDetected(int warn)
{ … }

#ifndef MS_WINDOWS
static const char *_C_LOCALE_WARNING = …;

static void
emit_stderr_warning_for_legacy_locale(_PyRuntimeState *runtime)
{ … }
#endif   /* !defined(MS_WINDOWS) */

_LocaleCoercionTarget;

static _LocaleCoercionTarget _TARGET_LOCALES[] = …;


int
_Py_IsLocaleCoercionTarget(const char *ctype_loc)
{ … }


#ifdef PY_COERCE_C_LOCALE
static const char C_LOCALE_COERCION_WARNING[] = …;

static int
_coerce_default_locale_settings(int warn, const _LocaleCoercionTarget *target)
{ … }
#endif

int
_Py_CoerceLegacyLocale(int warn)
{ … }

/* _Py_SetLocaleFromEnv() is a wrapper around setlocale(category, "") to
 * isolate the idiosyncrasies of different libc implementations. It reads the
 * appropriate environment variable and uses its value to select the locale for
 * 'category'. */
char *
_Py_SetLocaleFromEnv(int category)
{ … }


static int
interpreter_update_config(PyThreadState *tstate, int only_update_path_config)
{ … }


int
_PyInterpreterState_SetConfig(const PyConfig *src_config)
{ … }


/* Global initializations.  Can be undone by Py_Finalize().  Don't
   call this twice without an intervening Py_Finalize() call.

   Every call to Py_InitializeFromConfig, Py_Initialize or Py_InitializeEx
   must have a corresponding call to Py_Finalize.

   Locking: you must hold the interpreter lock while calling these APIs.
   (If the lock has not yet been initialized, that's equivalent to
   having the lock, but you cannot use multiple threads.)

*/

static PyStatus
pyinit_core_reconfigure(_PyRuntimeState *runtime,
                        PyThreadState **tstate_p,
                        const PyConfig *config)
{ … }


static PyStatus
pycore_init_runtime(_PyRuntimeState *runtime,
                    const PyConfig *config)
{ … }


static PyStatus
init_interp_settings(PyInterpreterState *interp,
                     const PyInterpreterConfig *config)
{ … }


static void
init_interp_create_gil(PyThreadState *tstate, int gil)
{ … }

static int
builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value)
{ … }

static PyStatus
pycore_create_interpreter(_PyRuntimeState *runtime,
                          const PyConfig *src_config,
                          PyThreadState **tstate_p)
{ … }


static PyStatus
pycore_init_global_objects(PyInterpreterState *interp)
{ … }


static PyStatus
pycore_init_types(PyInterpreterState *interp)
{ … }

static PyStatus
pycore_init_builtins(PyThreadState *tstate)
{ … }


static PyStatus
pycore_interp_init(PyThreadState *tstate)
{ … }


static PyStatus
pyinit_config(_PyRuntimeState *runtime,
              PyThreadState **tstate_p,
              const PyConfig *config)
{ … }


PyStatus
_Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
{ … }


PyStatus
Py_PreInitializeFromBytesArgs(const PyPreConfig *src_config, Py_ssize_t argc, char **argv)
{ … }


PyStatus
Py_PreInitializeFromArgs(const PyPreConfig *src_config, Py_ssize_t argc, wchar_t **argv)
{ … }


PyStatus
Py_PreInitialize(const PyPreConfig *src_config)
{ … }


PyStatus
_Py_PreInitializeFromConfig(const PyConfig *config,
                            const _PyArgv *args)
{ … }


/* Begin interpreter initialization
 *
 * On return, the first thread and interpreter state have been created,
 * but the compiler, signal handling, multithreading and
 * multiple interpreter support, and codec infrastructure are not yet
 * available.
 *
 * The import system will support builtin and frozen modules only.
 * The only supported io is writing to sys.stderr
 *
 * If any operation invoked by this function fails, a fatal error is
 * issued and the function does not return.
 *
 * Any code invoked from this function should *not* assume it has access
 * to the Python C API (unless the API is explicitly listed as being
 * safe to call without calling Py_Initialize first)
 */
static PyStatus
pyinit_core(_PyRuntimeState *runtime,
            const PyConfig *src_config,
            PyThreadState **tstate_p)
{ … }


/* Py_Initialize() has already been called: update the main interpreter
   configuration. Example of bpo-34008: Py_Main() called after
   Py_Initialize(). */
static PyStatus
pyinit_main_reconfigure(PyThreadState *tstate)
{ … }


#ifdef Py_DEBUG
static void
run_presite(PyThreadState *tstate)
{
    PyInterpreterState *interp = tstate->interp;
    const PyConfig *config = _PyInterpreterState_GetConfig(interp);

    if (!config->run_presite) {
        return;
    }

    PyObject *presite_modname = PyUnicode_FromWideChar(
        config->run_presite,
        wcslen(config->run_presite)
    );
    if (presite_modname == NULL) {
        fprintf(stderr, "Could not convert pre-site module name to unicode\n");
    }
    else {
        PyObject *presite = PyImport_Import(presite_modname);
        if (presite == NULL) {
            fprintf(stderr, "pre-site import failed:\n");
            _PyErr_Print(tstate);
        }
        Py_XDECREF(presite);
        Py_DECREF(presite_modname);
    }
}
#endif


static PyStatus
init_interp_main(PyThreadState *tstate)
{ … }


/* Update interpreter state based on supplied configuration settings
 *
 * After calling this function, most of the restrictions on the interpreter
 * are lifted. The only remaining incomplete settings are those related
 * to the main module (sys.argv[0], __main__ metadata)
 *
 * Calling this when the interpreter is not initializing, is already
 * initialized or without a valid current thread state is a fatal error.
 * Other errors should be reported as normal Python exceptions with a
 * non-zero return code.
 */
static PyStatus
pyinit_main(PyThreadState *tstate)
{ … }


PyStatus
Py_InitializeFromConfig(const PyConfig *config)
{ … }


void
Py_InitializeEx(int install_sigs)
{ … }

void
Py_Initialize(void)
{ … }


PyStatus
_Py_InitializeMain(void)
{ … }


static void
finalize_modules_delete_special(PyThreadState *tstate, int verbose)
{ … }


static PyObject*
finalize_remove_modules(PyObject *modules, int verbose)
{ … }


static void
finalize_clear_modules_dict(PyObject *modules)
{ … }


static void
finalize_restore_builtins(PyThreadState *tstate)
{ … }


static void
finalize_modules_clear_weaklist(PyInterpreterState *interp,
                                PyObject *weaklist, int verbose)
{ … }


static void
finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose)
{ … }


/* Clear modules, as good as we can */
// XXX Move most of this to import.c.
static void
finalize_modules(PyThreadState *tstate)
{ … }


/* Flush stdout and stderr */

static int
file_is_closed(PyObject *fobj)
{ … }


static int
flush_std_files(void)
{ … }

/* Undo the effect of Py_Initialize().

   Beware: if multiple interpreter and/or thread states exist, these
   are not wiped out; only the current thread and interpreter state
   are deleted.  But since everything else is deleted, those other
   interpreter and thread states should no longer be used.

   (XXX We should do better, e.g. wipe out all interpreters and
   threads.)

   Locking: as above.

*/


static void
finalize_interp_types(PyInterpreterState *interp)
{ … }


static void
finalize_interp_clear(PyThreadState *tstate)
{ … }


static void
finalize_interp_delete(PyInterpreterState *interp)
{ … }


/* Conceptually, there isn't a good reason for Py_Finalize()
   to be called in any other thread than the one where Py_Initialize()
   was called.  Consequently, it would make sense to fail if the thread
   or thread state (or interpreter) don't match.  However, such
   constraints have never been enforced, and, as unlikely as it may be,
   there may be users relying on the unconstrained behavior.  Thus,
   we do our best here to accommodate that possibility. */

static PyThreadState *
resolve_final_tstate(_PyRuntimeState *runtime)
{ … }

static int
_Py_Finalize(_PyRuntimeState *runtime)
{ … }

int
Py_FinalizeEx(void)
{ … }

void
Py_Finalize(void)
{ … }


/* Create and initialize a new interpreter and thread, and return the
   new thread.  This requires that Py_Initialize() has been called
   first.

   Unsuccessful initialization yields a NULL pointer.  Note that *no*
   exception information is available even in this case -- the
   exception information is held in the thread, and there is no
   thread.

   Locking: as above.

*/

static PyStatus
new_interpreter(PyThreadState **tstate_p,
                const PyInterpreterConfig *config, long whence)
{ … }

PyStatus
Py_NewInterpreterFromConfig(PyThreadState **tstate_p,
                            const PyInterpreterConfig *config)
{ … }

PyThreadState *
Py_NewInterpreter(void)
{ … }

/* Delete an interpreter and its last thread.  This requires that the
   given thread state is current, that the thread has no remaining
   frames, and that it is its interpreter's only remaining thread.
   It is a fatal error to violate these constraints.

   (Py_FinalizeEx() doesn't have these constraints -- it zaps
   everything, regardless.)

   Locking: as above.

*/

void
Py_EndInterpreter(PyThreadState *tstate)
{ … }

int
_Py_IsInterpreterFinalizing(PyInterpreterState *interp)
{ … }

static void
finalize_subinterpreters(void)
{ … }


/* Add the __main__ module */

static PyStatus
add_main_module(PyInterpreterState *interp)
{ … }

/* Import the site module (not into __main__ though) */

static PyStatus
init_import_site(void)
{ … }

/* returns Py_None if the fd is not valid */
static PyObject*
create_stdio(const PyConfig *config, PyObject* io,
    int fd, int write_mode, const char* name,
    const wchar_t* encoding, const wchar_t* errors)
{ … }

/* Set builtins.open to io.open */
static PyStatus
init_set_builtins_open(void)
{ … }


/* Create sys.stdin, sys.stdout and sys.stderr */
static PyStatus
init_sys_streams(PyThreadState *tstate)
{ … }


#ifdef __ANDROID__
#include <android/log.h>

static PyObject *
android_log_write_impl(PyObject *self, PyObject *args)
{
    int prio = 0;
    const char *tag = NULL;
    const char *text = NULL;
    if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) {
        return NULL;
    }

    // Despite its name, this function is part of the public API
    // (https://developer.android.com/ndk/reference/group/logging).
    __android_log_write(prio, tag, text);
    Py_RETURN_NONE;
}


static PyMethodDef android_log_write_method = {
    "android_log_write", android_log_write_impl, METH_VARARGS
};


static PyStatus
init_android_streams(PyThreadState *tstate)
{
    PyStatus status = _PyStatus_OK();
    PyObject *_android_support = NULL;
    PyObject *android_log_write = NULL;
    PyObject *result = NULL;

    _android_support = PyImport_ImportModule("_android_support");
    if (_android_support == NULL) {
        goto error;
    }

    android_log_write = PyCFunction_New(&android_log_write_method, NULL);
    if (android_log_write == NULL) {
        goto error;
    }

    // These log priorities match those used by Java's System.out and System.err.
    result = PyObject_CallMethod(
        _android_support, "init_streams", "Oii",
        android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN);
    if (result == NULL) {
        goto error;
    }

    goto done;

error:
    _PyErr_Print(tstate);
    status = _PyStatus_ERR("failed to initialize Android streams");

done:
    Py_XDECREF(result);
    Py_XDECREF(android_log_write);
    Py_XDECREF(_android_support);
    return status;
}

#endif  // __ANDROID__


static void
_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
                              PyThreadState *tstate)
{ … }

/* Print the current exception (if an exception is set) with its traceback,
   or display the current Python stack.

   Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is
   called on catastrophic cases.

   Return 1 if the traceback was displayed, 0 otherwise. */

static int
_Py_FatalError_PrintExc(PyThreadState *tstate)
{ … }

/* Print fatal error message and abort */

#ifdef MS_WINDOWS
static void
fatal_output_debug(const char *msg)
{
    /* buffer of 256 bytes allocated on the stack */
    WCHAR buffer[256 / sizeof(WCHAR)];
    size_t buflen = Py_ARRAY_LENGTH(buffer) - 1;
    size_t msglen;

    OutputDebugStringW(L"Fatal Python error: ");

    msglen = strlen(msg);
    while (msglen) {
        size_t i;

        if (buflen > msglen) {
            buflen = msglen;
        }

        /* Convert the message to wchar_t. This uses a simple one-to-one
           conversion, assuming that the this error message actually uses
           ASCII only. If this ceases to be true, we will have to convert. */
        for (i=0; i < buflen; ++i) {
            buffer[i] = msg[i];
        }
        buffer[i] = L'\0';
        OutputDebugStringW(buffer);

        msg += buflen;
        msglen -= buflen;
    }
    OutputDebugStringW(L"\n");
}
#endif


static void
fatal_error_dump_runtime(int fd, _PyRuntimeState *runtime)
{ … }


static inline void _Py_NO_RETURN
fatal_error_exit(int status)
{ … }

static inline int
acquire_dict_lock_for_dump(PyObject *obj)
{ … }

static inline void
release_dict_lock_for_dump(PyObject *obj)
{ … }

// Dump the list of extension modules of sys.modules, excluding stdlib modules
// (sys.stdlib_module_names), into fd file descriptor.
//
// This function is called by a signal handler in faulthandler: avoid memory
// allocations and keep the implementation simple. For example, the list is not
// sorted on purpose.
void
_Py_DumpExtensionModules(int fd, PyInterpreterState *interp)
{ … }


static void _Py_NO_RETURN
fatal_error(int fd, int header, const char *prefix, const char *msg,
            int status)
{ … }


#undef Py_FatalError

void _Py_NO_RETURN
Py_FatalError(const char *msg)
{ … }


void _Py_NO_RETURN
_Py_FatalErrorFunc(const char *func, const char *msg)
{ … }


void _Py_NO_RETURN
_Py_FatalErrorFormat(const char *func, const char *format, ...)
{ … }


void _Py_NO_RETURN
_Py_FatalRefcountErrorFunc(const char *func, const char *msg)
{ … }


void _Py_NO_RETURN
Py_ExitStatusException(PyStatus status)
{ … }


/* Wait until threading._shutdown completes, provided
   the threading module was imported in the first place.
   The shutdown routine will wait until all non-daemon
   "threading" threads have completed. */
static void
wait_for_thread_shutdown(PyThreadState *tstate)
{ … }

int Py_AtExit(void (*func)(void))
{ … }

static void
call_ll_exitfuncs(_PyRuntimeState *runtime)
{ … }

void _Py_NO_RETURN
Py_Exit(int sts)
{ … }


/*
 * The file descriptor fd is considered ``interactive'' if either
 *   a) isatty(fd) is TRUE, or
 *   b) the -i flag was given, and the filename associated with
 *      the descriptor is NULL or "<stdin>" or "???".
 */
int
Py_FdIsInteractive(FILE *fp, const char *filename)
{ … }


int
_Py_FdIsInteractive(FILE *fp, PyObject *filename)
{ … }


/* Wrappers around sigaction() or signal(). */

PyOS_sighandler_t
PyOS_getsig(int sig)
{ … }

/*
 * All of the code in this function must only use async-signal-safe functions,
 * listed at `man 7 signal` or
 * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
 */
PyOS_sighandler_t
PyOS_setsig(int sig, PyOS_sighandler_t handler)
{ … }