cpython/Objects/codeobject.c

#include <stdbool.h>

#include "Python.h"
#include "opcode.h"

#include "pycore_code.h"          // _PyCodeConstructor
#include "pycore_frame.h"         // FRAME_SPECIALS_SIZE
#include "pycore_hashtable.h"     // _Py_hashtable_t
#include "pycore_index_pool.h"     // _PyIndexPool
#include "pycore_initconfig.h"    // _PyStatus_OK()
#include "pycore_interp.h"        // PyInterpreterState.co_extra_freefuncs
#include "pycore_object.h"        // _PyObject_SetDeferredRefcount
#include "pycore_object_stack.h"
#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
#include "pycore_opcode_utils.h"  // RESUME_AT_FUNC_START
#include "pycore_pymem.h"         // _PyMem_FreeDelayed
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_setobject.h"     // _PySet_NextEntry()
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_uniqueid.h"      // _PyObject_AssignUniqueId()
#include "clinic/codeobject.c.h"

#define INITIAL_SPECIALIZED_CODE_SIZE …

static const char *
code_event_name(PyCodeEvent event) { … }

static void
notify_code_watchers(PyCodeEvent event, PyCodeObject *co)
{ … }

int
PyCode_AddWatcher(PyCode_WatchCallback callback)
{ … }

static inline int
validate_watcher_id(PyInterpreterState *interp, int watcher_id)
{ … }

int
PyCode_ClearWatcher(int watcher_id)
{ … }

/******************
 * generic helpers
 ******************/

static int
should_intern_string(PyObject *o)
{ … }

#ifdef Py_GIL_DISABLED
static PyObject *intern_one_constant(PyObject *op);
#endif

static int
intern_strings(PyObject *tuple)
{ … }

/* Intern constants. In the default build, this interns selected string
   constants. In the free-threaded build, this also interns non-string
   constants. */
static int
intern_constants(PyObject *tuple, int *modified)
{ … }

/* Return a shallow copy of a tuple that is
   guaranteed to contain exact strings, by converting string subclasses
   to exact strings and complaining if a non-string is found. */
static PyObject*
validate_and_copy_tuple(PyObject *tup)
{ … }

static int
init_co_cached(PyCodeObject *self)
{ … }

/******************
 * _PyCode_New()
 ******************/

// This is also used in compile.c.
void
_Py_set_localsplus_info(int offset, PyObject *name, _PyLocals_Kind kind,
                        PyObject *names, PyObject *kinds)
{ … }

static void
get_localsplus_counts(PyObject *names, PyObject *kinds,
                      int *pnlocals, int *pncellvars,
                      int *pnfreevars)
{ … }

static PyObject *
get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num)
{ … }

int
_PyCode_Validate(struct _PyCodeConstructor *con)
{ … }

extern void
_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts,
                int enable_counters);

#ifdef Py_GIL_DISABLED
static _PyCodeArray * _PyCodeArray_New(Py_ssize_t size);
#endif

static int
init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
{ … }

static int
scan_varint(const uint8_t *ptr)
{ … }

static int
scan_signed_varint(const uint8_t *ptr)
{ … }

static int
get_line_delta(const uint8_t *ptr)
{ … }

static PyObject *
remove_column_info(PyObject *locations)
{ … }

static int
intern_code_constants(struct _PyCodeConstructor *con)
{ … }

/* The caller is responsible for ensuring that the given data is valid. */

PyCodeObject *
_PyCode_New(struct _PyCodeConstructor *con)
{ … }


/******************
 * the legacy "constructors"
 ******************/

PyCodeObject *
PyUnstable_Code_NewWithPosOnlyArgs(
                          int argcount, int posonlyargcount, int kwonlyargcount,
                          int nlocals, int stacksize, int flags,
                          PyObject *code, PyObject *consts, PyObject *names,
                          PyObject *varnames, PyObject *freevars, PyObject *cellvars,
                          PyObject *filename, PyObject *name,
                          PyObject *qualname, int firstlineno,
                          PyObject *linetable,
                          PyObject *exceptiontable)
{ … }

PyCodeObject *
PyUnstable_Code_New(int argcount, int kwonlyargcount,
           int nlocals, int stacksize, int flags,
           PyObject *code, PyObject *consts, PyObject *names,
           PyObject *varnames, PyObject *freevars, PyObject *cellvars,
           PyObject *filename, PyObject *name, PyObject *qualname,
           int firstlineno,
           PyObject *linetable,
           PyObject *exceptiontable)
{ … }

// NOTE: When modifying the construction of PyCode_NewEmpty, please also change
// test.test_code.CodeLocationTest.test_code_new_empty to keep it in sync!

static const uint8_t assert0[6] = …;

static const uint8_t linetable[2] = …;

PyCodeObject *
PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
{ … }


/******************
 * source location tracking (co_lines/co_positions)
 ******************/

int
PyCode_Addr2Line(PyCodeObject *co, int addrq)
{ … }

void
_PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range)
{ … }

int
_PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
{ … }

/* Update *bounds to describe the first and one-past-the-last instructions in
   the same line as lasti.  Return the number of that line, or -1 if lasti is out of bounds. */
int
_PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds)
{ … }

static int
is_no_line_marker(uint8_t b)
{ … }


#define ASSERT_VALID_BOUNDS(bounds) …

static int
next_code_delta(PyCodeAddressRange *bounds)
{ … }

static int
previous_code_delta(PyCodeAddressRange *bounds)
{ … }

static int
read_byte(PyCodeAddressRange *bounds)
{ … }

static int
read_varint(PyCodeAddressRange *bounds)
{ … }

static int
read_signed_varint(PyCodeAddressRange *bounds)
{ … }

static void
retreat(PyCodeAddressRange *bounds)
{ … }

static void
advance(PyCodeAddressRange *bounds)
{ … }

static void
advance_with_locations(PyCodeAddressRange *bounds, int *endline, int *column, int *endcolumn)
{ … }
int
PyCode_Addr2Location(PyCodeObject *co, int addrq,
                     int *start_line, int *start_column,
                     int *end_line, int *end_column)
{ … }


static inline int
at_end(PyCodeAddressRange *bounds) { … }

int
_PyLineTable_PreviousAddressRange(PyCodeAddressRange *range)
{ … }

int
_PyLineTable_NextAddressRange(PyCodeAddressRange *range)
{ … }

static int
emit_pair(PyObject **bytes, int *offset, int a, int b)
{ … }

static int
emit_delta(PyObject **bytes, int bdelta, int ldelta, int *offset)
{ … }

static PyObject *
decode_linetable(PyCodeObject *code)
{ … }


lineiterator;


static void
lineiter_dealloc(PyObject *self)
{ … }

static PyObject *
_source_offset_converter(int *value) { … }

static PyObject *
lineiter_next(PyObject *self)
{ … }

PyTypeObject _PyLineIterator = …;

static lineiterator *
new_linesiterator(PyCodeObject *code)
{ … }

/* co_positions iterator object. */
positionsiterator;

static void
positionsiter_dealloc(PyObject *self)
{ … }

static PyObject*
positionsiter_next(PyObject *self)
{ … }

PyTypeObject _PyPositionsIterator = …;

static PyObject*
code_positionsiterator(PyObject *self, PyObject* Py_UNUSED(args))
{ … }


/******************
 * "extra" frame eval info (see PEP 523)
 ******************/

/* Holder for co_extra information */
_PyCodeObjectExtra;


int
PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
{ … }


int
PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
{ … }


/******************
 * other PyCodeObject accessor functions
 ******************/

static PyObject *
get_cached_locals(PyCodeObject *co, PyObject **cached_field,
    _PyLocals_Kind kind, int num)
{ … }

PyObject *
_PyCode_GetVarnames(PyCodeObject *co)
{ … }

PyObject *
PyCode_GetVarnames(PyCodeObject *code)
{ … }

PyObject *
_PyCode_GetCellvars(PyCodeObject *co)
{ … }

PyObject *
PyCode_GetCellvars(PyCodeObject *code)
{ … }

PyObject *
_PyCode_GetFreevars(PyCodeObject *co)
{ … }

PyObject *
PyCode_GetFreevars(PyCodeObject *code)
{ … }

#ifdef _Py_TIER2

static void
clear_executors(PyCodeObject *co)
{
    assert(co->co_executors);
    for (int i = 0; i < co->co_executors->size; i++) {
        if (co->co_executors->executors[i]) {
            _Py_ExecutorDetach(co->co_executors->executors[i]);
            assert(co->co_executors->executors[i] == NULL);
        }
    }
    PyMem_Free(co->co_executors);
    co->co_executors = NULL;
}

void
_PyCode_Clear_Executors(PyCodeObject *code)
{
    clear_executors(code);
}

#endif

static void
deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions)
{ … }

PyObject *
_PyCode_GetCode(PyCodeObject *co)
{ … }

PyObject *
PyCode_GetCode(PyCodeObject *co)
{ … }

/******************
 * PyCode_Type
 ******************/

/*[clinic input]
class code "PyCodeObject *" "&PyCode_Type"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=78aa5d576683bb4b]*/

/*[clinic input]
@classmethod
code.__new__ as code_new

    argcount: int
    posonlyargcount: int
    kwonlyargcount: int
    nlocals: int
    stacksize: int
    flags: int
    codestring as code: object(subclass_of="&PyBytes_Type")
    constants as consts: object(subclass_of="&PyTuple_Type")
    names: object(subclass_of="&PyTuple_Type")
    varnames: object(subclass_of="&PyTuple_Type")
    filename: unicode
    name: unicode
    qualname: unicode
    firstlineno: int
    linetable: object(subclass_of="&PyBytes_Type")
    exceptiontable: object(subclass_of="&PyBytes_Type")
    freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
    cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
    /

Create a code object.  Not for the faint of heart.
[clinic start generated code]*/

static PyObject *
code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
              int kwonlyargcount, int nlocals, int stacksize, int flags,
              PyObject *code, PyObject *consts, PyObject *names,
              PyObject *varnames, PyObject *filename, PyObject *name,
              PyObject *qualname, int firstlineno, PyObject *linetable,
              PyObject *exceptiontable, PyObject *freevars,
              PyObject *cellvars)
/*[clinic end generated code: output=069fa20d299f9dda input=e31da3c41ad8064a]*/
{ … }

static void
free_monitoring_data(_PyCoMonitoringData *data)
{ … }

static void
code_dealloc(PyCodeObject *co)
{ … }

#ifdef Py_GIL_DISABLED
static int
code_traverse(PyObject *self, visitproc visit, void *arg)
{
    PyCodeObject *co = (PyCodeObject*)self;
    Py_VISIT(co->co_consts);
    return 0;
}
#endif

static PyObject *
code_repr(PyObject *self)
{ … }

static PyObject *
code_richcompare(PyObject *self, PyObject *other, int op)
{ … }

static Py_hash_t
code_hash(PyObject *self)
{ … }


#define OFF(x) …

static PyMemberDef code_memberlist[] = …;


static PyObject *
code_getlnotab(PyObject *self, void *closure)
{ … }

static PyObject *
code_getvarnames(PyObject *self, void *closure)
{ … }

static PyObject *
code_getcellvars(PyObject *self, void *closure)
{ … }

static PyObject *
code_getfreevars(PyObject *self, void *closure)
{ … }

static PyObject *
code_getcodeadaptive(PyObject *self, void *closure)
{ … }

static PyObject *
code_getcode(PyObject *self, void *closure)
{ … }

static PyGetSetDef code_getsetlist[] = …;


static PyObject *
code_sizeof(PyObject *self, PyObject *Py_UNUSED(args))
{ … }

static PyObject *
code_linesiterator(PyObject *self, PyObject *Py_UNUSED(args))
{ … }

/*[clinic input]
@text_signature "($self, /, **changes)"
code.replace

    *
    co_argcount: int(c_default="self->co_argcount") = unchanged
    co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged
    co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged
    co_nlocals: int(c_default="self->co_nlocals") = unchanged
    co_stacksize: int(c_default="self->co_stacksize") = unchanged
    co_flags: int(c_default="self->co_flags") = unchanged
    co_firstlineno: int(c_default="self->co_firstlineno") = unchanged
    co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged
    co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged
    co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged
    co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged
    co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged
    co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged
    co_filename: unicode(c_default="self->co_filename") = unchanged
    co_name: unicode(c_default="self->co_name") = unchanged
    co_qualname: unicode(c_default="self->co_qualname") = unchanged
    co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged
    co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged

Return a copy of the code object with new values for the specified fields.
[clinic start generated code]*/

static PyObject *
code_replace_impl(PyCodeObject *self, int co_argcount,
                  int co_posonlyargcount, int co_kwonlyargcount,
                  int co_nlocals, int co_stacksize, int co_flags,
                  int co_firstlineno, PyObject *co_code, PyObject *co_consts,
                  PyObject *co_names, PyObject *co_varnames,
                  PyObject *co_freevars, PyObject *co_cellvars,
                  PyObject *co_filename, PyObject *co_name,
                  PyObject *co_qualname, PyObject *co_linetable,
                  PyObject *co_exceptiontable)
/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/
{ … }

/*[clinic input]
code._varname_from_oparg

    oparg: int

(internal-only) Return the local variable name for the given oparg.

WARNING: this method is for internal use only and may change or go away.
[clinic start generated code]*/

static PyObject *
code__varname_from_oparg_impl(PyCodeObject *self, int oparg)
/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/
{ … }

/* XXX code objects need to participate in GC? */

static struct PyMethodDef code_methods[] = …;


PyTypeObject PyCode_Type = …;


/******************
 * other API
 ******************/

PyObject*
_PyCode_ConstantKey(PyObject *op)
{ … }

#ifdef Py_GIL_DISABLED
static PyObject *
intern_one_constant(PyObject *op)
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    _Py_hashtable_t *consts = interp->code_state.constants;

    assert(!PyUnicode_CheckExact(op));  // strings are interned separately

    _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op);
    if (entry == NULL) {
        if (_Py_hashtable_set(consts, op, op) != 0) {
            return NULL;
        }

#ifdef Py_REF_DEBUG
        Py_ssize_t refcnt = Py_REFCNT(op);
        if (refcnt != 1) {
            // Adjust the reftotal to account for the fact that we only
            // restore a single reference in _PyCode_Fini.
            _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1));
        }
#endif

        _Py_SetImmortal(op);
        return op;
    }

    assert(_Py_IsImmortal(entry->value));
    return (PyObject *)entry->value;
}

static int
compare_constants(const void *key1, const void *key2) {
    PyObject *op1 = (PyObject *)key1;
    PyObject *op2 = (PyObject *)key2;
    if (op1 == op2) {
        return 1;
    }
    if (Py_TYPE(op1) != Py_TYPE(op2)) {
        return 0;
    }
    // We compare container contents by identity because we have already
    // internalized the items.
    if (PyTuple_CheckExact(op1)) {
        Py_ssize_t size = PyTuple_GET_SIZE(op1);
        if (size != PyTuple_GET_SIZE(op2)) {
            return 0;
        }
        for (Py_ssize_t i = 0; i < size; i++) {
            if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) {
                return 0;
            }
        }
        return 1;
    }
    else if (PyFrozenSet_CheckExact(op1)) {
        if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) {
            return 0;
        }
        Py_ssize_t pos1 = 0, pos2 = 0;
        PyObject *obj1, *obj2;
        Py_hash_t hash1, hash2;
        while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) &&
               (_PySet_NextEntry(op2, &pos2, &obj2, &hash2)))
        {
            if (obj1 != obj2) {
                return 0;
            }
        }
        return 1;
    }
    else if (PySlice_Check(op1)) {
        PySliceObject *s1 = (PySliceObject *)op1;
        PySliceObject *s2 = (PySliceObject *)op2;
        return (s1->start == s2->start &&
                s1->stop  == s2->stop  &&
                s1->step  == s2->step);
    }
    else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) {
        return PyObject_RichCompareBool(op1, op2, Py_EQ);
    }
    else if (PyFloat_CheckExact(op1)) {
        // Ensure that, for example, +0.0 and -0.0 are distinct
        double f1 = PyFloat_AS_DOUBLE(op1);
        double f2 = PyFloat_AS_DOUBLE(op2);
        return memcmp(&f1, &f2, sizeof(double)) == 0;
    }
    else if (PyComplex_CheckExact(op1)) {
        Py_complex c1 = ((PyComplexObject *)op1)->cval;
        Py_complex c2 = ((PyComplexObject *)op2)->cval;
        return memcmp(&c1, &c2, sizeof(Py_complex)) == 0;
    }
    _Py_FatalErrorFormat("unexpected type in compare_constants: %s",
                         Py_TYPE(op1)->tp_name);
    return 0;
}

static Py_uhash_t
hash_const(const void *key)
{
    PyObject *op = (PyObject *)key;
    if (PySlice_Check(op)) {
        PySliceObject *s = (PySliceObject *)op;
        PyObject *data[3] = { s->start, s->stop, s->step };
        return Py_HashBuffer(&data, sizeof(data));
    }
    else if (PyTuple_CheckExact(op)) {
        Py_ssize_t size = PyTuple_GET_SIZE(op);
        PyObject **data = _PyTuple_ITEMS(op);
        return Py_HashBuffer(data, sizeof(PyObject *) * size);
    }
    Py_hash_t h = PyObject_Hash(op);
    if (h == -1) {
        // This should never happen: all the constants we support have
        // infallible hash functions.
        Py_FatalError("code: hash failed");
    }
    return (Py_uhash_t)h;
}

static int
clear_containers(_Py_hashtable_t *ht, const void *key, const void *value,
                 void *user_data)
{
    // First clear containers to avoid recursive deallocation later on in
    // destroy_key.
    PyObject *op = (PyObject *)key;
    if (PyTuple_CheckExact(op)) {
        for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) {
            Py_CLEAR(_PyTuple_ITEMS(op)[i]);
        }
    }
    else if (PySlice_Check(op)) {
        PySliceObject *slice = (PySliceObject *)op;
        Py_SETREF(slice->start, Py_None);
        Py_SETREF(slice->stop, Py_None);
        Py_SETREF(slice->step, Py_None);
    }
    else if (PyFrozenSet_CheckExact(op)) {
        _PySet_ClearInternal((PySetObject *)op);
    }
    return 0;
}

static void
destroy_key(void *key)
{
    _Py_ClearImmortal(key);
}
#endif

PyStatus
_PyCode_Init(PyInterpreterState *interp)
{ … }

void
_PyCode_Fini(PyInterpreterState *interp)
{ … }

#ifdef Py_GIL_DISABLED

// Thread-local bytecode (TLBC)
//
// Each thread specializes a thread-local copy of the bytecode, created on the
// first RESUME, in free-threaded builds. All copies of the bytecode for a code
// object are stored in the `co_tlbc` array. Threads reserve a globally unique
// index identifying its copy of the bytecode in all `co_tlbc` arrays at thread
// creation and release the index at thread destruction. The first entry in
// every `co_tlbc` array always points to the "main" copy of the bytecode that
// is stored at the end of the code object. This ensures that no bytecode is
// copied for programs that do not use threads.
//
// Thread-local bytecode can be disabled at runtime by providing either `-X
// tlbc=0` or `PYTHON_TLBC=0`. Disabling thread-local bytecode also disables
// specialization. All threads share the main copy of the bytecode when
// thread-local bytecode is disabled.
//
// Concurrent modifications to the bytecode made by the specializing
// interpreter and instrumentation use atomics, with specialization taking care
// not to overwrite an instruction that was instrumented concurrently.

int32_t
_Py_ReserveTLBCIndex(PyInterpreterState *interp)
{
    if (interp->config.tlbc_enabled) {
        return _PyIndexPool_AllocIndex(&interp->tlbc_indices);
    }
    // All threads share the main copy of the bytecode when TLBC is disabled
    return 0;
}

void
_Py_ClearTLBCIndex(_PyThreadStateImpl *tstate)
{
    PyInterpreterState *interp = ((PyThreadState *)tstate)->interp;
    if (interp->config.tlbc_enabled) {
        _PyIndexPool_FreeIndex(&interp->tlbc_indices, tstate->tlbc_index);
    }
}

static _PyCodeArray *
_PyCodeArray_New(Py_ssize_t size)
{
    _PyCodeArray *arr = PyMem_Calloc(
        1, offsetof(_PyCodeArray, entries) + sizeof(void *) * size);
    if (arr == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    arr->size = size;
    return arr;
}

static void
copy_code(_Py_CODEUNIT *dst, PyCodeObject *co)
{
    int code_len = (int) Py_SIZE(co);
    for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) {
        dst[i] = _Py_GetBaseCodeUnit(co, i);
    }
    _PyCode_Quicken(dst, code_len, co->co_consts, 1);
}

static Py_ssize_t
get_pow2_greater(Py_ssize_t initial, Py_ssize_t limit)
{
    // initial must be a power of two
    assert(!(initial & (initial - 1)));
    Py_ssize_t res = initial;
    while (res && res < limit) {
        res <<= 1;
    }
    return res;
}

static _Py_CODEUNIT *
create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx)
{
    _PyCodeArray *tlbc = co->co_tlbc;
    if (idx >= tlbc->size) {
        Py_ssize_t new_size = get_pow2_greater(tlbc->size, idx + 1);
        if (!new_size) {
            PyErr_NoMemory();
            return NULL;
        }
        _PyCodeArray *new_tlbc = _PyCodeArray_New(new_size);
        if (new_tlbc == NULL) {
            return NULL;
        }
        memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *));
        _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc);
        _PyMem_FreeDelayed(tlbc);
        tlbc = new_tlbc;
    }
    char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co));
    if (bc == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    copy_code((_Py_CODEUNIT *) bc, co);
    assert(tlbc->entries[idx] == NULL);
    tlbc->entries[idx] = bc;
    return (_Py_CODEUNIT *) bc;
}

static _Py_CODEUNIT *
get_tlbc_lock_held(PyCodeObject *co)
{
    _PyCodeArray *tlbc = co->co_tlbc;
    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET();
    int32_t idx = tstate->tlbc_index;
    if (idx < tlbc->size && tlbc->entries[idx] != NULL) {
        return (_Py_CODEUNIT *)tlbc->entries[idx];
    }
    return create_tlbc_lock_held(co, idx);
}

_Py_CODEUNIT *
_PyCode_GetTLBC(PyCodeObject *co)
{
    _Py_CODEUNIT *result;
    Py_BEGIN_CRITICAL_SECTION(co);
    result = get_tlbc_lock_held(co);
    Py_END_CRITICAL_SECTION();
    return result;
}

// My kingdom for a bitset
struct flag_set {
    uint8_t *flags;
    Py_ssize_t size;
};

static inline int
flag_is_set(struct flag_set *flags, Py_ssize_t idx)
{
    assert(idx >= 0);
    return (idx < flags->size) && flags->flags[idx];
}

// Set the flag for each tlbc index in use
static int
get_indices_in_use(PyInterpreterState *interp, struct flag_set *in_use)
{
    assert(interp->stoptheworld.world_stopped);
    assert(in_use->flags == NULL);
    int32_t max_index = 0;
    _Py_FOR_EACH_TSTATE_BEGIN(interp, p) {
        int32_t idx = ((_PyThreadStateImpl *) p)->tlbc_index;
        if (idx > max_index) {
            max_index = idx;
        }
    }
    _Py_FOR_EACH_TSTATE_END(interp);
    in_use->size = (size_t) max_index + 1;
    in_use->flags = PyMem_Calloc(in_use->size, sizeof(*in_use->flags));
    if (in_use->flags == NULL) {
        return -1;
    }
    _Py_FOR_EACH_TSTATE_BEGIN(interp, p) {
        in_use->flags[((_PyThreadStateImpl *) p)->tlbc_index] = 1;
    }
    _Py_FOR_EACH_TSTATE_END(interp);
    return 0;
}

struct get_code_args {
    _PyObjectStack code_objs;
    struct flag_set indices_in_use;
    int err;
};

static void
clear_get_code_args(struct get_code_args *args)
{
    if (args->indices_in_use.flags != NULL) {
        PyMem_Free(args->indices_in_use.flags);
        args->indices_in_use.flags = NULL;
    }
    _PyObjectStack_Clear(&args->code_objs);
}

static inline int
is_bytecode_unused(_PyCodeArray *tlbc, Py_ssize_t idx,
                   struct flag_set *indices_in_use)
{
    assert(idx > 0 && idx < tlbc->size);
    return tlbc->entries[idx] != NULL && !flag_is_set(indices_in_use, idx);
}

static int
get_code_with_unused_tlbc(PyObject *obj, struct get_code_args *args)
{
    if (!PyCode_Check(obj)) {
        return 1;
    }
    PyCodeObject *co = (PyCodeObject *) obj;
    _PyCodeArray *tlbc = co->co_tlbc;
    // The first index always points at the main copy of the bytecode embedded
    // in the code object.
    for (Py_ssize_t i = 1; i < tlbc->size; i++) {
        if (is_bytecode_unused(tlbc, i, &args->indices_in_use)) {
            if (_PyObjectStack_Push(&args->code_objs, obj) < 0) {
                args->err = -1;
                return 0;
            }
            return 1;
        }
    }
    return 1;
}

static void
free_unused_bytecode(PyCodeObject *co, struct flag_set *indices_in_use)
{
    _PyCodeArray *tlbc = co->co_tlbc;
    // The first index always points at the main copy of the bytecode embedded
    // in the code object.
    for (Py_ssize_t i = 1; i < tlbc->size; i++) {
        if (is_bytecode_unused(tlbc, i, indices_in_use)) {
            PyMem_Free(tlbc->entries[i]);
            tlbc->entries[i] = NULL;
        }
    }
}

int
_Py_ClearUnusedTLBC(PyInterpreterState *interp)
{
    struct get_code_args args = {
        .code_objs = {NULL},
        .indices_in_use = {NULL, 0},
        .err = 0,
    };
    _PyEval_StopTheWorld(interp);
    // Collect in-use tlbc indices
    if (get_indices_in_use(interp, &args.indices_in_use) < 0) {
        goto err;
    }
    // Collect code objects that have bytecode not in use by any thread
    _PyGC_VisitObjectsWorldStopped(
        interp, (gcvisitobjects_t)get_code_with_unused_tlbc, &args);
    if (args.err < 0) {
        goto err;
    }
    // Free unused bytecode. This must happen outside of gc_visit_heaps; it is
    // unsafe to allocate or free any mimalloc managed memory when it's
    // running.
    PyObject *obj;
    while ((obj = _PyObjectStack_Pop(&args.code_objs)) != NULL) {
        free_unused_bytecode((PyCodeObject*) obj, &args.indices_in_use);
    }
    _PyEval_StartTheWorld(interp);
    clear_get_code_args(&args);
    return 0;

err:
    _PyEval_StartTheWorld(interp);
    clear_get_code_args(&args);
    PyErr_NoMemory();
    return -1;
}

#endif