#include "Python.h"
#include "pycore_object.h"
#include "pycore_brc.h"
#include "pycore_ceval.h"
#include "pycore_llist.h"
#include "pycore_pystate.h"
#ifdef Py_GIL_DISABLED
static struct _brc_bucket *
get_bucket(PyInterpreterState *interp, uintptr_t tid)
{
return &interp->brc.table[tid % _Py_BRC_NUM_BUCKETS];
}
static _PyThreadStateImpl *
find_thread_state(struct _brc_bucket *bucket, uintptr_t thread_id)
{
struct llist_node *node;
llist_for_each(node, &bucket->root) {
_PyThreadStateImpl *ts = llist_data(node, _PyThreadStateImpl,
brc.bucket_node);
if (ts->brc.tid == thread_id) {
return ts;
}
}
return NULL;
}
void
_Py_brc_queue_object(PyObject *ob)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
uintptr_t ob_tid = _Py_atomic_load_uintptr(&ob->ob_tid);
if (ob_tid == 0) {
Py_DECREF(ob);
return;
}
struct _brc_bucket *bucket = get_bucket(interp, ob_tid);
PyMutex_Lock(&bucket->mutex);
_PyThreadStateImpl *tstate = find_thread_state(bucket, ob_tid);
if (tstate == NULL) {
Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1);
PyMutex_Unlock(&bucket->mutex);
if (refcount == 0) {
_Py_Dealloc(ob);
}
return;
}
if (_PyObjectStack_Push(&tstate->brc.objects_to_merge, ob) < 0) {
PyMutex_Unlock(&bucket->mutex);
_PyEval_StopTheWorld(interp);
Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1);
_PyEval_StartTheWorld(interp);
if (refcount == 0) {
_Py_Dealloc(ob);
}
return;
}
_Py_set_eval_breaker_bit(&tstate->base, _PY_EVAL_EXPLICIT_MERGE_BIT);
PyMutex_Unlock(&bucket->mutex);
}
static void
merge_queued_objects(_PyObjectStack *to_merge)
{
PyObject *ob;
while ((ob = _PyObjectStack_Pop(to_merge)) != NULL) {
Py_ssize_t refcount = _Py_ExplicitMergeRefcount(ob, -1);
if (refcount == 0) {
_Py_Dealloc(ob);
}
}
}
void
_Py_brc_merge_refcounts(PyThreadState *tstate)
{
struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc;
struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid);
assert(brc->tid == _Py_ThreadId());
PyMutex_Lock(&bucket->mutex);
_PyObjectStack_Merge(&brc->local_objects_to_merge, &brc->objects_to_merge);
PyMutex_Unlock(&bucket->mutex);
merge_queued_objects(&brc->local_objects_to_merge);
}
void
_Py_brc_init_state(PyInterpreterState *interp)
{
struct _brc_state *brc = &interp->brc;
for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) {
llist_init(&brc->table[i].root);
}
}
void
_Py_brc_init_thread(PyThreadState *tstate)
{
struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc;
uintptr_t tid = _Py_ThreadId();
struct _brc_bucket *bucket = get_bucket(tstate->interp, tid);
PyMutex_Lock(&bucket->mutex);
brc->tid = tid;
llist_insert_tail(&bucket->root, &brc->bucket_node);
PyMutex_Unlock(&bucket->mutex);
}
void
_Py_brc_remove_thread(PyThreadState *tstate)
{
struct _brc_thread_state *brc = &((_PyThreadStateImpl *)tstate)->brc;
if (brc->tid == 0) {
assert(tstate->_status.bound == 0);
return;
}
struct _brc_bucket *bucket = get_bucket(tstate->interp, brc->tid);
bool empty = false;
while (!empty) {
merge_queued_objects(&brc->local_objects_to_merge);
PyMutex_Lock(&bucket->mutex);
empty = (brc->objects_to_merge.head == NULL);
if (empty) {
llist_remove(&brc->bucket_node);
}
else {
_PyObjectStack_Merge(&brc->local_objects_to_merge,
&brc->objects_to_merge);
}
PyMutex_Unlock(&bucket->mutex);
}
assert(brc->local_objects_to_merge.head == NULL);
assert(brc->objects_to_merge.head == NULL);
}
void
_Py_brc_after_fork(PyInterpreterState *interp)
{
for (Py_ssize_t i = 0; i < _Py_BRC_NUM_BUCKETS; i++) {
_PyMutex_at_fork_reinit(&interp->brc.table[i].mutex);
}
}
#endif