/*
* Copyright 2012 Red Hat Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* Authors: Ben Skeggs
*/
#include "chan.h"
#include "chid.h"
#include "cgrp.h"
#include "runl.h"
#include "priv.h"
#include <core/ramht.h>
#include <subdev/mmu.h>
#include <engine/dma.h>
#include <nvif/if0020.h>
const struct nvkm_event_func
nvkm_chan_event = {
};
void
nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx *cctx)
{
struct nvkm_cgrp *cgrp = chan->cgrp;
struct nvkm_runl *runl = cgrp->runl;
struct nvkm_engine *engine = engn->engine;
if (!engn->func->bind)
return;
CHAN_TRACE(chan, "%sbind cctx %d[%s]", cctx ? "" : "un", engn->id, engine->subdev.name);
/* Prevent any channel in channel group from being rescheduled, kick them
* off host and any engine(s) they're loaded on.
*/
if (cgrp->hw)
nvkm_runl_block(runl);
else
nvkm_chan_block(chan);
nvkm_chan_preempt(chan, true);
/* Update context pointer. */
engn->func->bind(engn, cctx, chan);
/* Resume normal operation. */
if (cgrp->hw)
nvkm_runl_allow(runl);
else
nvkm_chan_allow(chan);
}
void
nvkm_chan_cctx_put(struct nvkm_chan *chan, struct nvkm_cctx **pcctx)
{
struct nvkm_cctx *cctx = *pcctx;
if (cctx) {
struct nvkm_engn *engn = cctx->vctx->ectx->engn;
if (refcount_dec_and_mutex_lock(&cctx->refs, &chan->cgrp->mutex)) {
CHAN_TRACE(chan, "dtor cctx %d[%s]", engn->id, engn->engine->subdev.name);
nvkm_cgrp_vctx_put(chan->cgrp, &cctx->vctx);
list_del(&cctx->head);
kfree(cctx);
mutex_unlock(&chan->cgrp->mutex);
}
*pcctx = NULL;
}
}
int
nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx **pcctx,
struct nvkm_client *client)
{
struct nvkm_cgrp *cgrp = chan->cgrp;
struct nvkm_vctx *vctx;
struct nvkm_cctx *cctx;
int ret;
/* Look for an existing channel context for this engine+VEID. */
mutex_lock(&cgrp->mutex);
cctx = nvkm_list_find(cctx, &chan->cctxs, head,
cctx->vctx->ectx->engn == engn && cctx->vctx->vmm == chan->vmm);
if (cctx) {
refcount_inc(&cctx->refs);
*pcctx = cctx;
mutex_unlock(&chan->cgrp->mutex);
return 0;
}
/* Nope - create a fresh one. But, sub-context first. */
ret = nvkm_cgrp_vctx_get(cgrp, engn, chan, &vctx, client);
if (ret) {
CHAN_ERROR(chan, "vctx %d[%s]: %d", engn->id, engn->engine->subdev.name, ret);
goto done;
}
/* Now, create the channel context - to track engine binding. */
CHAN_TRACE(chan, "ctor cctx %d[%s]", engn->id, engn->engine->subdev.name);
if (!(cctx = *pcctx = kzalloc(sizeof(*cctx), GFP_KERNEL))) {
nvkm_cgrp_vctx_put(cgrp, &vctx);
ret = -ENOMEM;
goto done;
}
cctx->vctx = vctx;
refcount_set(&cctx->refs, 1);
refcount_set(&cctx->uses, 0);
list_add_tail(&cctx->head, &chan->cctxs);
done:
mutex_unlock(&cgrp->mutex);
return ret;
}
int
nvkm_chan_preempt_locked(struct nvkm_chan *chan, bool wait)
{
struct nvkm_runl *runl = chan->cgrp->runl;
CHAN_TRACE(chan, "preempt");
chan->func->preempt(chan);
if (!wait)
return 0;
return nvkm_runl_preempt_wait(runl);
}
int
nvkm_chan_preempt(struct nvkm_chan *chan, bool wait)
{
int ret;
if (!chan->func->preempt)
return 0;
mutex_lock(&chan->cgrp->runl->mutex);
ret = nvkm_chan_preempt_locked(chan, wait);
mutex_unlock(&chan->cgrp->runl->mutex);
return ret;
}
void
nvkm_chan_remove_locked(struct nvkm_chan *chan)
{
struct nvkm_cgrp *cgrp = chan->cgrp;
struct nvkm_runl *runl = cgrp->runl;
if (list_empty(&chan->head))
return;
CHAN_TRACE(chan, "remove");
if (!--cgrp->chan_nr) {
runl->cgrp_nr--;
list_del(&cgrp->head);
}
runl->chan_nr--;
list_del_init(&chan->head);
atomic_set(&runl->changed, 1);
}
void
nvkm_chan_remove(struct nvkm_chan *chan, bool preempt)
{
struct nvkm_runl *runl = chan->cgrp->runl;
mutex_lock(&runl->mutex);
if (preempt && chan->func->preempt)
nvkm_chan_preempt_locked(chan, true);
nvkm_chan_remove_locked(chan);
nvkm_runl_update_locked(runl, true);
mutex_unlock(&runl->mutex);
}
void
nvkm_chan_insert(struct nvkm_chan *chan)
{
struct nvkm_cgrp *cgrp = chan->cgrp;
struct nvkm_runl *runl = cgrp->runl;
mutex_lock(&runl->mutex);
if (WARN_ON(!list_empty(&chan->head))) {
mutex_unlock(&runl->mutex);
return;
}
CHAN_TRACE(chan, "insert");
list_add_tail(&chan->head, &cgrp->chans);
runl->chan_nr++;
if (!cgrp->chan_nr++) {
list_add_tail(&cgrp->head, &cgrp->runl->cgrps);
runl->cgrp_nr++;
}
atomic_set(&runl->changed, 1);
nvkm_runl_update_locked(runl, true);
mutex_unlock(&runl->mutex);
}
static void
nvkm_chan_block_locked(struct nvkm_chan *chan)
{
CHAN_TRACE(chan, "block %d", atomic_read(&chan->blocked));
if (atomic_inc_return(&chan->blocked) == 1)
chan->func->stop(chan);
}
void
nvkm_chan_error(struct nvkm_chan *chan, bool preempt)
{
unsigned long flags;
spin_lock_irqsave(&chan->lock, flags);
if (atomic_inc_return(&chan->errored) == 1) {
CHAN_ERROR(chan, "errored - disabling channel");
nvkm_chan_block_locked(chan);
if (preempt)
chan->func->preempt(chan);
nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED);
}
spin_unlock_irqrestore(&chan->lock, flags);
}
void
nvkm_chan_block(struct nvkm_chan *chan)
{
spin_lock_irq(&chan->lock);
nvkm_chan_block_locked(chan);
spin_unlock_irq(&chan->lock);
}
void
nvkm_chan_allow(struct nvkm_chan *chan)
{
spin_lock_irq(&chan->lock);
CHAN_TRACE(chan, "allow %d", atomic_read(&chan->blocked));
if (atomic_dec_and_test(&chan->blocked))
chan->func->start(chan);
spin_unlock_irq(&chan->lock);
}
void
nvkm_chan_del(struct nvkm_chan **pchan)
{
struct nvkm_chan *chan = *pchan;
if (!chan)
return;
if (chan->func->ramfc->clear)
chan->func->ramfc->clear(chan);
nvkm_ramht_del(&chan->ramht);
nvkm_gpuobj_del(&chan->pgd);
nvkm_gpuobj_del(&chan->eng);
nvkm_gpuobj_del(&chan->cache);
nvkm_gpuobj_del(&chan->ramfc);
if (chan->cgrp) {
if (!chan->func->id_put)
nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock);
else
chan->func->id_put(chan);
nvkm_cgrp_unref(&chan->cgrp);
}
nvkm_memory_unref(&chan->userd.mem);
if (chan->vmm) {
nvkm_vmm_part(chan->vmm, chan->inst->memory);
nvkm_vmm_unref(&chan->vmm);
}
nvkm_gpuobj_del(&chan->push);
nvkm_gpuobj_del(&chan->inst);
kfree(chan);
}
void
nvkm_chan_put(struct nvkm_chan **pchan, unsigned long irqflags)
{
struct nvkm_chan *chan = *pchan;
if (!chan)
return;
*pchan = NULL;
spin_unlock_irqrestore(&chan->cgrp->lock, irqflags);
}
struct nvkm_chan *
nvkm_chan_get_inst(struct nvkm_engine *engine, u64 inst, unsigned long *pirqflags)
{
struct nvkm_fifo *fifo = engine->subdev.device->fifo;
struct nvkm_runl *runl;
struct nvkm_engn *engn;
struct nvkm_chan *chan;
nvkm_runl_foreach(runl, fifo) {
nvkm_runl_foreach_engn(engn, runl) {
if (engine == &fifo->engine || engn->engine == engine) {
chan = nvkm_runl_chan_get_inst(runl, inst, pirqflags);
if (chan || engn->engine == engine)
return chan;
}
}
}
return NULL;
}
struct nvkm_chan *
nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags)
{
struct nvkm_fifo *fifo = engine->subdev.device->fifo;
struct nvkm_runl *runl;
struct nvkm_engn *engn;
nvkm_runl_foreach(runl, fifo) {
nvkm_runl_foreach_engn(engn, runl) {
if (fifo->chid || engn->engine == engine)
return nvkm_runl_chan_get_chid(runl, id, pirqflags);
}
}
return NULL;
}
int
nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int runq,
struct nvkm_cgrp *cgrp, const char *name, bool priv, u32 devm, struct nvkm_vmm *vmm,
struct nvkm_dmaobj *dmaobj, u64 offset, u64 length,
struct nvkm_memory *userd, u64 ouserd, struct nvkm_chan **pchan)
{
struct nvkm_fifo *fifo = runl->fifo;
struct nvkm_device *device = fifo->engine.subdev.device;
struct nvkm_chan *chan;
int ret;
/* Validate arguments against class requirements. */
if ((runq && runq >= runl->func->runqs) ||
(!func->inst->vmm != !vmm) ||
((func->userd->bar < 0) == !userd) ||
(!func->ramfc->ctxdma != !dmaobj) ||
((func->ramfc->devm < devm) && devm != BIT(0)) ||
(!func->ramfc->priv && priv)) {
RUNL_DEBUG(runl, "args runq:%d:%d vmm:%d:%p userd:%d:%p "
"push:%d:%p devm:%08x:%08x priv:%d:%d",
runl->func->runqs, runq, func->inst->vmm, vmm,
func->userd->bar < 0, userd, func->ramfc->ctxdma, dmaobj,
func->ramfc->devm, devm, func->ramfc->priv, priv);
return -EINVAL;
}
if (!(chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL)))
return -ENOMEM;
chan->func = func;
strscpy(chan->name, name, sizeof(chan->name));
chan->runq = runq;
chan->id = -1;
spin_lock_init(&chan->lock);
atomic_set(&chan->blocked, 1);
atomic_set(&chan->errored, 0);
INIT_LIST_HEAD(&chan->cctxs);
INIT_LIST_HEAD(&chan->head);
/* Join channel group.
*
* GK110 and newer support channel groups (aka TSGs), where individual channels
* share a timeslice, and, engine context(s).
*
* As such, engine contexts are tracked in nvkm_cgrp and we need them even when
* channels aren't in an API channel group, and on HW that doesn't support TSGs.
*/
if (!cgrp) {
ret = nvkm_cgrp_new(runl, chan->name, vmm, fifo->func->cgrp.force, &chan->cgrp);
if (ret) {
RUNL_DEBUG(runl, "cgrp %d", ret);
return ret;
}
cgrp = chan->cgrp;
} else {
if (cgrp->runl != runl || cgrp->vmm != vmm) {
RUNL_DEBUG(runl, "cgrp %d %d", cgrp->runl != runl, cgrp->vmm != vmm);
return -EINVAL;
}
chan->cgrp = nvkm_cgrp_ref(cgrp);
}
/* Allocate instance block. */
ret = nvkm_gpuobj_new(device, func->inst->size, 0x1000, func->inst->zero, NULL,
&chan->inst);
if (ret) {
RUNL_DEBUG(runl, "inst %d", ret);
return ret;
}
/* Initialise virtual address-space. */
if (func->inst->vmm) {
if (WARN_ON(vmm->mmu != device->mmu))
return -EINVAL;
ret = nvkm_vmm_join(vmm, chan->inst->memory);
if (ret) {
RUNL_DEBUG(runl, "vmm %d", ret);
return ret;
}
chan->vmm = nvkm_vmm_ref(vmm);
}
/* Allocate HW ctxdma for push buffer. */
if (func->ramfc->ctxdma) {
ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, &chan->push);
if (ret) {
RUNL_DEBUG(runl, "bind %d", ret);
return ret;
}
}
/* Allocate channel ID. */
if (!chan->func->id_get) {
chan->id = nvkm_chid_get(runl->chid, chan);
if (chan->id >= 0) {
if (func->userd->bar < 0) {
if (ouserd + chan->func->userd->size >=
nvkm_memory_size(userd)) {
RUNL_DEBUG(runl, "ouserd %llx", ouserd);
return -EINVAL;
}
ret = nvkm_memory_kmap(userd, &chan->userd.mem);
if (ret) {
RUNL_DEBUG(runl, "userd %d", ret);
return ret;
}
chan->userd.base = ouserd;
} else {
chan->userd.mem = nvkm_memory_ref(fifo->userd.mem);
chan->userd.base = chan->id * chan->func->userd->size;
}
}
} else {
chan->id = chan->func->id_get(chan, userd, ouserd);
}
if (chan->id < 0) {
RUNL_ERROR(runl, "!chids");
return -ENOSPC;
}
if (cgrp->id < 0)
cgrp->id = chan->id;
/* Initialise USERD. */
if (chan->func->userd->clear)
chan->func->userd->clear(chan);
/* Initialise RAMFC. */
ret = chan->func->ramfc->write(chan, offset, length, devm, priv);
if (ret) {
RUNL_DEBUG(runl, "ramfc %d", ret);
return ret;
}
return 0;
}
Codebase Browser
linux