nouveau: pushbuf code, now with 50% less suck!

Far more efficient, if not a bit more complicated.  Hopefully not too
buggy still.

This commit will potentially expose some unrelated bugs, fixes for them
will follow "real soon now".
This commit is contained in:
Ben Skeggs 2007-12-24 19:21:17 +11:00
parent f9cfc32376
commit d732728590
5 changed files with 153 additions and 121 deletions

View file

@ -28,23 +28,23 @@
#include "nouveau_dma.h"
#include "nouveau_local.h"
static __inline__ uint32_t
static inline uint32_t
READ_GET(struct nouveau_channel_priv *nvchan)
{
return ((*nvchan->get - nvchan->dma.base) >> 2);
return *nvchan->get;
}
static __inline__ void
static inline void
WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val)
{
uint32_t put = ((val << 2) + nvchan->dma.base);
uint32_t put = ((val << 2) + nvchan->dma->base);
volatile int dum;
NOUVEAU_DMA_BARRIER;
dum = READ_GET(nvchan);
*nvchan->put = put;
nvchan->dma.put = val;
nvchan->dma->put = val;
#ifdef NOUVEAU_DMA_TRACE
NOUVEAU_MSG("WRITE_PUT %d/0x%08x\n", nvchan->drm.channel, put);
#endif
@ -52,16 +52,30 @@ WRITE_PUT(struct nouveau_channel_priv *nvchan, uint32_t val)
NOUVEAU_DMA_BARRIER;
}
static inline int
LOCAL_GET(struct nouveau_dma_priv *dma, uint32_t *val)
{
uint32_t get = *val;
if (get >= dma->base && get <= (dma->base + (dma->max << 2))) {
*val = (get - dma->base) >> 2;
return 1;
}
return 0;
}
void
nouveau_dma_channel_init(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
int i;
nvchan->dma.base = nvchan->drm.put_base;
nvchan->dma.cur = nvchan->dma.put = 0;
nvchan->dma.max = (nvchan->drm.cmdbuf_size >> 2) - 2;
nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
nvchan->dma = &nvchan->dma_master;
nvchan->dma->base = nvchan->drm.put_base;
nvchan->dma->cur = nvchan->dma->put = 0;
nvchan->dma->max = (nvchan->drm.cmdbuf_size >> 2) - 2;
nvchan->dma->free = nvchan->dma->max - nvchan->dma->cur;
RING_SPACE_CH(chan, RING_SKIPS);
for (i = 0; i < RING_SKIPS; i++)
@ -73,54 +87,51 @@ nouveau_dma_channel_init(struct nouveau_channel *chan)
return - EBUSY; \
} while(0)
#define IN_MASTER_RING(chan, ptr) ((ptr) <= (chan)->dma.max)
int
nouveau_dma_wait(struct nouveau_channel *chan, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_dma_priv *dma = nvchan->dma;
uint32_t get, t_start;
FIRE_RING_CH(chan);
t_start = NOUVEAU_TIME_MSEC();
while (nvchan->dma.free < size) {
while (dma->free < size) {
CHECK_TIMEOUT();
get = READ_GET(nvchan);
if (!IN_MASTER_RING(nvchan, get))
if (!LOCAL_GET(dma, &get))
continue;
if (nvchan->dma.put >= get) {
nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
if (dma->put >= get) {
dma->free = dma->max - dma->cur;
if (nvchan->dma.free < size) {
if (dma->free < size) {
#ifdef NOUVEAU_DMA_DEBUG
nvchan->dma.push_free = 1;
dma->push_free = 1;
#endif
OUT_RING_CH(chan,
0x20000000 | nvchan->dma.base);
OUT_RING_CH(chan, 0x20000000 | dma->base);
if (get <= RING_SKIPS) {
/*corner case - will be idle*/
if (nvchan->dma.put <= RING_SKIPS)
if (dma->put <= RING_SKIPS)
WRITE_PUT(nvchan,
RING_SKIPS + 1);
do {
CHECK_TIMEOUT();
get = READ_GET(nvchan);
if (!IN_MASTER_RING(nvchan,
get))
continue;
if (!LOCAL_GET(dma, &get))
get = 0;
} while (get <= RING_SKIPS);
}
WRITE_PUT(nvchan, RING_SKIPS);
nvchan->dma.cur = nvchan->dma.put = RING_SKIPS;
nvchan->dma.free = get - (RING_SKIPS + 1);
dma->cur = dma->put = RING_SKIPS;
dma->free = get - (RING_SKIPS + 1);
}
} else {
nvchan->dma.free = get - nvchan->dma.cur - 1;
dma->free = get - dma->cur - 1;
}
}
@ -135,7 +146,7 @@ nouveau_dma_parse_pushbuf(struct nouveau_channel *chan, int get, int put)
unsigned mthd_count = 0;
while (get != put) {
uint32_t gpuget = (get << 2) + nvchan->dma.base;
uint32_t gpuget = (get << 2) + nvchan->drm.put_base;
uint32_t data;
if (get < 0 || get >= nvchan->drm.cmdbuf_size) {
@ -188,21 +199,21 @@ void
nouveau_dma_kickoff(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_dma_priv *dma = nvchan->dma;
if (nvchan->dma.cur == nvchan->dma.put)
if (dma->cur == dma->put)
return;
#ifdef NOUVEAU_DMA_DEBUG
if (nvchan->dma.push_free) {
NOUVEAU_ERR("Packet incomplete: %d left\n",
nvchan->dma.push_free);
if (dma->push_free) {
NOUVEAU_ERR("Packet incomplete: %d left\n", dma->push_free);
return;
}
#endif
#ifdef NOUVEAU_DMA_DUMP_POSTRELOC_PUSHBUF
nouveau_dma_parse_pushbuf(chan, nvchan->dma.put, nvchan->dma.cur);
nouveau_dma_parse_pushbuf(chan, dma->put, dma->cur);
#endif
WRITE_PUT(nvchan, nvchan->dma.cur);
WRITE_PUT(nvchan, dma->cur);
}

View file

@ -42,34 +42,37 @@ static inline void
nouveau_dma_out(struct nouveau_channel *chan, uint32_t data)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_dma_priv *dma = nvchan->dma;
#ifdef NOUVEAU_DMA_DEBUG
if (nvchan->dma.push_free == 0) {
NOUVEAU_ERR("No space left in packet. Error at %s\n",faulty);
if (dma->push_free == 0) {
NOUVEAU_ERR("No space left in packet at %s\n", faulty);
return;
}
nvchan->dma.push_free--;
dma->push_free--;
#endif
#ifdef NOUVEAU_DMA_TRACE
{
uint32_t offset = (nvchan->dma.cur << 2) + nvchan->dma.base;
uint32_t offset = (dma->cur << 2) + dma->base;
NOUVEAU_MSG("\tOUT_RING %d/0x%08x -> 0x%08x\n",
nvchan->drm.channel, offset, data);
}
#endif
nvchan->pushbuf[nvchan->dma.cur++] = data;
nvchan->pushbuf[dma->cur + (dma->base - nvchan->drm.put_base)/4] = data;
dma->cur++;
}
static inline void
nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
(void)chan;
struct nouveau_dma_priv *dma = nvchan->dma;
(void)dma;
#ifdef NOUVEAU_DMA_DEBUG
if (nvchan->dma.push_free < size) {
if (dma->push_free < size) {
NOUVEAU_ERR("Packet too small. Free=%d, Need=%d\n",
nvchan->dma.push_free, size);
dma->push_free, size);
return;
}
#endif
@ -79,11 +82,11 @@ nouveau_dma_outp(struct nouveau_channel *chan, uint32_t *ptr, int size)
ptr++;
}
#else
memcpy(&nvchan->pushbuf[nvchan->dma.cur], ptr, size << 2);
memcpy(&nvchan->pushbuf[dma->cur], ptr, size << 2);
#ifdef NOUVEAU_DMA_DEBUG
nvchan->dma.push_free -= size;
dma->push_free -= size;
#endif
nvchan->dma.cur += size;
dma->cur += size;
#endif
}
@ -91,14 +94,15 @@ static inline void
nouveau_dma_space(struct nouveau_channel *chan, int size)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_dma_priv *dma = nvchan->dma;
if (nvchan->dma.free < size) {
if (dma->free < size) {
if (nouveau_dma_wait(chan, size) && chan->hang_notify)
chan->hang_notify(chan);
}
nvchan->dma.free -= size;
dma->free -= size;
#ifdef NOUVEAU_DMA_DEBUG
nvchan->dma.push_free = size;
dma->push_free = size;
#endif
}
@ -107,7 +111,8 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj,
int method, int size, const char* file, int line)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
(void)nvchan;
struct nouveau_dma_priv *dma = nvchan->dma;
(void)dma;
#ifdef NOUVEAU_DMA_TRACE
NOUVEAU_MSG("BEGIN_RING %d/%08x/%d/0x%04x/%d\n", nvchan->drm.channel,
@ -115,9 +120,9 @@ nouveau_dma_begin(struct nouveau_channel *chan, struct nouveau_grobj *grobj,
#endif
#ifdef NOUVEAU_DMA_DEBUG
if (nvchan->dma.push_free) {
NOUVEAU_ERR("Previous packet incomplete: %d left. Error at %s\n",
nvchan->dma.push_free,faulty);
if (dma->push_free) {
NOUVEAU_ERR("Previous packet incomplete: %d left at %s\n",
dma->push_free, faulty);
return;
}
sprintf(faulty,"%s:%d",file,line);

View file

@ -128,10 +128,10 @@ struct nouveau_pushbuf_bo {
struct nouveau_pushbuf_priv {
struct nouveau_pushbuf base;
struct nouveau_pushbuf *next;
struct nouveau_resource *res;
struct nouveau_fence *fence;
unsigned nop_jump;
unsigned start;
unsigned size;
uint64_t buffers;
int nr_buffers;
@ -149,13 +149,23 @@ extern int
nouveau_pushbuf_init(struct nouveau_channel *);
extern int
nouveau_pushbuf_flush(struct nouveau_channel *);
nouveau_pushbuf_flush(struct nouveau_channel *, unsigned min);
extern int
nouveau_pushbuf_emit_reloc(struct nouveau_channel *, void *ptr,
struct nouveau_bo *, uint32_t data, uint32_t flags,
uint32_t vor, uint32_t tor);
struct nouveau_dma_priv {
uint32_t base;
uint32_t max;
uint32_t cur;
uint32_t put;
uint32_t free;
int push_free;
} dma;
struct nouveau_channel_priv {
struct nouveau_channel base;
@ -169,21 +179,15 @@ struct nouveau_channel_priv {
volatile uint32_t *get;
volatile uint32_t *ref_cnt;
struct {
uint32_t base, max;
uint32_t cur, put;
uint32_t free;
int push_free;
} dma;
struct nouveau_dma_priv dma_master;
struct nouveau_dma_priv dma_bufmgr;
struct nouveau_dma_priv *dma;
struct nouveau_fence *fence_head;
struct nouveau_fence *fence_tail;
uint32_t fence_sequence;
struct nouveau_resource *pb_heap;
struct nouveau_pushbuf *pb_head;
struct nouveau_pushbuf *pb_tail;
struct nouveau_pushbuf_priv pb;
unsigned user_charge;
};

View file

@ -27,28 +27,34 @@
#include "nouveau_drmif.h"
#include "nouveau_dma.h"
#define PB_RSVD_DWORDS 2
#define PB_BUFMGR_DWORDS (4096 / 2)
#define PB_MIN_USER_DWORDS 2048
static int
nouveau_pushbuf_space(struct nouveau_channel *chan)
nouveau_pushbuf_space(struct nouveau_channel *chan, unsigned min)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_pushbuf_priv *nvpb;
struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
nvpb = calloc(1, sizeof(struct nouveau_pushbuf_priv));
if (!nvpb)
return -ENOMEM;
assert((min + 1) <= nvchan->dma->max);
while (nouveau_resource_alloc(nvchan->pb_heap, 0x2100, NULL,
&nvpb->res)) {
nouveau_fence_flush(chan);
}
/* Wait for enough space in push buffer */
min = min < PB_MIN_USER_DWORDS ? PB_MIN_USER_DWORDS : min;
min += 1; /* a bit extra for the NOP */
if (nvchan->dma->free < min)
WAIT_RING_CH(chan, min);
/* Insert NOP, may turn into a jump later */
RING_SPACE_CH(chan, 1);
nvpb->nop_jump = nvchan->dma->cur;
OUT_RING_CH(chan, 0);
/* Any remaining space is available to the user */
nvpb->start = nvchan->dma->cur;
nvpb->size = nvchan->dma->free;
nvpb->base.channel = chan;
nvpb->base.remaining = (nvpb->res->size / 4) - PB_RSVD_DWORDS;
nvpb->base.cur = &nvchan->pushbuf[nvpb->res->start/4];
nvchan->pb_tail = &nvpb->base;
nvchan->base.pushbuf = nvchan->pb_tail;
nvpb->base.remaining = nvpb->size;
nvpb->base.cur = &nvchan->pushbuf[nvpb->start];
return 0;
}
@ -57,53 +63,65 @@ int
nouveau_pushbuf_init(struct nouveau_channel *chan)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_dma_priv *m = &nvchan->dma_master;
struct nouveau_dma_priv *b = &nvchan->dma_bufmgr;
int i;
if (!nvchan)
return -EINVAL;
/* Everything except first 4KiB of the push buffer is managed by us */
if (nouveau_resource_init(&nvchan->pb_heap, 4096,
nvchan->drm.cmdbuf_size - 4096))
return -EINVAL;
/* Reassign last bit of push buffer for a "separate" bufmgr
* ring buffer
*/
m->max -= PB_BUFMGR_DWORDS;
m->free -= PB_BUFMGR_DWORDS;
/* Shrink master ring to 4KiB */
assert(nvchan->dma.cur <= (4096/4));
nvchan->dma.max = (4096 / 4) - 2;
nvchan->dma.free = nvchan->dma.max - nvchan->dma.cur;
b->base = m->base + ((m->max + 2) << 2);
b->max = PB_BUFMGR_DWORDS - 2;
b->cur = b->put = 0;
b->free = b->max - b->cur;
assert(!nouveau_pushbuf_space(chan));
/* Some NOPs just to be safe
*XXX: RING_SKIPS
*/
nvchan->dma = b;
RING_SPACE_CH(chan, 8);
for (i = 0; i < 8; i++)
OUT_RING_CH(chan, 0);
nvchan->dma = m;
nouveau_pushbuf_space(chan, 0);
chan->pushbuf = &nvchan->pb.base;
return 0;
}
static void
nouveau_pushbuf_fence_signalled(void *priv)
{
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(priv);
nouveau_fence_del(&nvpb->fence);
nouveau_resource_free(&nvpb->res);
free(nvpb);
}
/* This would be our TTM "superioctl" */
int
nouveau_pushbuf_flush(struct nouveau_channel *chan)
nouveau_pushbuf_flush(struct nouveau_channel *chan, unsigned min)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
struct nouveau_pushbuf_priv *nvpb = &nvchan->pb;
struct nouveau_pushbuf_bo *pbbo;
struct nouveau_fence *fence = NULL;
int ret;
if (nvpb->base.remaining == (nvpb->res->size / 4) - PB_RSVD_DWORDS)
if (nvpb->base.remaining == nvpb->size)
return 0;
nvchan->pb_tail = NULL;
nvpb->size -= nvpb->base.remaining;
nvchan->dma->cur += nvpb->size;
nvchan->dma->free -= nvpb->size;
assert(nvchan->dma->cur <= nvchan->dma->max);
ret = nouveau_fence_new(chan, &fence);
if (ret)
return ret;
nvchan->dma = &nvchan->dma_bufmgr;
nvchan->pushbuf[nvpb->nop_jump] = 0x20000000 |
(nvchan->dma->base + (nvchan->dma->cur << 2));
/* Validate buffers + apply relocations */
nvchan->user_charge = 0;
while ((pbbo = ptr_to_pbbo(nvpb->buffers))) {
@ -142,25 +160,19 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan)
}
nvpb->nr_buffers = 0;
/* Emit JMP to indirect pushbuf */
/* Switch back to user's ring */
RING_SPACE_CH(chan, 1);
OUT_RING_CH(chan, 0x20000000 | nvpb->res->start);
OUT_RING_CH(chan, 0x20000000 | ((nvpb->start << 2) +
nvchan->dma_master.base));
nvchan->dma = &nvchan->dma_master;
/* Add JMP back to master pushbuf from indirect pushbuf */
(*nvpb->base.cur++) =
0x20000000 | ((nvchan->dma.cur << 2) + nvchan->dma.base);
/* Fence */
nvpb->fence = fence;
nouveau_fence_signal_cb(nvpb->fence, nouveau_pushbuf_fence_signalled,
nvpb);
nouveau_fence_emit(nvpb->fence);
/* Kickoff */
/* Fence + kickoff */
nouveau_fence_emit(fence);
FIRE_RING_CH(chan);
nouveau_fence_del(&fence);
/* Allocate space for next push buffer */
assert(!nouveau_pushbuf_space(chan));
assert(!nouveau_pushbuf_space(chan, min));
return 0;
}
@ -168,8 +180,7 @@ nouveau_pushbuf_flush(struct nouveau_channel *chan)
static struct nouveau_pushbuf_bo *
nouveau_pushbuf_emit_buffer(struct nouveau_channel *chan, struct nouveau_bo *bo)
{
struct nouveau_channel_priv *nvchan = nouveau_channel(chan);
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(nvchan->pb_tail);
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(chan->pushbuf);
struct nouveau_pushbuf_bo *pbbo = ptr_to_pbbo(nvpb->buffers);
while (pbbo) {

View file

@ -40,8 +40,9 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size)
struct nouveau_channel *chan = grobj->channel;
uint32_t *pushbuf;
if (chan->pushbuf->remaining < (size + 1))
nouveau_pushbuf_flush(chan);
if (chan->pushbuf->remaining < (size + 1)) {
nouveau_pushbuf_flush(chan, size + 1);
}
pushbuf = chan->pushbuf->cur;
chan->pushbuf->cur += (size + 1);
@ -54,7 +55,7 @@ nouveau_pipe_dma_beginp(struct nouveau_grobj *grobj, int mthd, int size)
void
nouveau_pipe_dma_kickoff(struct nouveau_channel *chan)
{
nouveau_pushbuf_flush(chan);
nouveau_pushbuf_flush(chan, 0);
}
static int