From f2af3a9cae22f5e644c55a3c75513fbf726d8004 Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Wed, 30 Jun 2021 10:26:35 +0200 Subject: [PATCH] nouveau: make fencing race free Signed-off-by: Karol Herbst Acked-by: M Henning Part-of: --- src/gallium/drivers/nouveau/nouveau_fence.c | 145 ++++++++++++++---- src/gallium/drivers/nouveau/nouveau_fence.h | 37 +++-- src/gallium/drivers/nouveau/nouveau_screen.c | 4 +- src/gallium/drivers/nouveau/nouveau_winsys.h | 33 +++- .../drivers/nouveau/nv30/nv30_context.c | 8 +- .../drivers/nouveau/nv50/nv50_context.c | 4 +- src/gallium/drivers/nouveau/nv50/nv50_vbo.c | 2 +- .../drivers/nouveau/nvc0/nvc0_context.c | 4 +- src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c | 2 +- 9 files changed, 179 insertions(+), 60 deletions(-) diff --git a/src/gallium/drivers/nouveau/nouveau_fence.c b/src/gallium/drivers/nouveau/nouveau_fence.c index 05808ebd68e..4ac3fa4f7b6 100644 --- a/src/gallium/drivers/nouveau/nouveau_fence.c +++ b/src/gallium/drivers/nouveau/nouveau_fence.c @@ -30,6 +30,9 @@ #include #endif +static bool +_nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug); + bool nouveau_fence_new(struct nouveau_context *nv, struct nouveau_fence **fence) { @@ -48,6 +51,8 @@ nouveau_fence_new(struct nouveau_context *nv, struct nouveau_fence **fence) static void nouveau_fence_trigger_work(struct nouveau_fence *fence) { + simple_mtx_assert_locked(&fence->screen->fence.lock); + struct nouveau_fence_work *work, *tmp; LIST_FOR_EACH_ENTRY_SAFE(work, tmp, &fence->work, list) { @@ -57,12 +62,14 @@ nouveau_fence_trigger_work(struct nouveau_fence *fence) } } -void -nouveau_fence_emit(struct nouveau_fence *fence) +static void +_nouveau_fence_emit(struct nouveau_fence *fence) { struct nouveau_screen *screen = fence->screen; struct nouveau_fence_list *fence_list = &screen->fence; + simple_mtx_assert_locked(&fence_list->lock); + assert(fence->state != NOUVEAU_FENCE_STATE_EMITTING); if (fence->state >= NOUVEAU_FENCE_STATE_EMITTED) return; @@ -70,7 +77,7 @@ nouveau_fence_emit(struct nouveau_fence *fence) /* set this now, so that if fence.emit triggers a flush we don't recurse */ fence->state = NOUVEAU_FENCE_STATE_EMITTING; - ++fence->ref; + p_atomic_inc(&fence->ref); if (fence_list->tail) fence_list->tail->next = fence; @@ -85,12 +92,14 @@ nouveau_fence_emit(struct nouveau_fence *fence) fence->state = NOUVEAU_FENCE_STATE_EMITTED; } -void +static void nouveau_fence_del(struct nouveau_fence *fence) { struct nouveau_fence *it; struct nouveau_fence_list *fence_list = &fence->screen->fence; + simple_mtx_assert_locked(&fence_list->lock); + if (fence->state == NOUVEAU_FENCE_STATE_EMITTED || fence->state == NOUVEAU_FENCE_STATE_FLUSHED) { if (fence == fence_list->head) { @@ -117,26 +126,31 @@ void nouveau_fence_cleanup(struct nouveau_context *nv) { if (nv->fence) { + struct nouveau_fence_list *fence_list = &nv->screen->fence; struct nouveau_fence *current = NULL; /* nouveau_fence_wait will create a new current fence, so wait on the * _current_ one, and remove both. */ - nouveau_fence_ref(nv->fence, ¤t); - nouveau_fence_wait(current, NULL); - nouveau_fence_ref(NULL, ¤t); - nouveau_fence_ref(NULL, &nv->fence); + simple_mtx_lock(&fence_list->lock); + _nouveau_fence_ref(nv->fence, ¤t); + _nouveau_fence_wait(current, NULL); + _nouveau_fence_ref(NULL, ¤t); + _nouveau_fence_ref(NULL, &nv->fence); + simple_mtx_unlock(&fence_list->lock); } } void -nouveau_fence_update(struct nouveau_screen *screen, bool flushed) +_nouveau_fence_update(struct nouveau_screen *screen, bool flushed) { struct nouveau_fence *fence; struct nouveau_fence *next = NULL; struct nouveau_fence_list *fence_list = &screen->fence; u32 sequence = fence_list->update(&screen->base); + simple_mtx_assert_locked(&fence_list->lock); + /* If running under drm-shim, let all fences be signalled so things run to * completion (avoids a hang at the end of shader-db). */ @@ -154,7 +168,7 @@ nouveau_fence_update(struct nouveau_screen *screen, bool flushed) fence->state = NOUVEAU_FENCE_STATE_SIGNALLED; nouveau_fence_trigger_work(fence); - nouveau_fence_ref(NULL, &fence); + _nouveau_fence_ref(NULL, &fence); if (sequence == fence_list->sequence_ack) break; @@ -172,16 +186,18 @@ nouveau_fence_update(struct nouveau_screen *screen, bool flushed) #define NOUVEAU_FENCE_MAX_SPINS (1 << 31) -bool -nouveau_fence_signalled(struct nouveau_fence *fence) +static bool +_nouveau_fence_signalled(struct nouveau_fence *fence) { struct nouveau_screen *screen = fence->screen; + simple_mtx_assert_locked(&screen->fence.lock); + if (fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) return true; if (fence->state >= NOUVEAU_FENCE_STATE_EMITTED) - nouveau_fence_update(screen, false); + _nouveau_fence_update(screen, false); return fence->state == NOUVEAU_FENCE_STATE_SIGNALLED; } @@ -191,36 +207,43 @@ nouveau_fence_kick(struct nouveau_fence *fence) { struct nouveau_context *context = fence->context; struct nouveau_screen *screen = fence->screen; + struct nouveau_fence_list *fence_list = &screen->fence; bool current = !fence->sequence; + simple_mtx_assert_locked(&fence_list->lock); + /* wtf, someone is waiting on a fence in flush_notify handler? */ assert(fence->state != NOUVEAU_FENCE_STATE_EMITTING); if (fence->state < NOUVEAU_FENCE_STATE_EMITTED) { - PUSH_SPACE(context->pushbuf, 8); - nouveau_fence_emit(fence); + if (PUSH_AVAIL(context->pushbuf) < 16) + nouveau_pushbuf_space(context->pushbuf, 16, 0, 0); + _nouveau_fence_emit(fence); } - if (fence->state < NOUVEAU_FENCE_STATE_FLUSHED) + if (fence->state < NOUVEAU_FENCE_STATE_FLUSHED) { if (nouveau_pushbuf_kick(context->pushbuf, context->pushbuf->channel)) return false; + } if (current) - nouveau_fence_next(fence->context); + _nouveau_fence_next(fence->context); - nouveau_fence_update(screen, false); + _nouveau_fence_update(screen, false); return true; } -bool -nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug) +static bool +_nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug) { struct nouveau_screen *screen = fence->screen; struct nouveau_fence_list *fence_list = &screen->fence; uint32_t spins = 0; int64_t start = 0; + simple_mtx_assert_locked(&fence_list->lock); + if (debug && debug->debug_message) start = os_time_get_nano(); @@ -243,7 +266,7 @@ nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debu sched_yield(); #endif - nouveau_fence_update(screen, false); + _nouveau_fence_update(screen, false); } while (spins < NOUVEAU_FENCE_MAX_SPINS); debug_printf("Wait on fence %u (ack = %u, next = %u) timed out !\n", @@ -254,16 +277,20 @@ nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debu } void -nouveau_fence_next(struct nouveau_context *nv) +_nouveau_fence_next(struct nouveau_context *nv) { + struct nouveau_fence_list *fence_list = &nv->screen->fence; + + simple_mtx_assert_locked(&fence_list->lock); + if (nv->fence->state < NOUVEAU_FENCE_STATE_EMITTING) { - if (nv->fence->ref > 1) - nouveau_fence_emit(nv->fence); + if (p_atomic_read(&nv->fence->ref) > 1) + _nouveau_fence_emit(nv->fence); else return; } - nouveau_fence_ref(NULL, &nv->fence); + _nouveau_fence_ref(NULL, &nv->fence); nouveau_fence_new(nv, &nv->fence); } @@ -281,6 +308,7 @@ nouveau_fence_work(struct nouveau_fence *fence, void (*func)(void *), void *data) { struct nouveau_fence_work *work; + struct nouveau_screen *screen; if (!fence || fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) { func(data); @@ -292,9 +320,72 @@ nouveau_fence_work(struct nouveau_fence *fence, return false; work->func = func; work->data = data; + + /* the fence might get deleted by fence_kick */ + screen = fence->screen; + + simple_mtx_lock(&screen->fence.lock); list_add(&work->list, &fence->work); - p_atomic_inc(&fence->work_count); - if (fence->work_count > 64) + if (++fence->work_count > 64) nouveau_fence_kick(fence); + simple_mtx_unlock(&screen->fence.lock); return true; } + +void +_nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref) +{ + if (fence) + p_atomic_inc(&fence->ref); + + if (*ref) { + simple_mtx_assert_locked(&(*ref)->screen->fence.lock); + if (p_atomic_dec_zero(&(*ref)->ref)) + nouveau_fence_del(*ref); + } + + *ref = fence; +} + +void +nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref) +{ + struct nouveau_fence_list *fence_list = NULL; + if (ref && *ref) + fence_list = &(*ref)->screen->fence; + + if (fence_list) + simple_mtx_lock(&fence_list->lock); + + _nouveau_fence_ref(fence, ref); + + if (fence_list) + simple_mtx_unlock(&fence_list->lock); +} + +bool +nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug) +{ + struct nouveau_fence_list *fence_list = &fence->screen->fence; + simple_mtx_lock(&fence_list->lock); + bool res = _nouveau_fence_wait(fence, debug); + simple_mtx_unlock(&fence_list->lock); + return res; +} + +void +nouveau_fence_emit(struct nouveau_fence *fence) +{ + simple_mtx_lock(&fence->screen->fence.lock); + _nouveau_fence_emit(fence); + simple_mtx_unlock(&fence->screen->fence.lock); +} + +bool +nouveau_fence_signalled(struct nouveau_fence *fence) +{ + simple_mtx_lock(&fence->screen->fence.lock); + bool ret = _nouveau_fence_signalled(fence); + simple_mtx_unlock(&fence->screen->fence.lock); + return ret; +} diff --git a/src/gallium/drivers/nouveau/nouveau_fence.h b/src/gallium/drivers/nouveau/nouveau_fence.h index 49e74bfc52c..444598b3eb1 100644 --- a/src/gallium/drivers/nouveau/nouveau_fence.h +++ b/src/gallium/drivers/nouveau/nouveau_fence.h @@ -4,6 +4,7 @@ #include "util/u_inlines.h" #include "util/list.h" +#include "util/simple_mtx.h" #define NOUVEAU_FENCE_STATE_AVAILABLE 0 #define NOUVEAU_FENCE_STATE_EMITTING 1 @@ -35,13 +36,29 @@ struct nouveau_fence_list { struct nouveau_fence *tail; uint32_t sequence; uint32_t sequence_ack; + simple_mtx_t lock; void (*emit)(struct pipe_context *, uint32_t *sequence); uint32_t (*update)(struct pipe_screen *); }; -void nouveau_fence_emit(struct nouveau_fence *); -void nouveau_fence_del(struct nouveau_fence *); +static inline void +nouveau_fence_list_init(struct nouveau_fence_list *fence_list) +{ + simple_mtx_init(&fence_list->lock, mtx_plain); +} +static inline void +nouveau_fence_list_destroy(struct nouveau_fence_list *fence_list) +{ + simple_mtx_destroy(&fence_list->lock); +} + +/* unlocked versions, use with care */ +void _nouveau_fence_update(struct nouveau_screen *, bool flushed); +void _nouveau_fence_next(struct nouveau_context *); +void _nouveau_fence_ref(struct nouveau_fence *, struct nouveau_fence **); + +void nouveau_fence_emit(struct nouveau_fence *); bool nouveau_fence_new(struct nouveau_context *, struct nouveau_fence **); void nouveau_fence_cleanup(struct nouveau_context *); bool nouveau_fence_work(struct nouveau_fence *, void (*)(void *), void *); @@ -49,24 +66,10 @@ void nouveau_fence_update(struct nouveau_screen *, bool flushed); void nouveau_fence_next(struct nouveau_context *); bool nouveau_fence_wait(struct nouveau_fence *, struct util_debug_callback *); bool nouveau_fence_signalled(struct nouveau_fence *); +void nouveau_fence_ref(struct nouveau_fence *, struct nouveau_fence **); void nouveau_fence_unref_bo(void *data); /* generic unref bo callback */ - -static inline void -nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref) -{ - if (fence) - ++fence->ref; - - if (*ref) { - if (--(*ref)->ref == 0) - nouveau_fence_del(*ref); - } - - *ref = fence; -} - static inline struct nouveau_fence * nouveau_fence(struct pipe_fence_handle *fence) { diff --git a/src/gallium/drivers/nouveau/nouveau_screen.c b/src/gallium/drivers/nouveau/nouveau_screen.c index 8121082c9e3..b055115992c 100644 --- a/src/gallium/drivers/nouveau/nouveau_screen.c +++ b/src/gallium/drivers/nouveau/nouveau_screen.c @@ -209,7 +209,7 @@ nouveau_pushbuf_cb(struct nouveau_pushbuf *push) if (p->context) p->context->kick_notify(p->context); else - nouveau_fence_update(p->screen, true); + _nouveau_fence_update(p->screen, true); NOUVEAU_DRV_STAT(p->screen, pushbuf_count, 1); } @@ -400,6 +400,7 @@ nouveau_screen_init(struct nouveau_screen *screen, struct nouveau_device *dev) PIPE_BIND_COMMAND_ARGS_BUFFER; memset(&mm_config, 0, sizeof(mm_config)); + nouveau_fence_list_init(&screen->fence); screen->mm_GART = nouveau_mm_create(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, @@ -436,6 +437,7 @@ nouveau_screen_fini(struct nouveau_screen *screen) close(fd); disk_cache_destroy(screen->disk_shader_cache); + nouveau_fence_list_destroy(&screen->fence); } static void diff --git a/src/gallium/drivers/nouveau/nouveau_winsys.h b/src/gallium/drivers/nouveau/nouveau_winsys.h index afff3ab6eef..6a479ef6689 100644 --- a/src/gallium/drivers/nouveau/nouveau_winsys.h +++ b/src/gallium/drivers/nouveau/nouveau_winsys.h @@ -28,7 +28,11 @@ PUSH_AVAIL(struct nouveau_pushbuf *push) static inline bool PUSH_SPACE_EX(struct nouveau_pushbuf *push, uint32_t size, uint32_t relocs, uint32_t pushes) { - return nouveau_pushbuf_space(push, size, relocs, pushes) == 0; + struct nouveau_pushbuf_priv *ppush = push->user_priv; + simple_mtx_lock(&ppush->screen->fence.lock); + bool res = nouveau_pushbuf_space(push, size, relocs, pushes) == 0; + simple_mtx_unlock(&ppush->screen->fence.lock); + return res; } static inline bool @@ -72,7 +76,11 @@ PUSH_DATAf(struct nouveau_pushbuf *push, float f) static inline int PUSH_REFN(struct nouveau_pushbuf *push, struct nouveau_pushbuf_refn *refs, int nr) { - return nouveau_pushbuf_refn(push, refs, nr); + struct nouveau_pushbuf_priv *ppush = push->user_priv; + simple_mtx_lock(&ppush->screen->fence.lock); + int ret = nouveau_pushbuf_refn(push, refs, nr); + simple_mtx_unlock(&ppush->screen->fence.lock); + return ret; } static inline int @@ -85,25 +93,40 @@ PUSH_REF1(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t flags) static inline void PUSH_KICK(struct nouveau_pushbuf *push) { + struct nouveau_pushbuf_priv *ppush = push->user_priv; + simple_mtx_lock(&ppush->screen->fence.lock); nouveau_pushbuf_kick(push, push->channel); + simple_mtx_unlock(&ppush->screen->fence.lock); } static inline int PUSH_VAL(struct nouveau_pushbuf *push) { - return nouveau_pushbuf_validate(push); + struct nouveau_pushbuf_priv *ppush = push->user_priv; + simple_mtx_lock(&ppush->screen->fence.lock); + int res = nouveau_pushbuf_validate(push); + simple_mtx_unlock(&ppush->screen->fence.lock); + return res; } static inline int BO_MAP(struct nouveau_screen *screen, struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client) { - return nouveau_bo_map(bo, access, client); + int res; + simple_mtx_lock(&screen->fence.lock); + res = nouveau_bo_map(bo, access, client); + simple_mtx_unlock(&screen->fence.lock); + return res; } static inline int BO_WAIT(struct nouveau_screen *screen, struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client) { - return nouveau_bo_wait(bo, access, client); + int res; + simple_mtx_lock(&screen->fence.lock); + res = nouveau_bo_wait(bo, access, client); + simple_mtx_unlock(&screen->fence.lock); + return res; } #define NOUVEAU_RESOURCE_FLAG_LINEAR (PIPE_RESOURCE_FLAG_DRV_PRIV << 0) diff --git a/src/gallium/drivers/nouveau/nv30/nv30_context.c b/src/gallium/drivers/nouveau/nv30/nv30_context.c index 50c5071b266..7ce9f33a165 100644 --- a/src/gallium/drivers/nouveau/nv30/nv30_context.c +++ b/src/gallium/drivers/nouveau/nv30/nv30_context.c @@ -41,21 +41,21 @@ nv30_context_kick_notify(struct nouveau_pushbuf *push) struct nouveau_pushbuf_priv *p = push->user_priv; struct nouveau_screen *screen = p->screen; - nouveau_fence_next(p->context); - nouveau_fence_update(screen, true); + _nouveau_fence_next(p->context); + _nouveau_fence_update(screen, true); if (push->bufctx) { struct nouveau_bufref *bref; LIST_FOR_EACH_ENTRY(bref, &push->bufctx->current, thead) { struct nv04_resource *res = bref->priv; if (res && res->mm) { - nouveau_fence_ref(p->context->fence, &res->fence); + _nouveau_fence_ref(p->context->fence, &res->fence); if (bref->flags & NOUVEAU_BO_RD) res->status |= NOUVEAU_BUFFER_STATUS_GPU_READING; if (bref->flags & NOUVEAU_BO_WR) { - nouveau_fence_ref(p->context->fence, &res->fence_wr); + _nouveau_fence_ref(p->context->fence, &res->fence_wr); res->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING | NOUVEAU_BUFFER_STATUS_DIRTY; } diff --git a/src/gallium/drivers/nouveau/nv50/nv50_context.c b/src/gallium/drivers/nouveau/nv50/nv50_context.c index bbc4131a499..977ce2e27e7 100644 --- a/src/gallium/drivers/nouveau/nv50/nv50_context.c +++ b/src/gallium/drivers/nouveau/nv50/nv50_context.c @@ -136,8 +136,8 @@ nv50_default_kick_notify(struct nouveau_context *context) { struct nv50_context *nv50 = nv50_context(&context->pipe); - nouveau_fence_next(context); - nouveau_fence_update(context->screen, true); + _nouveau_fence_next(context); + _nouveau_fence_update(context->screen, true); nv50->state.flushed = true; } diff --git a/src/gallium/drivers/nouveau/nv50/nv50_vbo.c b/src/gallium/drivers/nouveau/nv50/nv50_vbo.c index f2aebccb235..2a49539b0cd 100644 --- a/src/gallium/drivers/nouveau/nv50/nv50_vbo.c +++ b/src/gallium/drivers/nouveau/nv50/nv50_vbo.c @@ -749,7 +749,7 @@ nva0_draw_stream_output(struct nv50_context *nv50, static void nv50_draw_vbo_kick_notify(struct nouveau_context *context) { - nouveau_fence_update(context->screen, true); + _nouveau_fence_update(context->screen, true); } void diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_context.c b/src/gallium/drivers/nouveau/nvc0/nvc0_context.c index 1fd77b1db64..1c90ba17853 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_context.c +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_context.c @@ -280,8 +280,8 @@ nvc0_default_kick_notify(struct nouveau_context *context) { struct nvc0_context *nvc0 = nvc0_context(&context->pipe); - nouveau_fence_next(context); - nouveau_fence_update(context->screen, true); + _nouveau_fence_next(context); + _nouveau_fence_update(context->screen, true); nvc0->state.flushed = true; } diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c b/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c index 8f11296f812..d55792c44e8 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c @@ -559,7 +559,7 @@ nvc0_prim_gl(unsigned prim) static void nvc0_draw_vbo_kick_notify(struct nouveau_context *context) { - nouveau_fence_update(context->screen, true); + _nouveau_fence_update(context->screen, true); } static void