nouveau: make fencing race free

Signed-off-by: Karol Herbst <kherbst@redhat.com>
Acked-by: M Henning <drawoc@darkrefraction.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10752>

src/gallium/drivers/nouveau/nouveau_fence.c

@@ -30,6 +30,9 @@
 #include <sched.h>
 #endif
 
+static bool
+_nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug);
+
 bool
 nouveau_fence_new(struct nouveau_context *nv, struct nouveau_fence **fence)
 {
@@ -48,6 +51,8 @@ nouveau_fence_new(struct nouveau_context *nv, struct nouveau_fence **fence)
 static void
 nouveau_fence_trigger_work(struct nouveau_fence *fence)
 {
+   simple_mtx_assert_locked(&fence->screen->fence.lock);
+
    struct nouveau_fence_work *work, *tmp;
 
    LIST_FOR_EACH_ENTRY_SAFE(work, tmp, &fence->work, list) {
@@ -57,12 +62,14 @@ nouveau_fence_trigger_work(struct nouveau_fence *fence)
    }
 }
 
-void
-nouveau_fence_emit(struct nouveau_fence *fence)
+static void
+_nouveau_fence_emit(struct nouveau_fence *fence)
 {
    struct nouveau_screen *screen = fence->screen;
    struct nouveau_fence_list *fence_list = &screen->fence;
 
+   simple_mtx_assert_locked(&fence_list->lock);
+
    assert(fence->state != NOUVEAU_FENCE_STATE_EMITTING);
    if (fence->state >= NOUVEAU_FENCE_STATE_EMITTED)
       return;
@@ -70,7 +77,7 @@ nouveau_fence_emit(struct nouveau_fence *fence)
    /* set this now, so that if fence.emit triggers a flush we don't recurse */
    fence->state = NOUVEAU_FENCE_STATE_EMITTING;
 
-   ++fence->ref;
+   p_atomic_inc(&fence->ref);
 
    if (fence_list->tail)
       fence_list->tail->next = fence;
@@ -85,12 +92,14 @@ nouveau_fence_emit(struct nouveau_fence *fence)
    fence->state = NOUVEAU_FENCE_STATE_EMITTED;
 }
 
-void
+static void
 nouveau_fence_del(struct nouveau_fence *fence)
 {
    struct nouveau_fence *it;
    struct nouveau_fence_list *fence_list = &fence->screen->fence;
 
+   simple_mtx_assert_locked(&fence_list->lock);
+
    if (fence->state == NOUVEAU_FENCE_STATE_EMITTED ||
        fence->state == NOUVEAU_FENCE_STATE_FLUSHED) {
       if (fence == fence_list->head) {
@@ -117,26 +126,31 @@ void
 nouveau_fence_cleanup(struct nouveau_context *nv)
 {
    if (nv->fence) {
+      struct nouveau_fence_list *fence_list = &nv->screen->fence;
       struct nouveau_fence *current = NULL;
 
       /* nouveau_fence_wait will create a new current fence, so wait on the
        * _current_ one, and remove both.
        */
-      nouveau_fence_ref(nv->fence, &current);
-      nouveau_fence_wait(current, NULL);
-      nouveau_fence_ref(NULL, &current);
-      nouveau_fence_ref(NULL, &nv->fence);
+      simple_mtx_lock(&fence_list->lock);
+      _nouveau_fence_ref(nv->fence, &current);
+      _nouveau_fence_wait(current, NULL);
+      _nouveau_fence_ref(NULL, &current);
+      _nouveau_fence_ref(NULL, &nv->fence);
+      simple_mtx_unlock(&fence_list->lock);
    }
 }
 
 void
-nouveau_fence_update(struct nouveau_screen *screen, bool flushed)
+_nouveau_fence_update(struct nouveau_screen *screen, bool flushed)
 {
    struct nouveau_fence *fence;
    struct nouveau_fence *next = NULL;
    struct nouveau_fence_list *fence_list = &screen->fence;
    u32 sequence = fence_list->update(&screen->base);
 
+   simple_mtx_assert_locked(&fence_list->lock);
+
    /* If running under drm-shim, let all fences be signalled so things run to
     * completion (avoids a hang at the end of shader-db).
     */
@@ -154,7 +168,7 @@ nouveau_fence_update(struct nouveau_screen *screen, bool flushed)
       fence->state = NOUVEAU_FENCE_STATE_SIGNALLED;
 
       nouveau_fence_trigger_work(fence);
-      nouveau_fence_ref(NULL, &fence);
+      _nouveau_fence_ref(NULL, &fence);
 
       if (sequence == fence_list->sequence_ack)
          break;
@@ -172,16 +186,18 @@ nouveau_fence_update(struct nouveau_screen *screen, bool flushed)
 
 #define NOUVEAU_FENCE_MAX_SPINS (1 << 31)
 
-bool
-nouveau_fence_signalled(struct nouveau_fence *fence)
+static bool
+_nouveau_fence_signalled(struct nouveau_fence *fence)
 {
    struct nouveau_screen *screen = fence->screen;
 
+   simple_mtx_assert_locked(&screen->fence.lock);
+
    if (fence->state == NOUVEAU_FENCE_STATE_SIGNALLED)
       return true;
 
    if (fence->state >= NOUVEAU_FENCE_STATE_EMITTED)
-      nouveau_fence_update(screen, false);
+      _nouveau_fence_update(screen, false);
 
    return fence->state == NOUVEAU_FENCE_STATE_SIGNALLED;
 }
@@ -191,36 +207,43 @@ nouveau_fence_kick(struct nouveau_fence *fence)
 {
    struct nouveau_context *context = fence->context;
    struct nouveau_screen *screen = fence->screen;
+   struct nouveau_fence_list *fence_list = &screen->fence;
    bool current = !fence->sequence;
 
+   simple_mtx_assert_locked(&fence_list->lock);
+
    /* wtf, someone is waiting on a fence in flush_notify handler? */
    assert(fence->state != NOUVEAU_FENCE_STATE_EMITTING);
 
    if (fence->state < NOUVEAU_FENCE_STATE_EMITTED) {
-      PUSH_SPACE(context->pushbuf, 8);
-      nouveau_fence_emit(fence);
+      if (PUSH_AVAIL(context->pushbuf) < 16)
+         nouveau_pushbuf_space(context->pushbuf, 16, 0, 0);
+      _nouveau_fence_emit(fence);
    }
 
-   if (fence->state < NOUVEAU_FENCE_STATE_FLUSHED)
+   if (fence->state < NOUVEAU_FENCE_STATE_FLUSHED) {
       if (nouveau_pushbuf_kick(context->pushbuf, context->pushbuf->channel))
          return false;
+   }
 
    if (current)
-      nouveau_fence_next(fence->context);
+      _nouveau_fence_next(fence->context);
 
-   nouveau_fence_update(screen, false);
+   _nouveau_fence_update(screen, false);
 
    return true;
 }
 
-bool
-nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug)
+static bool
+_nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug)
 {
    struct nouveau_screen *screen = fence->screen;
    struct nouveau_fence_list *fence_list = &screen->fence;
    uint32_t spins = 0;
    int64_t start = 0;
 
+   simple_mtx_assert_locked(&fence_list->lock);
+
    if (debug && debug->debug_message)
       start = os_time_get_nano();
 
@@ -243,7 +266,7 @@ nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debu
          sched_yield();
 #endif
 
-      nouveau_fence_update(screen, false);
+      _nouveau_fence_update(screen, false);
    } while (spins < NOUVEAU_FENCE_MAX_SPINS);
 
    debug_printf("Wait on fence %u (ack = %u, next = %u) timed out !\n",
@@ -254,16 +277,20 @@ nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debu
 }
 
 void
-nouveau_fence_next(struct nouveau_context *nv)
+_nouveau_fence_next(struct nouveau_context *nv)
 {
+   struct nouveau_fence_list *fence_list = &nv->screen->fence;
+
+   simple_mtx_assert_locked(&fence_list->lock);
+
    if (nv->fence->state < NOUVEAU_FENCE_STATE_EMITTING) {
-      if (nv->fence->ref > 1)
-         nouveau_fence_emit(nv->fence);
+      if (p_atomic_read(&nv->fence->ref) > 1)
+         _nouveau_fence_emit(nv->fence);
       else
          return;
    }
 
-   nouveau_fence_ref(NULL, &nv->fence);
+   _nouveau_fence_ref(NULL, &nv->fence);
 
    nouveau_fence_new(nv, &nv->fence);
 }
@@ -281,6 +308,7 @@ nouveau_fence_work(struct nouveau_fence *fence,
                    void (*func)(void *), void *data)
 {
    struct nouveau_fence_work *work;
+   struct nouveau_screen *screen;
 
    if (!fence || fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) {
       func(data);
@@ -292,9 +320,72 @@ nouveau_fence_work(struct nouveau_fence *fence,
       return false;
    work->func = func;
    work->data = data;
+
+   /* the fence might get deleted by fence_kick */
+   screen = fence->screen;
+
+   simple_mtx_lock(&screen->fence.lock);
    list_add(&work->list, &fence->work);
-   p_atomic_inc(&fence->work_count);
-   if (fence->work_count > 64)
+   if (++fence->work_count > 64)
       nouveau_fence_kick(fence);
+   simple_mtx_unlock(&screen->fence.lock);
    return true;
 }
+
+void
+_nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref)
+{
+   if (fence)
+      p_atomic_inc(&fence->ref);
+
+   if (*ref) {
+      simple_mtx_assert_locked(&(*ref)->screen->fence.lock);
+      if (p_atomic_dec_zero(&(*ref)->ref))
+         nouveau_fence_del(*ref);
+   }
+
+   *ref = fence;
+}
+
+void
+nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref)
+{
+   struct nouveau_fence_list *fence_list = NULL;
+   if (ref && *ref)
+      fence_list = &(*ref)->screen->fence;
+
+   if (fence_list)
+      simple_mtx_lock(&fence_list->lock);
+
+   _nouveau_fence_ref(fence, ref);
+
+   if (fence_list)
+      simple_mtx_unlock(&fence_list->lock);
+}
+
+bool
+nouveau_fence_wait(struct nouveau_fence *fence, struct util_debug_callback *debug)
+{
+   struct nouveau_fence_list *fence_list = &fence->screen->fence;
+   simple_mtx_lock(&fence_list->lock);
+   bool res = _nouveau_fence_wait(fence, debug);
+   simple_mtx_unlock(&fence_list->lock);
+   return res;
+}
+
+void
+nouveau_fence_emit(struct nouveau_fence *fence)
+{
+   simple_mtx_lock(&fence->screen->fence.lock);
+   _nouveau_fence_emit(fence);
+   simple_mtx_unlock(&fence->screen->fence.lock);
+}
+
+bool
+nouveau_fence_signalled(struct nouveau_fence *fence)
+{
+   simple_mtx_lock(&fence->screen->fence.lock);
+   bool ret = _nouveau_fence_signalled(fence);
+   simple_mtx_unlock(&fence->screen->fence.lock);
+   return ret;
+}

src/gallium/drivers/nouveau/nouveau_fence.h

@@ -4,6 +4,7 @@
 
 #include "util/u_inlines.h"
 #include "util/list.h"
+#include "util/simple_mtx.h"
 
 #define NOUVEAU_FENCE_STATE_AVAILABLE 0
 #define NOUVEAU_FENCE_STATE_EMITTING  1
@@ -35,13 +36,29 @@ struct nouveau_fence_list {
    struct nouveau_fence *tail;
    uint32_t sequence;
    uint32_t sequence_ack;
+   simple_mtx_t lock;
    void (*emit)(struct pipe_context *, uint32_t *sequence);
    uint32_t (*update)(struct pipe_screen *);
 };
 
-void nouveau_fence_emit(struct nouveau_fence *);
-void nouveau_fence_del(struct nouveau_fence *);
+static inline void
+nouveau_fence_list_init(struct nouveau_fence_list *fence_list)
+{
+   simple_mtx_init(&fence_list->lock, mtx_plain);
+}
 
+static inline void
+nouveau_fence_list_destroy(struct nouveau_fence_list *fence_list)
+{
+   simple_mtx_destroy(&fence_list->lock);
+}
+
+/* unlocked versions, use with care */
+void _nouveau_fence_update(struct nouveau_screen *, bool flushed);
+void _nouveau_fence_next(struct nouveau_context *);
+void _nouveau_fence_ref(struct nouveau_fence *, struct nouveau_fence **);
+
+void nouveau_fence_emit(struct nouveau_fence *);
 bool nouveau_fence_new(struct nouveau_context *, struct nouveau_fence **);
 void nouveau_fence_cleanup(struct nouveau_context *);
 bool nouveau_fence_work(struct nouveau_fence *, void (*)(void *), void *);
@@ -49,24 +66,10 @@ void nouveau_fence_update(struct nouveau_screen *, bool flushed);
 void nouveau_fence_next(struct nouveau_context *);
 bool nouveau_fence_wait(struct nouveau_fence *, struct util_debug_callback *);
 bool nouveau_fence_signalled(struct nouveau_fence *);
+void nouveau_fence_ref(struct nouveau_fence *, struct nouveau_fence **);
 
 void nouveau_fence_unref_bo(void *data); /* generic unref bo callback */
 
-
-static inline void
-nouveau_fence_ref(struct nouveau_fence *fence, struct nouveau_fence **ref)
-{
-   if (fence)
-      ++fence->ref;
-
-   if (*ref) {
-      if (--(*ref)->ref == 0)
-         nouveau_fence_del(*ref);
-   }
-
-   *ref = fence;
-}
-
 static inline struct nouveau_fence *
 nouveau_fence(struct pipe_fence_handle *fence)
 {

src/gallium/drivers/nouveau/nouveau_screen.c

@@ -209,7 +209,7 @@ nouveau_pushbuf_cb(struct nouveau_pushbuf *push)
    if (p->context)
       p->context->kick_notify(p->context);
    else
-      nouveau_fence_update(p->screen, true);
+      _nouveau_fence_update(p->screen, true);
 
    NOUVEAU_DRV_STAT(p->screen, pushbuf_count, 1);
 }
@@ -400,6 +400,7 @@ nouveau_screen_init(struct nouveau_screen *screen, struct nouveau_device *dev)
       PIPE_BIND_COMMAND_ARGS_BUFFER;
 
    memset(&mm_config, 0, sizeof(mm_config));
+   nouveau_fence_list_init(&screen->fence);
 
    screen->mm_GART = nouveau_mm_create(dev,
                                        NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
@@ -436,6 +437,7 @@ nouveau_screen_fini(struct nouveau_screen *screen)
    close(fd);
 
    disk_cache_destroy(screen->disk_shader_cache);
+   nouveau_fence_list_destroy(&screen->fence);
 }
 
 static void

src/gallium/drivers/nouveau/nouveau_winsys.h

@@ -28,7 +28,11 @@ PUSH_AVAIL(struct nouveau_pushbuf *push)
 static inline bool
 PUSH_SPACE_EX(struct nouveau_pushbuf *push, uint32_t size, uint32_t relocs, uint32_t pushes)
 {
-   return nouveau_pushbuf_space(push, size, relocs, pushes) == 0;
+   struct nouveau_pushbuf_priv *ppush = push->user_priv;
+   simple_mtx_lock(&ppush->screen->fence.lock);
+   bool res = nouveau_pushbuf_space(push, size, relocs, pushes) == 0;
+   simple_mtx_unlock(&ppush->screen->fence.lock);
+   return res;
 }
 
 static inline bool
@@ -72,7 +76,11 @@ PUSH_DATAf(struct nouveau_pushbuf *push, float f)
 static inline int
 PUSH_REFN(struct nouveau_pushbuf *push, struct nouveau_pushbuf_refn *refs, int nr)
 {
-   return nouveau_pushbuf_refn(push, refs, nr);
+   struct nouveau_pushbuf_priv *ppush = push->user_priv;
+   simple_mtx_lock(&ppush->screen->fence.lock);
+   int ret = nouveau_pushbuf_refn(push, refs, nr);
+   simple_mtx_unlock(&ppush->screen->fence.lock);
+   return ret;
 }
 
 static inline int
@@ -85,25 +93,40 @@ PUSH_REF1(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t flags)
 static inline void
 PUSH_KICK(struct nouveau_pushbuf *push)
 {
+   struct nouveau_pushbuf_priv *ppush = push->user_priv;
+   simple_mtx_lock(&ppush->screen->fence.lock);
    nouveau_pushbuf_kick(push, push->channel);
+   simple_mtx_unlock(&ppush->screen->fence.lock);
 }
 
 static inline int
 PUSH_VAL(struct nouveau_pushbuf *push)
 {
-   return nouveau_pushbuf_validate(push);
+   struct nouveau_pushbuf_priv *ppush = push->user_priv;
+   simple_mtx_lock(&ppush->screen->fence.lock);
+   int res = nouveau_pushbuf_validate(push);
+   simple_mtx_unlock(&ppush->screen->fence.lock);
+   return res;
 }
 
 static inline int
 BO_MAP(struct nouveau_screen *screen, struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
 {
-   return nouveau_bo_map(bo, access, client);
+   int res;
+   simple_mtx_lock(&screen->fence.lock);
+   res = nouveau_bo_map(bo, access, client);
+   simple_mtx_unlock(&screen->fence.lock);
+   return res;
 }
 
 static inline int
 BO_WAIT(struct nouveau_screen *screen, struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
 {
-   return nouveau_bo_wait(bo, access, client);
+   int res;
+   simple_mtx_lock(&screen->fence.lock);
+   res = nouveau_bo_wait(bo, access, client);
+   simple_mtx_unlock(&screen->fence.lock);
+   return res;
 }
 
 #define NOUVEAU_RESOURCE_FLAG_LINEAR   (PIPE_RESOURCE_FLAG_DRV_PRIV << 0)

src/gallium/drivers/nouveau/nv30/nv30_context.c

@@ -41,21 +41,21 @@ nv30_context_kick_notify(struct nouveau_pushbuf *push)
    struct nouveau_pushbuf_priv *p = push->user_priv;
    struct nouveau_screen *screen = p->screen;
 
-   nouveau_fence_next(p->context);
-   nouveau_fence_update(screen, true);
+   _nouveau_fence_next(p->context);
+   _nouveau_fence_update(screen, true);
 
    if (push->bufctx) {
       struct nouveau_bufref *bref;
       LIST_FOR_EACH_ENTRY(bref, &push->bufctx->current, thead) {
          struct nv04_resource *res = bref->priv;
          if (res && res->mm) {
-            nouveau_fence_ref(p->context->fence, &res->fence);
+            _nouveau_fence_ref(p->context->fence, &res->fence);
 
             if (bref->flags & NOUVEAU_BO_RD)
                res->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
 
             if (bref->flags & NOUVEAU_BO_WR) {
-               nouveau_fence_ref(p->context->fence, &res->fence_wr);
+               _nouveau_fence_ref(p->context->fence, &res->fence_wr);
                res->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING |
                   NOUVEAU_BUFFER_STATUS_DIRTY;
             }

src/gallium/drivers/nouveau/nv50/nv50_context.c

@@ -136,8 +136,8 @@ nv50_default_kick_notify(struct nouveau_context *context)
 {
    struct nv50_context *nv50 = nv50_context(&context->pipe);
 
-   nouveau_fence_next(context);
-   nouveau_fence_update(context->screen, true);
+   _nouveau_fence_next(context);
+   _nouveau_fence_update(context->screen, true);
    nv50->state.flushed = true;
 }
 

src/gallium/drivers/nouveau/nv50/nv50_vbo.c

@@ -749,7 +749,7 @@ nva0_draw_stream_output(struct nv50_context *nv50,
 static void
 nv50_draw_vbo_kick_notify(struct nouveau_context *context)
 {
-   nouveau_fence_update(context->screen, true);
+   _nouveau_fence_update(context->screen, true);
 }
 
 void

src/gallium/drivers/nouveau/nvc0/nvc0_context.c

@@ -280,8 +280,8 @@ nvc0_default_kick_notify(struct nouveau_context *context)
 {
    struct nvc0_context *nvc0 = nvc0_context(&context->pipe);
 
-   nouveau_fence_next(context);
-   nouveau_fence_update(context->screen, true);
+   _nouveau_fence_next(context);
+   _nouveau_fence_update(context->screen, true);
 
    nvc0->state.flushed = true;
 }

src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c

@@ -559,7 +559,7 @@ nvc0_prim_gl(unsigned prim)
 static void
 nvc0_draw_vbo_kick_notify(struct nouveau_context *context)
 {
-   nouveau_fence_update(context->screen, true);
+   _nouveau_fence_update(context->screen, true);
 }
 
 static void