cso: remove cso_draw_vbo from all draws, call the driver or u_vbuf directly

Instead of calling like this:
    st_draw_gallium -> cso_draw_vbo -> driver_draw_vbo

Do it like this:
    st_draw_gallium -> driver_draw_vbo
OR
    st_draw_gallium -> u_vbuf_draw_vbo

It's accomplished by adding a draw_vbo function pointer into cso_context.
The pointer is equal to pipe_context::draw_vbo when needed, so there is
no call overhead from this if cso's draw_vbo callback is indeed equal to
driver_draw_vbo. We just call cso_context_base::draw_vbo to jump into
the driver directly, or u_vbuf if needed.

The cso function with the indirect function call is inlined, so draws
don't actually visit any cso_context function.

Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20025>

src/gallium/auxiliary/cso_cache/cso_context.c

@@ -50,6 +50,7 @@
 #include "cso_cache/cso_hash.h"
 #include "cso_context.h"
 #include "driver_trace/tr_dump.h"
+#include "util/u_threaded_context.h"
 
 /**
  * Per-shader sampler information.
@@ -260,6 +261,19 @@ cso_init_vbuf(struct cso_context *cso, unsigned flags)
    }
 }
 
+static void
+cso_draw_vbo_default(struct pipe_context *pipe,
+                     const struct pipe_draw_info *info,
+                     unsigned drawid_offset,
+                     const struct pipe_draw_indirect_info *indirect,
+                     const struct pipe_draw_start_count_bias *draws,
+                     unsigned num_draws)
+{
+   if (pipe->vbuf)
+      u_vbuf_draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
+   else
+      pipe->draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
+}
 
 struct cso_context *
 cso_create_context(struct pipe_context *pipe, unsigned flags)
@@ -277,6 +291,21 @@ cso_create_context(struct pipe_context *pipe, unsigned flags)
    if (!(flags & CSO_NO_VBUF))
       cso_init_vbuf(ctx, flags);
 
+   /* Only drivers using u_threaded_context benefit from the direct call.
+    * This is because drivers can change draw_vbo, but u_threaded_context
+    * never changes it.
+    */
+   if (pipe->draw_vbo == tc_draw_vbo) {
+      if (ctx->vbuf_current)
+         ctx->base.draw_vbo = u_vbuf_draw_vbo;
+      else
+         ctx->base.draw_vbo = pipe->draw_vbo;
+   } else if (ctx->always_use_vbuf) {
+      ctx->base.draw_vbo = u_vbuf_draw_vbo;
+   } else {
+      ctx->base.draw_vbo = cso_draw_vbo_default;
+   }
+
    /* Enable for testing: */
    if (0) cso_set_maximum_cache_size(&ctx->cache, 4);
 
@@ -1268,7 +1297,9 @@ cso_set_vertex_buffers_and_elements(struct cso_context *ctx,
 
          /* Unset this to make sure the CSO is re-bound on the next use. */
          ctx->velements = NULL;
-         ctx->vbuf_current = vbuf;
+         ctx->vbuf_current = pipe->vbuf = vbuf;
+         if (pipe->draw_vbo == tc_draw_vbo)
+            ctx->base.draw_vbo = u_vbuf_draw_vbo;
          unbind_trailing_vb_count = 0;
       }
 
@@ -1289,7 +1320,9 @@ cso_set_vertex_buffers_and_elements(struct cso_context *ctx,
 
       /* Unset this to make sure the CSO is re-bound on the next use. */
       u_vbuf_unset_vertex_elements(vbuf);
-      ctx->vbuf_current = NULL;
+      ctx->vbuf_current = pipe->vbuf = NULL;
+      if (pipe->draw_vbo == tc_draw_vbo)
+         ctx->base.draw_vbo = pipe->draw_vbo;
       unbind_trailing_vb_count = 0;
    }
 
@@ -1732,37 +1765,6 @@ cso_restore_compute_state(struct cso_context *cso)
 
 /* drawing */
 
-void
-cso_draw_vbo(struct cso_context *cso,
-             struct pipe_draw_info *info,
-             unsigned drawid_offset,
-             const struct pipe_draw_indirect_info *indirect,
-             const struct pipe_draw_start_count_bias *draws,
-             unsigned num_draws)
-{
-   /* We can't have both indirect drawing and SO-vertex-count drawing */
-   assert(!indirect ||
-          indirect->buffer == NULL ||
-          indirect->count_from_stream_output == NULL);
-
-   /* We can't have SO-vertex-count drawing with an index buffer */
-   assert(info->index_size == 0 ||
-          !indirect ||
-          indirect->count_from_stream_output == NULL);
-
-   /* Indirect only uses indirect->draw_count, not num_draws. */
-   assert(!indirect || num_draws == 1);
-
-   struct pipe_context *pipe = cso->base.pipe;
-
-   if (cso->vbuf_current) {
-      u_vbuf_draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
-   } else {
-      pipe->draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
-   }
-}
-
-
 void
 cso_draw_arrays(struct cso_context *cso, uint mode, uint start, uint count)
 {

src/gallium/auxiliary/cso_cache/cso_context.h

@@ -44,6 +44,14 @@ struct u_vbuf;
 
 struct cso_context_base {
    struct pipe_context *pipe;
+
+   /* This is equal to either pipe_context::draw_vbo or u_vbuf_draw_vbo. */
+   void (*draw_vbo)(struct pipe_context *pipe,
+                    const struct pipe_draw_info *info,
+                    unsigned drawid_offset,
+                    const struct pipe_draw_indirect_info *indirect,
+                    const struct pipe_draw_start_count_bias *draws,
+                    unsigned num_draws);
 };
 
 #define CSO_NO_USER_VERTEX_BUFFERS (1 << 0)
@@ -210,14 +218,6 @@ cso_set_vertex_buffers_and_elements(struct cso_context *ctx,
                                     bool uses_user_vertex_buffers,
                                     const struct pipe_vertex_buffer *vbuffers);
 
-void
-cso_draw_vbo(struct cso_context *cso,
-             struct pipe_draw_info *info,
-             unsigned drawid_offset,
-             const struct pipe_draw_indirect_info *indirect,
-             const struct pipe_draw_start_count_bias *draws,
-             unsigned num_draws);
-
 void
 cso_draw_arrays_instanced(struct cso_context *cso, uint mode,
                           uint start, uint count,
@@ -236,6 +236,33 @@ cso_get_pipe_context(struct cso_context *cso)
    return cso_base->pipe;
 }
 
+static ALWAYS_INLINE void
+cso_draw_vbo(struct cso_context *cso,
+             struct pipe_draw_info *info,
+             unsigned drawid_offset,
+             const struct pipe_draw_indirect_info *indirect,
+             const struct pipe_draw_start_count_bias *draws,
+             unsigned num_draws)
+{
+   /* We can't have both indirect drawing and SO-vertex-count drawing */
+   assert(!indirect ||
+          indirect->buffer == NULL ||
+          indirect->count_from_stream_output == NULL);
+
+   /* We can't have SO-vertex-count drawing with an index buffer */
+   assert(info->index_size == 0 ||
+          !indirect ||
+          indirect->count_from_stream_output == NULL);
+
+   /* Indirect only uses indirect->draw_count, not num_draws. */
+   assert(!indirect || num_draws == 1);
+
+   struct cso_context_base *cso_base = (struct cso_context_base *)cso;
+
+   cso_base->draw_vbo(cso_base->pipe, info, drawid_offset, indirect, draws,
+                      num_draws);
+}
+
 #ifdef __cplusplus
 }
 #endif