nouveau/push: Cache the last header DW to avoid read-back

The pushbuf may live in VRAM in which case readback gets very expensive.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30033>

src/nouveau/headers/nv_push.h

@@ -15,7 +15,12 @@ struct nv_push {
    uint32_t *start;
    uint32_t *end;
    uint32_t *limit;
-   uint32_t *last_size;
+
+   /* A pointer to the last method header */
+   uint32_t *last_hdr;
+
+   /* The value in the last method header, used to avoid read-back */
+   uint32_t last_hdr_dw;
 };
 
 static inline void
@@ -24,7 +29,8 @@ nv_push_init(struct nv_push *push, uint32_t *start, size_t dw_count)
    push->start = start;
    push->end = start;
    push->limit = start + dw_count;
-   push->last_size = NULL;
+   push->last_hdr = NULL;
+   push->last_hdr_dw = 0;
 }
 
 static inline size_t
@@ -76,28 +82,32 @@ NVC0_FIFO_PKHDR_SQ(int subc, int mthd, unsigned size)
 static inline void
 __push_verify(struct nv_push *push)
 {
-   if (!push->last_size)
+   if (push->last_hdr == NULL)
       return;
 
-   /* make sure we don't add a new method if the last one wasn't used */
-   uint32_t last_hdr = *push->last_size;
-
    /* check for immd */
-   if (last_hdr >> 29 == 4)
+   if (push->last_hdr_dw >> 29 == 4)
       return;
 
-   UNUSED uint32_t last_count = (last_hdr & 0x1fff0000);
+   ASSERTED uint32_t last_count = (push->last_hdr_dw & 0x1fff0000);
    assert(last_count);
 }
 
+static inline void
+__push_hdr(struct nv_push *push, uint32_t hdr)
+{
+   __push_verify(push);
+
+   *push->end = hdr;
+   push->last_hdr_dw = hdr;
+   push->last_hdr = push->end;
+   push->end++;
+}
+
 static inline void
 __push_mthd_size(struct nv_push *push, int subc, uint32_t mthd, unsigned size)
 {
-   __push_verify(push);
-
-   push->last_size = push->end;
-   *push->end = NVC0_FIFO_PKHDR_SQ(subc, mthd, size);
-   push->end++;
+   __push_hdr(push, NVC0_FIFO_PKHDR_SQ(subc, mthd, size));
 }
 
 static inline void
@@ -118,10 +128,7 @@ NVC0_FIFO_PKHDR_IL(int subc, int mthd, uint16_t data)
 static inline void
 __push_immd(struct nv_push *push, int subc, uint32_t mthd, uint32_t val)
 {
-   __push_verify(push);
-   push->last_size = push->end;
-   *push->end = NVC0_FIFO_PKHDR_IL(subc, mthd, val);
-   push->end++;
+   __push_hdr(push, NVC0_FIFO_PKHDR_IL(subc, mthd, val));
 }
 
 #define P_IMMD(push, class, mthd, args...) do {                         \
@@ -144,10 +151,7 @@ NVC0_FIFO_PKHDR_1I(int subc, int mthd, unsigned size)
 static inline void
 __push_1inc(struct nv_push *push, int subc, uint32_t mthd)
 {
-   __push_verify(push);
-   push->last_size = push->end;
-   *push->end = NVC0_FIFO_PKHDR_1I(subc, mthd, 0);
-   push->end++;
+   __push_hdr(push, NVC0_FIFO_PKHDR_1I(subc, mthd, 0));
 }
 
 #define P_1INC(push, class, mthd) __push_1inc(push, SUBC_##class, class##_##mthd)
@@ -161,10 +165,7 @@ NVC0_FIFO_PKHDR_0I(int subc, int mthd, unsigned size)
 static inline void
 __push_0inc(struct nv_push *push, int subc, uint32_t mthd)
 {
-   __push_verify(push);
-   push->last_size = push->end;
-   *push->end = NVC0_FIFO_PKHDR_0I(subc, mthd, 0);
-   push->end++;
+   __push_hdr(push, NVC0_FIFO_PKHDR_0I(subc, mthd, 0));
 }
 
 #define P_0INC(push, class, mthd) __push_0inc(push, SUBC_##class, class##_##mthd)
@@ -174,23 +175,26 @@ __push_0inc(struct nv_push *push, int subc, uint32_t mthd)
 static inline bool
 nv_push_update_count(struct nv_push *push, uint16_t count)
 {
-   uint32_t last_hdr_val = *push->last_size;
+   assert(push->last_hdr != NULL);
 
    assert(count <= NV_PUSH_MAX_COUNT);
    if (count > NV_PUSH_MAX_COUNT)
       return false;
 
+   uint32_t hdr_dw = push->last_hdr_dw;
+
    /* size is encoded at 28:16 */
-   uint32_t new_count = (count + (last_hdr_val >> 16)) & NV_PUSH_MAX_COUNT;
+   uint32_t new_count = (count + (hdr_dw >> 16)) & NV_PUSH_MAX_COUNT;
    bool overflow = new_count < count;
    /* if we would overflow, don't change anything and just let it be */
    assert(!overflow);
    if (overflow)
       return false;
 
-   last_hdr_val &= ~0x1fff0000;
-   last_hdr_val |= new_count << 16;
-   *push->last_size = last_hdr_val;
+   hdr_dw &= ~0x1fff0000;
+   hdr_dw |= new_count << 16;
+   push->last_hdr_dw = hdr_dw;
+   *push->last_hdr = hdr_dw;
    return true;
 }
 
@@ -228,13 +232,13 @@ P_INLINE_ARRAY(struct nv_push *push, const uint32_t *data, int num_dw)
 static inline void
 nv_push_val(struct nv_push *push, uint32_t idx, uint32_t val)
 {
-   UNUSED uint32_t last_hdr_val = *push->last_size;
-   UNUSED bool is_0inc = (last_hdr_val & 0xe0000000) == 0x60000000;
-   UNUSED bool is_1inc = (last_hdr_val & 0xe0000000) == 0xa0000000;
-   UNUSED bool is_immd = (last_hdr_val & 0xe0000000) == 0x80000000;
-   UNUSED uint16_t last_method = (last_hdr_val & 0x1fff) << 2;
+   ASSERTED uint32_t last_hdr_dw = push->last_hdr_dw;
+   ASSERTED bool is_0inc = (last_hdr_dw & 0xe0000000) == 0x60000000;
+   ASSERTED bool is_1inc = (last_hdr_dw & 0xe0000000) == 0xa0000000;
+   ASSERTED bool is_immd = (last_hdr_dw & 0xe0000000) == 0x80000000;
+   ASSERTED uint16_t last_method = (last_hdr_dw & 0x1fff) << 2;
 
-   uint16_t distance = push->end - push->last_size - 1;
+   uint16_t distance = push->end - push->last_hdr - 1;
    if (is_0inc)
       distance = 0;
    else if (is_1inc)
@@ -242,7 +246,7 @@ nv_push_val(struct nv_push *push, uint32_t idx, uint32_t val)
    last_method += distance * 4;
 
    /* can't have empty headers ever */
-   assert(last_hdr_val);
+   assert(last_hdr_dw);
    assert(!is_immd);
    assert(last_method == idx);
    assert(push->end < push->limit);
@@ -256,7 +260,7 @@ nv_push_raw(struct nv_push *push, uint32_t *raw_dw, uint32_t dw_count)
    assert(push->end + dw_count <= push->limit);
    memcpy(push->end, raw_dw, dw_count * 4);
    push->end += dw_count;
-   push->last_size = NULL;
+   push->last_hdr = NULL;
 }
 
 #endif /* NV_PUSH_H */

src/nouveau/headers/nv_push_dump.c

@@ -133,7 +133,6 @@ int main(int argc, char **argv) {
   fclose(file);
 
   nv_push_init(&pushbuf, data, file_size / 4);
-  pushbuf.last_size = pushbuf.limit - 1;
   pushbuf.end = pushbuf.limit;
 
   vk_push_print(stdout, &pushbuf, &device_info);