intel: Buffered upload

Rather than performing lots of little writes to update the common bo
upon each update, write those into a static buffer and flush that when
full (or at the end of the batch). Doing so gives a dramatic performance
improvement over and above using mmaped access.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

src/mesa/drivers/dri/i965/brw_draw_upload.c

@@ -555,7 +555,7 @@ static void brw_prepare_indices(struct brw_context *brw)
 
    ib_type_size = get_size(index_buffer->type);
    ib_size = ib_type_size * index_buffer->count;
-   bufferobj = index_buffer->obj;;
+   bufferobj = index_buffer->obj;
 
    /* Turn into a proper VBO:
     */

src/mesa/drivers/dri/intel/intel_batchbuffer.c

@@ -27,6 +27,7 @@
 
 #include "intel_context.h"
 #include "intel_batchbuffer.h"
+#include "intel_buffer_objects.h"
 #include "intel_decode.h"
 #include "intel_reg.h"
 #include "intel_bufmgr.h"
@@ -175,11 +176,7 @@ _intel_batchbuffer_flush(struct intel_batchbuffer *batch, const char *file,
    if (intel->vtbl.finish_batch)
       intel->vtbl.finish_batch(intel);
 
-   if (intel->upload.bo) {
-      drm_intel_bo_unreference(intel->upload.bo);
-      intel->upload.bo = NULL;
-      intel->upload.offset = 0;
-   }
+   intel_upload_finish(intel);
 
    /* Check that we didn't just wrap our batchbuffer at a bad time. */
    assert(!intel->no_batch_wrap);

src/mesa/drivers/dri/intel/intel_buffer_objects.c

@@ -573,14 +573,31 @@ intel_bufferobj_buffer(struct intel_context *intel,
 
 #define INTEL_UPLOAD_SIZE (64*1024)
 
+void
+intel_upload_finish(struct intel_context *intel)
+{
+   if (!intel->upload.bo)
+	   return;
+
+   if (intel->upload.buffer_len) {
+	   drm_intel_bo_subdata(intel->upload.bo,
+				intel->upload.buffer_offset,
+				intel->upload.buffer_len,
+				intel->upload.buffer);
+	   intel->upload.buffer_len = 0;
+   }
+
+   drm_intel_bo_unreference(intel->upload.bo);
+   intel->upload.bo = NULL;
+}
+
 static void wrap_buffers(struct intel_context *intel, GLuint size)
 {
+   intel_upload_finish(intel);
+
    if (size < INTEL_UPLOAD_SIZE)
       size = INTEL_UPLOAD_SIZE;
 
-   if (intel->upload.bo != NULL)
-      drm_intel_bo_unreference(intel->upload.bo);
-
    intel->upload.bo = drm_intel_bo_alloc(intel->bufmgr, "upload", size, 0);
    intel->upload.offset = 0;
 }
@@ -590,6 +607,8 @@ void intel_upload_data(struct intel_context *intel,
 		       drm_intel_bo **return_bo,
 		       GLuint *return_offset)
 {
+   GLuint asize = ALIGN(size, 64);
+
    if (intel->upload.bo == NULL ||
        intel->upload.offset + size > intel->upload.bo->size) {
       wrap_buffers(intel, size);
@@ -599,11 +618,32 @@ void intel_upload_data(struct intel_context *intel,
    *return_bo = intel->upload.bo;
    *return_offset = intel->upload.offset;
 
-   drm_intel_bo_subdata(intel->upload.bo,
-			intel->upload.offset,
-			size, ptr);
+   if (intel->upload.buffer_len &&
+       intel->upload.buffer_len + asize > sizeof(intel->upload.buffer))
+   {
+      drm_intel_bo_subdata(intel->upload.bo,
+			   intel->upload.buffer_offset,
+			   intel->upload.buffer_len,
+			   intel->upload.buffer);
+      intel->upload.buffer_len = 0;
+   }
 
-   intel->upload.offset += ALIGN(size, 64);
+   if (size < sizeof(intel->upload.buffer))
+   {
+      if (intel->upload.buffer_len == 0)
+	 intel->upload.buffer_offset = intel->upload.offset;
+
+      memcpy(intel->upload.buffer + intel->upload.buffer_len, ptr, size);
+      intel->upload.buffer_len += asize;
+   }
+   else
+   {
+      drm_intel_bo_subdata(intel->upload.bo,
+			   intel->upload.offset,
+			   size, ptr);
+   }
+
+   intel->upload.offset += asize;
 }
 
 void *intel_upload_map(struct intel_context *intel,
@@ -611,6 +651,7 @@ void *intel_upload_map(struct intel_context *intel,
 		       drm_intel_bo **return_bo,
 		       GLuint *return_offset)
 {
+   GLuint asize = ALIGN(size, 64);
    char *ptr;
 
    if (intel->upload.bo == NULL ||
@@ -622,10 +663,32 @@ void *intel_upload_map(struct intel_context *intel,
    *return_bo = intel->upload.bo;
    *return_offset = intel->upload.offset;
 
-   drm_intel_gem_bo_map_gtt(intel->upload.bo);
-   ptr = intel->upload.bo->virtual;
-   ptr += intel->upload.offset;
-   intel->upload.offset += ALIGN(size, 64);
+   if (intel->upload.buffer_len &&
+       intel->upload.buffer_len + asize > sizeof(intel->upload.buffer))
+   {
+      drm_intel_bo_subdata(intel->upload.bo,
+			   intel->upload.buffer_offset,
+			   intel->upload.buffer_len,
+			   intel->upload.buffer);
+      intel->upload.buffer_len = 0;
+   }
+
+   if (size < sizeof(intel->upload.buffer))
+   {
+      if (intel->upload.buffer_len == 0)
+	 intel->upload.buffer_offset = intel->upload.offset;
+
+      ptr = intel->upload.buffer + intel->upload.buffer_len;
+      intel->upload.buffer_len += asize;
+   }
+   else
+   {
+      drm_intel_gem_bo_map_gtt(intel->upload.bo);
+      ptr = intel->upload.bo->virtual;
+      ptr += intel->upload.offset;
+   }
+
+   intel->upload.offset += asize;
 
    return ptr;
 }

src/mesa/drivers/dri/intel/intel_buffer_objects.h

@@ -80,6 +80,8 @@ void *intel_upload_map(struct intel_context *intel,
 		       drm_intel_bo **return_bo,
 		       GLuint *return_offset);
 
+void intel_upload_finish(struct intel_context *intel);
+
 /* Hook the bufferobject implementation into mesa:
  */
 void intelInitBufferObjectFuncs(struct dd_function_table *functions);

src/mesa/drivers/dri/intel/intel_context.h

@@ -190,6 +190,9 @@ struct intel_context
    struct {
       drm_intel_bo *bo;
       GLuint offset;
+      uint16_t buffer_len;
+      uint16_t buffer_offset;
+      char buffer[4096];
    } upload;
 
    GLuint stats_wm;