diff --git a/src/mesa/drivers/dri/i965/brw_blorp.c b/src/mesa/drivers/dri/i965/brw_blorp.c index 70c8e0d5816..d7a2cb25c67 100644 --- a/src/mesa/drivers/dri/i965/brw_blorp.c +++ b/src/mesa/drivers/dri/i965/brw_blorp.c @@ -816,7 +816,8 @@ blorp_get_client_bo(struct brw_context *brw, * data which we need to copy into a BO. */ struct brw_bo *bo = - brw_bo_alloc(brw->bufmgr, "tmp_tex_subimage_src", size); + brw_bo_alloc(brw->bufmgr, "tmp_tex_subimage_src", size, + BRW_MEMZONE_OTHER); if (bo == NULL) { perf_debug("intel_texsubimage: temp bo creation failed: size = %u\n", size); diff --git a/src/mesa/drivers/dri/i965/brw_bufmgr.c b/src/mesa/drivers/dri/i965/brw_bufmgr.c index 66f30a1637f..66828f319be 100644 --- a/src/mesa/drivers/dri/i965/brw_bufmgr.c +++ b/src/mesa/drivers/dri/i965/brw_bufmgr.c @@ -268,6 +268,7 @@ static struct brw_bo * bo_alloc_internal(struct brw_bufmgr *bufmgr, const char *name, uint64_t size, + enum brw_memory_zone memzone, unsigned flags, uint32_t tiling_mode, uint32_t stride) @@ -426,23 +427,27 @@ err: struct brw_bo * brw_bo_alloc(struct brw_bufmgr *bufmgr, - const char *name, uint64_t size) + const char *name, uint64_t size, + enum brw_memory_zone memzone) { - return bo_alloc_internal(bufmgr, name, size, 0, I915_TILING_NONE, 0); + return bo_alloc_internal(bufmgr, name, size, memzone, + 0, I915_TILING_NONE, 0); } struct brw_bo * brw_bo_alloc_tiled(struct brw_bufmgr *bufmgr, const char *name, - uint64_t size, uint32_t tiling_mode, uint32_t pitch, + uint64_t size, enum brw_memory_zone memzone, + uint32_t tiling_mode, uint32_t pitch, unsigned flags) { - return bo_alloc_internal(bufmgr, name, size, flags, tiling_mode, pitch); + return bo_alloc_internal(bufmgr, name, size, memzone, + flags, tiling_mode, pitch); } struct brw_bo * brw_bo_alloc_tiled_2d(struct brw_bufmgr *bufmgr, const char *name, - int x, int y, int cpp, uint32_t tiling, - uint32_t *pitch, unsigned flags) + int x, int y, int cpp, enum brw_memory_zone memzone, + uint32_t tiling, uint32_t *pitch, unsigned flags) { uint64_t size; uint32_t stride; @@ -477,7 +482,8 @@ brw_bo_alloc_tiled_2d(struct brw_bufmgr *bufmgr, const char *name, if (tiling == I915_TILING_NONE) stride = 0; - return bo_alloc_internal(bufmgr, name, size, flags, tiling, stride); + return bo_alloc_internal(bufmgr, name, size, flags, + memzone, tiling, stride); } /** diff --git a/src/mesa/drivers/dri/i965/brw_bufmgr.h b/src/mesa/drivers/dri/i965/brw_bufmgr.h index 68f5e0c2c85..9ad129744b2 100644 --- a/src/mesa/drivers/dri/i965/brw_bufmgr.h +++ b/src/mesa/drivers/dri/i965/brw_bufmgr.h @@ -47,6 +47,42 @@ extern "C" { struct gen_device_info; struct brw_context; +/** + * Memory zones. When allocating a buffer, you can request that it is + * placed into a specific region of the virtual address space (PPGTT). + * + * Most buffers can go anywhere (BRW_MEMZONE_OTHER). Some buffers are + * accessed via an offset from a base address. STATE_BASE_ADDRESS has + * a maximum 4GB size for each region, so we need to restrict those + * buffers to be within 4GB of the base. Each memory zone corresponds + * to a particular base address. + * + * Currently, i965 partitions the address space into two regions: + * + * - Low 4GB + * - Full 48-bit address space + * + * Eventually, we hope to carve out 4GB of VMA for each base address. + */ +enum brw_memory_zone { + BRW_MEMZONE_LOW_4G, + BRW_MEMZONE_OTHER, + + /* Shaders - Instruction State Base Address */ + BRW_MEMZONE_SHADER = BRW_MEMZONE_LOW_4G, + + /* Scratch - General State Base Address */ + BRW_MEMZONE_SCRATCH = BRW_MEMZONE_LOW_4G, + + /* Surface State Base Address */ + BRW_MEMZONE_SURFACE = BRW_MEMZONE_LOW_4G, + + /* Dynamic State Base Address */ + BRW_MEMZONE_DYNAMIC = BRW_MEMZONE_LOW_4G, +}; + +#define BRW_MEMZONE_COUNT (BRW_MEMZONE_OTHER + 1) + struct brw_bo { /** * Size in bytes of the buffer object. @@ -168,7 +204,7 @@ struct brw_bo { * using brw_bo_map() to be used by the CPU. */ struct brw_bo *brw_bo_alloc(struct brw_bufmgr *bufmgr, const char *name, - uint64_t size); + uint64_t size, enum brw_memory_zone memzone); /** * Allocate a tiled buffer object. @@ -184,6 +220,7 @@ struct brw_bo *brw_bo_alloc(struct brw_bufmgr *bufmgr, const char *name, struct brw_bo *brw_bo_alloc_tiled(struct brw_bufmgr *bufmgr, const char *name, uint64_t size, + enum brw_memory_zone memzone, uint32_t tiling_mode, uint32_t pitch, unsigned flags); @@ -206,6 +243,7 @@ struct brw_bo *brw_bo_alloc_tiled(struct brw_bufmgr *bufmgr, struct brw_bo *brw_bo_alloc_tiled_2d(struct brw_bufmgr *bufmgr, const char *name, int x, int y, int cpp, + enum brw_memory_zone memzone, uint32_t tiling_mode, uint32_t *pitch, unsigned flags); diff --git a/src/mesa/drivers/dri/i965/brw_context.h b/src/mesa/drivers/dri/i965/brw_context.h index 7dcbd040f01..0844400bc53 100644 --- a/src/mesa/drivers/dri/i965/brw_context.h +++ b/src/mesa/drivers/dri/i965/brw_context.h @@ -479,6 +479,7 @@ struct brw_growing_bo { struct brw_bo *partial_bo; uint32_t *partial_bo_map; unsigned partial_bytes; + enum brw_memory_zone memzone; }; struct intel_batchbuffer { diff --git a/src/mesa/drivers/dri/i965/brw_performance_query.c b/src/mesa/drivers/dri/i965/brw_performance_query.c index 77d23133ad4..d45529fc0c7 100644 --- a/src/mesa/drivers/dri/i965/brw_performance_query.c +++ b/src/mesa/drivers/dri/i965/brw_performance_query.c @@ -1181,7 +1181,8 @@ brw_begin_perf_query(struct gl_context *ctx, } obj->oa.bo = - brw_bo_alloc(brw->bufmgr, "perf. query OA MI_RPC bo", MI_RPC_BO_SIZE); + brw_bo_alloc(brw->bufmgr, "perf. query OA MI_RPC bo", MI_RPC_BO_SIZE, + BRW_MEMZONE_OTHER); #ifdef DEBUG /* Pre-filling the BO helps debug whether writes landed. */ void *map = brw_bo_map(brw, obj->oa.bo, MAP_WRITE); @@ -1240,7 +1241,7 @@ brw_begin_perf_query(struct gl_context *ctx, obj->pipeline_stats.bo = brw_bo_alloc(brw->bufmgr, "perf. query pipeline stats bo", - STATS_BO_SIZE); + STATS_BO_SIZE, BRW_MEMZONE_OTHER); /* Take starting snapshots. */ snapshot_statistics_registers(brw, obj, 0); diff --git a/src/mesa/drivers/dri/i965/brw_pipe_control.c b/src/mesa/drivers/dri/i965/brw_pipe_control.c index e31d625ddba..cbd2853f58c 100644 --- a/src/mesa/drivers/dri/i965/brw_pipe_control.c +++ b/src/mesa/drivers/dri/i965/brw_pipe_control.c @@ -580,7 +580,8 @@ brw_init_pipe_control(struct brw_context *brw, * the gen6 workaround because it involves actually writing to * the buffer, and the kernel doesn't let us write to the batch. */ - brw->workaround_bo = brw_bo_alloc(brw->bufmgr, "workaround", 4096); + brw->workaround_bo = brw_bo_alloc(brw->bufmgr, "workaround", 4096, + BRW_MEMZONE_OTHER); if (brw->workaround_bo == NULL) return -ENOMEM; diff --git a/src/mesa/drivers/dri/i965/brw_program.c b/src/mesa/drivers/dri/i965/brw_program.c index fc77926d6e0..446aaee0673 100644 --- a/src/mesa/drivers/dri/i965/brw_program.c +++ b/src/mesa/drivers/dri/i965/brw_program.c @@ -348,7 +348,8 @@ brw_get_scratch_bo(struct brw_context *brw, } if (!old_bo) { - *scratch_bo = brw_bo_alloc(brw->bufmgr, "scratch bo", size); + *scratch_bo = + brw_bo_alloc(brw->bufmgr, "scratch bo", size, BRW_MEMZONE_SCRATCH); } } @@ -443,7 +444,7 @@ brw_alloc_stage_scratch(struct brw_context *brw, stage_state->scratch_bo = brw_bo_alloc(brw->bufmgr, "shader scratch space", - per_thread_size * thread_count); + per_thread_size * thread_count, BRW_MEMZONE_SCRATCH); } void brwInitFragProgFuncs( struct dd_function_table *functions ) @@ -472,7 +473,8 @@ brw_init_shader_time(struct brw_context *brw) const int max_entries = 2048; brw->shader_time.bo = brw_bo_alloc(brw->bufmgr, "shader time", - max_entries * BRW_SHADER_TIME_STRIDE * 3); + max_entries * BRW_SHADER_TIME_STRIDE * 3, + BRW_MEMZONE_OTHER); brw->shader_time.names = rzalloc_array(brw, const char *, max_entries); brw->shader_time.ids = rzalloc_array(brw, int, max_entries); brw->shader_time.types = rzalloc_array(brw, enum shader_time_shader_type, diff --git a/src/mesa/drivers/dri/i965/brw_program_cache.c b/src/mesa/drivers/dri/i965/brw_program_cache.c index 78159288af0..31f6f1acbf8 100644 --- a/src/mesa/drivers/dri/i965/brw_program_cache.c +++ b/src/mesa/drivers/dri/i965/brw_program_cache.c @@ -221,7 +221,8 @@ brw_cache_new_bo(struct brw_cache *cache, uint32_t new_size) perf_debug("Copying to larger program cache: %u kB -> %u kB\n", (unsigned) cache->bo->size / 1024, new_size / 1024); - new_bo = brw_bo_alloc(brw->bufmgr, "program cache", new_size); + new_bo = brw_bo_alloc(brw->bufmgr, "program cache", new_size, + BRW_MEMZONE_SHADER); if (can_do_exec_capture(brw->screen)) new_bo->kflags |= EXEC_OBJECT_CAPTURE; @@ -388,7 +389,8 @@ brw_init_caches(struct brw_context *brw) cache->items = calloc(cache->size, sizeof(struct brw_cache_item *)); - cache->bo = brw_bo_alloc(brw->bufmgr, "program cache", 16384); + cache->bo = brw_bo_alloc(brw->bufmgr, "program cache", 16384, + BRW_MEMZONE_SHADER); if (can_do_exec_capture(brw->screen)) cache->bo->kflags |= EXEC_OBJECT_CAPTURE; diff --git a/src/mesa/drivers/dri/i965/brw_queryobj.c b/src/mesa/drivers/dri/i965/brw_queryobj.c index 0015a4ee9d6..bc4b8c43e7b 100644 --- a/src/mesa/drivers/dri/i965/brw_queryobj.c +++ b/src/mesa/drivers/dri/i965/brw_queryobj.c @@ -287,7 +287,8 @@ brw_begin_query(struct gl_context *ctx, struct gl_query_object *q) * the system was doing other work, such as running other applications. */ brw_bo_unreference(query->bo); - query->bo = brw_bo_alloc(brw->bufmgr, "timer query", 4096); + query->bo = + brw_bo_alloc(brw->bufmgr, "timer query", 4096, BRW_MEMZONE_OTHER); brw_write_timestamp(brw, query->bo, 0); break; @@ -450,7 +451,7 @@ ensure_bo_has_space(struct gl_context *ctx, struct brw_query_object *query) brw_queryobj_get_results(ctx, query); } - query->bo = brw_bo_alloc(brw->bufmgr, "query", 4096); + query->bo = brw_bo_alloc(brw->bufmgr, "query", 4096, BRW_MEMZONE_OTHER); query->last_index = 0; } } @@ -530,7 +531,8 @@ brw_query_counter(struct gl_context *ctx, struct gl_query_object *q) assert(q->Target == GL_TIMESTAMP); brw_bo_unreference(query->bo); - query->bo = brw_bo_alloc(brw->bufmgr, "timestamp query", 4096); + query->bo = + brw_bo_alloc(brw->bufmgr, "timestamp query", 4096, BRW_MEMZONE_OTHER); brw_write_timestamp(brw, query->bo, 0); query->flushed = false; diff --git a/src/mesa/drivers/dri/i965/gen6_queryobj.c b/src/mesa/drivers/dri/i965/gen6_queryobj.c index 75060d47210..ce9bb474e18 100644 --- a/src/mesa/drivers/dri/i965/gen6_queryobj.c +++ b/src/mesa/drivers/dri/i965/gen6_queryobj.c @@ -329,7 +329,8 @@ gen6_begin_query(struct gl_context *ctx, struct gl_query_object *q) /* Since we're starting a new query, we need to throw away old results. */ brw_bo_unreference(query->bo); - query->bo = brw_bo_alloc(brw->bufmgr, "query results", 4096); + query->bo = + brw_bo_alloc(brw->bufmgr, "query results", 4096, BRW_MEMZONE_OTHER); /* For ARB_query_buffer_object: The result is not available */ set_query_availability(brw, query, false); diff --git a/src/mesa/drivers/dri/i965/gen6_sol.c b/src/mesa/drivers/dri/i965/gen6_sol.c index 0c830c7342c..a2d2606a35d 100644 --- a/src/mesa/drivers/dri/i965/gen6_sol.c +++ b/src/mesa/drivers/dri/i965/gen6_sol.c @@ -195,9 +195,11 @@ brw_new_transform_feedback(struct gl_context *ctx, GLuint name) _mesa_init_transform_feedback_object(&brw_obj->base, name); brw_obj->offset_bo = - brw_bo_alloc(brw->bufmgr, "transform feedback offsets", 16); + brw_bo_alloc(brw->bufmgr, "transform feedback offsets", 16, + BRW_MEMZONE_OTHER); brw_obj->prim_count_bo = - brw_bo_alloc(brw->bufmgr, "xfb primitive counts", 16384); + brw_bo_alloc(brw->bufmgr, "xfb primitive counts", 16384, + BRW_MEMZONE_OTHER); return &brw_obj->base; } diff --git a/src/mesa/drivers/dri/i965/intel_batchbuffer.c b/src/mesa/drivers/dri/i965/intel_batchbuffer.c index 8c5fd50123a..fe1ea02ca41 100644 --- a/src/mesa/drivers/dri/i965/intel_batchbuffer.c +++ b/src/mesa/drivers/dri/i965/intel_batchbuffer.c @@ -225,17 +225,19 @@ add_exec_bo(struct intel_batchbuffer *batch, struct brw_bo *bo) static void recreate_growing_buffer(struct brw_context *brw, struct brw_growing_bo *grow, - const char *name, unsigned size) + const char *name, unsigned size, + enum brw_memory_zone memzone) { struct intel_screen *screen = brw->screen; struct intel_batchbuffer *batch = &brw->batch; struct brw_bufmgr *bufmgr = screen->bufmgr; - grow->bo = brw_bo_alloc(bufmgr, name, size); + grow->bo = brw_bo_alloc(bufmgr, name, size, memzone); grow->bo->kflags |= can_do_exec_capture(screen) ? EXEC_OBJECT_CAPTURE : 0; grow->partial_bo = NULL; grow->partial_bo_map = NULL; grow->partial_bytes = 0; + grow->memzone = memzone; if (batch->use_shadow_copy) grow->map = realloc(grow->map, grow->bo->size); @@ -254,10 +256,12 @@ intel_batchbuffer_reset(struct brw_context *brw) } batch->last_bo = batch->batch.bo; - recreate_growing_buffer(brw, &batch->batch, "batchbuffer", BATCH_SZ); + recreate_growing_buffer(brw, &batch->batch, "batchbuffer", BATCH_SZ, + BRW_MEMZONE_OTHER); batch->map_next = batch->batch.map; - recreate_growing_buffer(brw, &batch->state, "statebuffer", STATE_SZ); + recreate_growing_buffer(brw, &batch->state, "statebuffer", STATE_SZ, + BRW_MEMZONE_DYNAMIC); /* Avoid making 0 a valid state offset - otherwise the decoder will try * and decode data when we use offset 0 as a null pointer. @@ -396,7 +400,8 @@ grow_buffer(struct brw_context *brw, finish_growing_bos(grow); } - struct brw_bo *new_bo = brw_bo_alloc(bufmgr, bo->name, new_size); + struct brw_bo *new_bo = + brw_bo_alloc(bufmgr, bo->name, new_size, grow->memzone); /* Copy existing data to the new larger buffer */ grow->partial_bo_map = grow->map; diff --git a/src/mesa/drivers/dri/i965/intel_buffer_objects.c b/src/mesa/drivers/dri/i965/intel_buffer_objects.c index 9deb5e7434a..452e6d33c07 100644 --- a/src/mesa/drivers/dri/i965/intel_buffer_objects.c +++ b/src/mesa/drivers/dri/i965/intel_buffer_objects.c @@ -96,7 +96,8 @@ alloc_buffer_object(struct brw_context *brw, */ size += 64 * 32; /* max read length of 64 256-bit units */ } - intel_obj->buffer = brw_bo_alloc(brw->bufmgr, "bufferobj", size); + intel_obj->buffer = + brw_bo_alloc(brw->bufmgr, "bufferobj", size, BRW_MEMZONE_OTHER); /* the buffer might be bound as a uniform buffer, need to update it */ @@ -290,7 +291,7 @@ brw_buffer_subdata(struct gl_context *ctx, intel_obj->valid_data_start, intel_obj->valid_data_end); struct brw_bo *temp_bo = - brw_bo_alloc(brw->bufmgr, "subdata temp", size); + brw_bo_alloc(brw->bufmgr, "subdata temp", size, BRW_MEMZONE_OTHER); brw_bo_subdata(temp_bo, 0, size, data); @@ -462,7 +463,8 @@ brw_map_buffer_range(struct gl_context *ctx, intel_obj->map_extra[index] = (uintptr_t) offset % alignment; intel_obj->range_map_bo[index] = brw_bo_alloc(brw->bufmgr, "BO blit temp", - length + intel_obj->map_extra[index]); + length + intel_obj->map_extra[index], + BRW_MEMZONE_OTHER); void *map = brw_bo_map(brw, intel_obj->range_map_bo[index], access); obj->Mappings[index].Pointer = map + intel_obj->map_extra[index]; return obj->Mappings[index].Pointer; diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c index 5f19e23b018..f3c171eb7f3 100644 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c @@ -627,6 +627,7 @@ make_surface(struct brw_context *brw, GLenum target, mesa_format format, if (!bo) { mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree", mt->surf.size, + BRW_MEMZONE_OTHER, isl_tiling_to_i915_tiling( mt->surf.tiling), mt->surf.row_pitch, alloc_flags); @@ -990,7 +991,8 @@ create_ccs_buf_for_image(struct brw_context *brw, mt->aux_buf->clear_color_bo = brw_bo_alloc_tiled(brw->bufmgr, "clear_color_bo", brw->isl_dev.ss.clear_color_state_size, - I915_TILING_NONE, 0, BO_ALLOC_ZEROED); + BRW_MEMZONE_OTHER, I915_TILING_NONE, 0, + BO_ALLOC_ZEROED); if (!mt->aux_buf->clear_color_bo) { free(mt->aux_buf); mt->aux_buf = NULL; @@ -1701,8 +1703,8 @@ intel_alloc_aux_buffer(struct brw_context *brw, * trying to recalculate based on different format block sizes. */ buf->bo = brw_bo_alloc_tiled(brw->bufmgr, "aux-miptree", size, - I915_TILING_Y, aux_surf->row_pitch, - alloc_flags); + BRW_MEMZONE_OTHER, I915_TILING_Y, + aux_surf->row_pitch, alloc_flags); if (!buf->bo) { free(buf); return NULL; diff --git a/src/mesa/drivers/dri/i965/intel_screen.c b/src/mesa/drivers/dri/i965/intel_screen.c index 409f763b640..4b0a573eaf3 100644 --- a/src/mesa/drivers/dri/i965/intel_screen.c +++ b/src/mesa/drivers/dri/i965/intel_screen.c @@ -745,6 +745,7 @@ intel_create_image_common(__DRIscreen *dri_screen, */ image->bo = brw_bo_alloc_tiled(screen->bufmgr, "image", surf.size + aux_surf.size, + BRW_MEMZONE_OTHER, isl_tiling_to_i915_tiling(mod_info->tiling), surf.row_pitch, BO_ALLOC_ZEROED); if (image->bo == NULL) { @@ -1834,7 +1835,7 @@ intel_detect_swizzling(struct intel_screen *screen) uint32_t swizzle_mode = 0; struct brw_bo *buffer = brw_bo_alloc_tiled(screen->bufmgr, "swizzle test", 32768, - tiling, 512, 0); + BRW_MEMZONE_OTHER, tiling, 512, 0); if (buffer == NULL) return false; @@ -1909,11 +1910,11 @@ intel_detect_pipelined_register(struct intel_screen *screen, bool success = false; /* Create a zero'ed temporary buffer for reading our results */ - results = brw_bo_alloc(screen->bufmgr, "registers", 4096); + results = brw_bo_alloc(screen->bufmgr, "registers", 4096, BRW_MEMZONE_OTHER); if (results == NULL) goto err; - bo = brw_bo_alloc(screen->bufmgr, "batchbuffer", 4096); + bo = brw_bo_alloc(screen->bufmgr, "batchbuffer", 4096, BRW_MEMZONE_OTHER); if (bo == NULL) goto err_results; @@ -2721,6 +2722,7 @@ intelAllocateBuffer(__DRIscreen *dri_screen, width, height, cpp, + BRW_MEMZONE_OTHER, I915_TILING_X, &pitch, BO_ALLOC_BUSY); diff --git a/src/mesa/drivers/dri/i965/intel_upload.c b/src/mesa/drivers/dri/i965/intel_upload.c index f165a7b4322..d81ae43e8a0 100644 --- a/src/mesa/drivers/dri/i965/intel_upload.c +++ b/src/mesa/drivers/dri/i965/intel_upload.c @@ -86,7 +86,8 @@ brw_upload_space(struct brw_uploader *upload, assert((upload->bo == NULL) == (upload->map == NULL)); if (!upload->bo) { upload->bo = brw_bo_alloc(upload->bufmgr, "streamed data", - MAX2(upload->default_size, size)); + MAX2(upload->default_size, size), + BRW_MEMZONE_OTHER); upload->map = brw_bo_map(NULL, upload->bo, MAP_READ | MAP_WRITE | MAP_PERSISTENT | MAP_ASYNC);