mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2026-05-05 00:58:05 +02:00
radv: use a compute shader for copying timestamp query results
When the timestamp is not ready (ie. UINT64_MAX), the availabily bit should be zero. The previous code used to copy the timestamp value as the availabily bit and that's completely wrong. Because it's not that simple to emit a conditional with the CP, the driver now uses a compute shader for copying timestamp query results. Fixes dEQP-VK.pipeline.timestamp.misc_tests.reset_query_before_copy. Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com> Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
This commit is contained in:
Samuel Pitoiset
parent
dad80eadb2
commit
9d17d97ee4
2 changed files with 227 additions and 30 deletions
|
|
@ -638,6 +638,7 @@ struct radv_meta_state {
|
|||
VkPipeline occlusion_query_pipeline;
|
||||
VkPipeline pipeline_statistics_query_pipeline;
|
||||
VkPipeline tfb_query_pipeline;
|
||||
VkPipeline timestamp_query_pipeline;
|
||||
} query;
|
||||
|
||||
struct {
|
||||
|
|
|
|||
|
|
@ -741,12 +741,196 @@ build_tfb_query_shader(struct radv_device *device)
|
|||
return b.shader;
|
||||
}
|
||||
|
||||
static nir_shader *
|
||||
build_timestamp_query_shader(struct radv_device *device)
|
||||
{
|
||||
/* the shader this builds is roughly
|
||||
*
|
||||
* uint32_t src_stride = 8;
|
||||
*
|
||||
* location(binding = 0) buffer dst_buf;
|
||||
* location(binding = 1) buffer src_buf;
|
||||
*
|
||||
* void main() {
|
||||
* uint64_t result = 0;
|
||||
* bool available = false;
|
||||
* uint64_t src_offset = src_stride * global_id.x;
|
||||
* uint64_t dst_offset = dst_stride * global_id.x;
|
||||
* uint64_t timestamp = src_buf[src_offset];
|
||||
* if (timestamp != TIMESTAMP_NOT_READY) {
|
||||
* result = timestamp;
|
||||
* available = true;
|
||||
* }
|
||||
* uint32_t result_size = flags & VK_QUERY_RESULT_64_BIT ? 8 : 4;
|
||||
* if ((flags & VK_QUERY_RESULT_PARTIAL_BIT) || available) {
|
||||
* if (flags & VK_QUERY_RESULT_64_BIT) {
|
||||
* dst_buf[dst_offset] = result;
|
||||
* } else {
|
||||
* dst_buf[dst_offset] = (uint32_t)result;
|
||||
* }
|
||||
* }
|
||||
* if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
|
||||
* dst_buf[dst_offset + result_size] = available;
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
nir_builder b;
|
||||
nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL);
|
||||
b.shader->info.name = ralloc_strdup(b.shader, "timestamp_query");
|
||||
b.shader->info.cs.local_size[0] = 64;
|
||||
b.shader->info.cs.local_size[1] = 1;
|
||||
b.shader->info.cs.local_size[2] = 1;
|
||||
|
||||
/* Create and initialize local variables. */
|
||||
nir_variable *result =
|
||||
nir_local_variable_create(b.impl, glsl_uint64_t_type(), "result");
|
||||
nir_variable *available =
|
||||
nir_local_variable_create(b.impl, glsl_bool_type(), "available");
|
||||
|
||||
nir_store_var(&b, result, nir_imm_int64(&b, 0), 0x1);
|
||||
nir_store_var(&b, available, nir_imm_false(&b), 0x1);
|
||||
|
||||
nir_ssa_def *flags = radv_load_push_int(&b, 0, "flags");
|
||||
|
||||
/* Load resources. */
|
||||
nir_intrinsic_instr *dst_buf = nir_intrinsic_instr_create(b.shader,
|
||||
nir_intrinsic_vulkan_resource_index);
|
||||
dst_buf->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
|
||||
dst_buf->num_components = 1;
|
||||
nir_intrinsic_set_desc_set(dst_buf, 0);
|
||||
nir_intrinsic_set_binding(dst_buf, 0);
|
||||
nir_ssa_dest_init(&dst_buf->instr, &dst_buf->dest, dst_buf->num_components, 32, NULL);
|
||||
nir_builder_instr_insert(&b, &dst_buf->instr);
|
||||
|
||||
nir_intrinsic_instr *src_buf = nir_intrinsic_instr_create(b.shader,
|
||||
nir_intrinsic_vulkan_resource_index);
|
||||
src_buf->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0));
|
||||
src_buf->num_components = 1;
|
||||
nir_intrinsic_set_desc_set(src_buf, 0);
|
||||
nir_intrinsic_set_binding(src_buf, 1);
|
||||
nir_ssa_dest_init(&src_buf->instr, &src_buf->dest, src_buf->num_components, 32, NULL);
|
||||
nir_builder_instr_insert(&b, &src_buf->instr);
|
||||
|
||||
/* Compute global ID. */
|
||||
nir_ssa_def *invoc_id = nir_load_local_invocation_id(&b);
|
||||
nir_ssa_def *wg_id = nir_load_work_group_id(&b);
|
||||
nir_ssa_def *block_size = nir_imm_ivec4(&b,
|
||||
b.shader->info.cs.local_size[0],
|
||||
b.shader->info.cs.local_size[1],
|
||||
b.shader->info.cs.local_size[2], 0);
|
||||
nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
|
||||
global_id = nir_channel(&b, global_id, 0); // We only care about x here.
|
||||
|
||||
/* Compute src/dst strides. */
|
||||
nir_ssa_def *input_stride = nir_imm_int(&b, 8);
|
||||
nir_ssa_def *input_base = nir_imul(&b, input_stride, global_id);
|
||||
nir_ssa_def *output_stride = radv_load_push_int(&b, 4, "output_stride");
|
||||
nir_ssa_def *output_base = nir_imul(&b, output_stride, global_id);
|
||||
|
||||
/* Load data from the query pool. */
|
||||
nir_intrinsic_instr *load = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_ssbo);
|
||||
load->src[0] = nir_src_for_ssa(&src_buf->dest.ssa);
|
||||
load->src[1] = nir_src_for_ssa(input_base);
|
||||
nir_ssa_dest_init(&load->instr, &load->dest, 2, 32, NULL);
|
||||
load->num_components = 2;
|
||||
nir_builder_instr_insert(&b, &load->instr);
|
||||
|
||||
/* Pack the timestamp. */
|
||||
nir_ssa_def *timestamp;
|
||||
timestamp = nir_pack_64_2x32(&b, nir_vec2(&b,
|
||||
nir_channel(&b, &load->dest.ssa, 0),
|
||||
nir_channel(&b, &load->dest.ssa, 1)));
|
||||
|
||||
/* Check if result is available. */
|
||||
nir_ssa_def *result_is_available =
|
||||
nir_i2b(&b, nir_ine(&b, timestamp,
|
||||
nir_imm_int64(&b, TIMESTAMP_NOT_READY)));
|
||||
|
||||
/* Only store result if available. */
|
||||
nir_if *available_if = nir_if_create(b.shader);
|
||||
available_if->condition = nir_src_for_ssa(result_is_available);
|
||||
nir_cf_node_insert(b.cursor, &available_if->cf_node);
|
||||
|
||||
b.cursor = nir_after_cf_list(&available_if->then_list);
|
||||
|
||||
nir_store_var(&b, result, timestamp, 0x1);
|
||||
nir_store_var(&b, available, nir_imm_true(&b), 0x1);
|
||||
|
||||
b.cursor = nir_after_cf_node(&available_if->cf_node);
|
||||
|
||||
/* Determine if result is 64 or 32 bit. */
|
||||
nir_ssa_def *result_is_64bit =
|
||||
nir_test_flag(&b, flags, VK_QUERY_RESULT_64_BIT);
|
||||
nir_ssa_def *result_size =
|
||||
nir_bcsel(&b, result_is_64bit, nir_imm_int(&b, 8),
|
||||
nir_imm_int(&b, 4));
|
||||
|
||||
/* Store the result if complete or partial results have been requested. */
|
||||
nir_if *store_if = nir_if_create(b.shader);
|
||||
store_if->condition =
|
||||
nir_src_for_ssa(nir_ior(&b, nir_test_flag(&b, flags, VK_QUERY_RESULT_PARTIAL_BIT),
|
||||
nir_load_var(&b, available)));
|
||||
nir_cf_node_insert(b.cursor, &store_if->cf_node);
|
||||
|
||||
b.cursor = nir_after_cf_list(&store_if->then_list);
|
||||
|
||||
/* Store result. */
|
||||
nir_if *store_64bit_if = nir_if_create(b.shader);
|
||||
store_64bit_if->condition = nir_src_for_ssa(result_is_64bit);
|
||||
nir_cf_node_insert(b.cursor, &store_64bit_if->cf_node);
|
||||
|
||||
b.cursor = nir_after_cf_list(&store_64bit_if->then_list);
|
||||
|
||||
nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_ssbo);
|
||||
store->src[0] = nir_src_for_ssa(nir_load_var(&b, result));
|
||||
store->src[1] = nir_src_for_ssa(&dst_buf->dest.ssa);
|
||||
store->src[2] = nir_src_for_ssa(output_base);
|
||||
nir_intrinsic_set_write_mask(store, 0x1);
|
||||
store->num_components = 1;
|
||||
nir_builder_instr_insert(&b, &store->instr);
|
||||
|
||||
b.cursor = nir_after_cf_list(&store_64bit_if->else_list);
|
||||
|
||||
store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_ssbo);
|
||||
store->src[0] = nir_src_for_ssa(nir_u2u32(&b, nir_load_var(&b, result)));
|
||||
store->src[1] = nir_src_for_ssa(&dst_buf->dest.ssa);
|
||||
store->src[2] = nir_src_for_ssa(output_base);
|
||||
nir_intrinsic_set_write_mask(store, 0x1);
|
||||
store->num_components = 1;
|
||||
nir_builder_instr_insert(&b, &store->instr);
|
||||
|
||||
b.cursor = nir_after_cf_node(&store_64bit_if->cf_node);
|
||||
|
||||
b.cursor = nir_after_cf_node(&store_if->cf_node);
|
||||
|
||||
/* Store the availability bit if requested. */
|
||||
nir_if *availability_if = nir_if_create(b.shader);
|
||||
availability_if->condition =
|
||||
nir_src_for_ssa(nir_test_flag(&b, flags, VK_QUERY_RESULT_WITH_AVAILABILITY_BIT));
|
||||
nir_cf_node_insert(b.cursor, &availability_if->cf_node);
|
||||
|
||||
b.cursor = nir_after_cf_list(&availability_if->then_list);
|
||||
|
||||
store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_ssbo);
|
||||
store->src[0] = nir_src_for_ssa(nir_b2i32(&b, nir_load_var(&b, available)));
|
||||
store->src[1] = nir_src_for_ssa(&dst_buf->dest.ssa);
|
||||
store->src[2] = nir_src_for_ssa(nir_iadd(&b, result_size, output_base));
|
||||
nir_intrinsic_set_write_mask(store, 0x1);
|
||||
store->num_components = 1;
|
||||
nir_builder_instr_insert(&b, &store->instr);
|
||||
|
||||
b.cursor = nir_after_cf_node(&availability_if->cf_node);
|
||||
|
||||
return b.shader;
|
||||
}
|
||||
|
||||
static VkResult radv_device_init_meta_query_state_internal(struct radv_device *device)
|
||||
{
|
||||
VkResult result;
|
||||
struct radv_shader_module occlusion_cs = { .nir = NULL };
|
||||
struct radv_shader_module pipeline_statistics_cs = { .nir = NULL };
|
||||
struct radv_shader_module tfb_cs = { .nir = NULL };
|
||||
struct radv_shader_module timestamp_cs = { .nir = NULL };
|
||||
|
||||
mtx_lock(&device->meta_state.mtx);
|
||||
if (device->meta_state.query.pipeline_statistics_query_pipeline) {
|
||||
|
|
@ -756,6 +940,7 @@ static VkResult radv_device_init_meta_query_state_internal(struct radv_device *d
|
|||
occlusion_cs.nir = build_occlusion_query_shader(device);
|
||||
pipeline_statistics_cs.nir = build_pipeline_statistics_query_shader(device);
|
||||
tfb_cs.nir = build_tfb_query_shader(device);
|
||||
timestamp_cs.nir = build_timestamp_query_shader(device);
|
||||
|
||||
VkDescriptorSetLayoutCreateInfo occlusion_ds_create_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
||||
|
|
@ -864,12 +1049,36 @@ static VkResult radv_device_init_meta_query_state_internal(struct radv_device *d
|
|||
radv_pipeline_cache_to_handle(&device->meta_state.cache),
|
||||
1, &tfb_pipeline_info, NULL,
|
||||
&device->meta_state.query.tfb_query_pipeline);
|
||||
if (result != VK_SUCCESS)
|
||||
goto fail;
|
||||
|
||||
VkPipelineShaderStageCreateInfo timestamp_pipeline_shader_stage = {
|
||||
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
||||
.stage = VK_SHADER_STAGE_COMPUTE_BIT,
|
||||
.module = radv_shader_module_to_handle(×tamp_cs),
|
||||
.pName = "main",
|
||||
.pSpecializationInfo = NULL,
|
||||
};
|
||||
|
||||
VkComputePipelineCreateInfo timestamp_pipeline_info = {
|
||||
.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
||||
.stage = timestamp_pipeline_shader_stage,
|
||||
.flags = 0,
|
||||
.layout = device->meta_state.query.p_layout,
|
||||
};
|
||||
|
||||
result = radv_CreateComputePipelines(radv_device_to_handle(device),
|
||||
radv_pipeline_cache_to_handle(&device->meta_state.cache),
|
||||
1, ×tamp_pipeline_info, NULL,
|
||||
&device->meta_state.query.timestamp_query_pipeline);
|
||||
|
||||
fail:
|
||||
if (result != VK_SUCCESS)
|
||||
radv_device_finish_meta_query_state(device);
|
||||
ralloc_free(occlusion_cs.nir);
|
||||
ralloc_free(pipeline_statistics_cs.nir);
|
||||
ralloc_free(tfb_cs.nir);
|
||||
ralloc_free(timestamp_cs.nir);
|
||||
mtx_unlock(&device->meta_state.mtx);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -899,6 +1108,11 @@ void radv_device_finish_meta_query_state(struct radv_device *device)
|
|||
device->meta_state.query.occlusion_query_pipeline,
|
||||
&device->meta_state.alloc);
|
||||
|
||||
if (device->meta_state.query.timestamp_query_pipeline)
|
||||
radv_DestroyPipeline(radv_device_to_handle(device),
|
||||
device->meta_state.query.timestamp_query_pipeline,
|
||||
&device->meta_state.alloc);
|
||||
|
||||
if (device->meta_state.query.p_layout)
|
||||
radv_DestroyPipelineLayout(radv_device_to_handle(device),
|
||||
device->meta_state.query.p_layout,
|
||||
|
|
@ -1308,7 +1522,6 @@ void radv_CmdCopyQueryPoolResults(
|
|||
RADV_FROM_HANDLE(radv_query_pool, pool, queryPool);
|
||||
RADV_FROM_HANDLE(radv_buffer, dst_buffer, dstBuffer);
|
||||
struct radeon_cmdbuf *cs = cmd_buffer->cs;
|
||||
unsigned elem_size = (flags & VK_QUERY_RESULT_64_BIT) ? 8 : 4;
|
||||
uint64_t va = radv_buffer_get_va(pool->bo);
|
||||
uint64_t dest_va = radv_buffer_get_va(dst_buffer->bo);
|
||||
dest_va += dst_buffer->offset + dstOffset;
|
||||
|
|
@ -1368,14 +1581,13 @@ void radv_CmdCopyQueryPoolResults(
|
|||
pool->availability_offset + 4 * firstQuery);
|
||||
break;
|
||||
case VK_QUERY_TYPE_TIMESTAMP:
|
||||
for(unsigned i = 0; i < queryCount; ++i, dest_va += stride) {
|
||||
unsigned query = firstQuery + i;
|
||||
uint64_t local_src_va = va + query * pool->stride;
|
||||
if (flags & VK_QUERY_RESULT_WAIT_BIT) {
|
||||
for(unsigned i = 0; i < queryCount; ++i, dest_va += stride) {
|
||||
unsigned query = firstQuery + i;
|
||||
uint64_t local_src_va = va + query * pool->stride;
|
||||
|
||||
ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cs, 19);
|
||||
radeon_check_space(cmd_buffer->device->ws, cs, 7);
|
||||
|
||||
|
||||
if (flags & VK_QUERY_RESULT_WAIT_BIT) {
|
||||
/* Wait on the high 32 bits of the timestamp in
|
||||
* case the low part is 0xffffffff.
|
||||
*/
|
||||
|
|
@ -1384,30 +1596,14 @@ void radv_CmdCopyQueryPoolResults(
|
|||
TIMESTAMP_NOT_READY >> 32,
|
||||
0xffffffff);
|
||||
}
|
||||
if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
|
||||
uint64_t avail_dest_va = dest_va + elem_size;
|
||||
|
||||
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
|
||||
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
|
||||
COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM));
|
||||
radeon_emit(cs, local_src_va);
|
||||
radeon_emit(cs, local_src_va >> 32);
|
||||
radeon_emit(cs, avail_dest_va);
|
||||
radeon_emit(cs, avail_dest_va >> 32);
|
||||
}
|
||||
|
||||
radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
|
||||
radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_SRC_MEM) |
|
||||
COPY_DATA_DST_SEL(COPY_DATA_DST_MEM_GRBM) |
|
||||
((flags & VK_QUERY_RESULT_64_BIT) ? COPY_DATA_COUNT_SEL : 0));
|
||||
radeon_emit(cs, local_src_va);
|
||||
radeon_emit(cs, local_src_va >> 32);
|
||||
radeon_emit(cs, dest_va);
|
||||
radeon_emit(cs, dest_va >> 32);
|
||||
|
||||
|
||||
assert(cs->cdw <= cdw_max);
|
||||
}
|
||||
|
||||
radv_query_shader(cmd_buffer, &cmd_buffer->device->meta_state.query.timestamp_query_pipeline,
|
||||
pool->bo, dst_buffer->bo,
|
||||
firstQuery * pool->stride,
|
||||
dst_buffer->offset + dstOffset,
|
||||
pool->stride, stride,
|
||||
queryCount, flags, 0, 0);
|
||||
break;
|
||||
case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
|
||||
if (flags & VK_QUERY_RESULT_WAIT_BIT) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue