diff --git a/src/amd/vulkan/meta/radv_meta_buffer.c b/src/amd/vulkan/meta/radv_meta_buffer.c index 9db54b01d9f..3835c1f368d 100644 --- a/src/amd/vulkan/meta/radv_meta_buffer.c +++ b/src/amd/vulkan/meta/radv_meta_buffer.c @@ -266,7 +266,7 @@ radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *im flush_bits = RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, NULL); } else if (size) radv_cp_dma_clear_buffer(cmd_buffer, va, size, value); diff --git a/src/amd/vulkan/meta/radv_meta_clear.c b/src/amd/vulkan/meta/radv_meta_clear.c index 6f211a85dba..d2f38e150af 100644 --- a/src/amd/vulkan/meta/radv_meta_clear.c +++ b/src/amd/vulkan/meta/radv_meta_clear.c @@ -692,7 +692,7 @@ clear_htile_mask(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *im radv_meta_restore(&saved_state, cmd_buffer); return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, NULL); } static uint32_t @@ -854,8 +854,8 @@ radv_fast_clear_depth(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag if (pre_flush) { enum radv_cmd_flush_bits bits = radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, iview->image, &range) | - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, 0, iview->image, &range) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, 0, iview->image, &range); cmd_buffer->state.flush_bits |= bits & ~*pre_flush; *pre_flush |= cmd_buffer->state.flush_bits; @@ -1208,7 +1208,7 @@ radv_clear_dcc_comp_to_single(struct radv_cmd_buffer *cmd_buffer, struct radv_im radv_meta_restore(&saved_state, cmd_buffer); return RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, NULL); } uint32_t @@ -1579,8 +1579,9 @@ radv_fast_clear_color(struct radv_cmd_buffer *cmd_buffer, const struct radv_imag VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); if (pre_flush) { - enum radv_cmd_flush_bits bits = radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, iview->image, NULL); + enum radv_cmd_flush_bits bits = + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, iview->image, NULL); cmd_buffer->state.flush_bits |= bits & ~*pre_flush; *pre_flush |= cmd_buffer->state.flush_bits; } diff --git a/src/amd/vulkan/meta/radv_meta_copy.c b/src/amd/vulkan/meta/radv_meta_copy.c index c2160fe6e9f..7362e3601cb 100644 --- a/src/amd/vulkan/meta/radv_meta_copy.c +++ b/src/amd/vulkan/meta/radv_meta_copy.c @@ -234,9 +234,9 @@ radv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer, const VkCopyBufferToIm if (radv_is_format_emulated(pdev, dst_image->vk.format)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_TRANSFER_WRITE_BIT, dst_image, NULL) | + VK_ACCESS_TRANSFER_WRITE_BIT, 0, dst_image, NULL) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_TRANSFER_READ_BIT, dst_image, NULL); + VK_ACCESS_TRANSFER_READ_BIT, 0, dst_image, NULL); const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout; for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) { @@ -614,9 +614,9 @@ radv_CmdCopyImage2(VkCommandBuffer commandBuffer, const VkCopyImageInfo2 *pCopyI if (radv_is_format_emulated(pdev, dst_image->vk.format)) { cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_TRANSFER_WRITE_BIT, dst_image, NULL) | + VK_ACCESS_TRANSFER_WRITE_BIT, 0, dst_image, NULL) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_TRANSFER_READ_BIT, dst_image, NULL); + VK_ACCESS_TRANSFER_READ_BIT, 0, dst_image, NULL); const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout; for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) { diff --git a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c index e5f36ccbff0..b7f2e10bfc8 100644 --- a/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c +++ b/src/amd/vulkan/meta/radv_meta_copy_vrs_htile.c @@ -181,8 +181,9 @@ radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view * cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL, NULL) | - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL, NULL); + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, 0, NULL, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, 0, NULL, + NULL); radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS); @@ -233,5 +234,5 @@ radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view * cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, NULL, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL); } diff --git a/src/amd/vulkan/meta/radv_meta_dcc_retile.c b/src/amd/vulkan/meta/radv_meta_dcc_retile.c index 470345f4295..021a6e95cde 100644 --- a/src/amd/vulkan/meta/radv_meta_dcc_retile.c +++ b/src/amd/vulkan/meta/radv_meta_dcc_retile.c @@ -169,8 +169,8 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) return; } - state->flush_bits |= - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, image, NULL); + state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_SHADER_READ_BIT, 0, image, NULL); radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS); @@ -245,5 +245,5 @@ radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image) state->flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, NULL); } diff --git a/src/amd/vulkan/meta/radv_meta_decompress.c b/src/amd/vulkan/meta/radv_meta_decompress.c index 3b9398b398e..40162ec4906 100644 --- a/src/amd/vulkan/meta/radv_meta_decompress.c +++ b/src/amd/vulkan/meta/radv_meta_decompress.c @@ -473,7 +473,7 @@ radv_expand_depth_stencil_compute(struct radv_cmd_buffer *cmd_buffer, struct rad cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, subresourceRange); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, subresourceRange); /* Initialize the HTILE metadata as "fully expanded". */ uint32_t htile_value = radv_get_htile_initial_value(device, image); diff --git a/src/amd/vulkan/meta/radv_meta_fast_clear.c b/src/amd/vulkan/meta/radv_meta_fast_clear.c index b83e386908a..dd5071ef33d 100644 --- a/src/amd/vulkan/meta/radv_meta_fast_clear.c +++ b/src/amd/vulkan/meta/radv_meta_fast_clear.c @@ -312,13 +312,13 @@ radv_process_color_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_i if (flush_cb) cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, image, range); + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, 0, image, range); radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0); if (flush_cb) cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image, range); + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, image, range); radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer)); @@ -505,7 +505,7 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag } cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_READ_BIT, image, subresourceRange); + VK_ACCESS_2_SHADER_READ_BIT, 0, image, subresourceRange); radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_DESCRIPTORS | RADV_META_SAVE_COMPUTE_PIPELINE); @@ -591,7 +591,7 @@ radv_decompress_dcc_compute(struct radv_cmd_buffer *cmd_buffer, struct radv_imag cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, subresourceRange); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, subresourceRange); /* Initialize the DCC metadata as "fully expanded". */ cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, image, subresourceRange, 0xffffffff); diff --git a/src/amd/vulkan/meta/radv_meta_fmask_expand.c b/src/amd/vulkan/meta/radv_meta_fmask_expand.c index 86bd06eee8c..5f76c0e225c 100644 --- a/src/amd/vulkan/meta/radv_meta_fmask_expand.c +++ b/src/amd/vulkan/meta/radv_meta_fmask_expand.c @@ -167,7 +167,7 @@ radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_ const VkImageSubresourceRange range = vk_image_view_subresource_range(&iview.vk); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_SHADER_READ_BIT, image, &range); + VK_ACCESS_2_SHADER_READ_BIT, 0, image, &range); radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 2, (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, @@ -200,7 +200,7 @@ radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_ cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, image, &range); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, image, &range); /* Re-initialize FMASK in fully expanded mode. */ cmd_buffer->state.flush_bits |= radv_init_fmask(cmd_buffer, image, subresourceRange); diff --git a/src/amd/vulkan/meta/radv_meta_resolve.c b/src/amd/vulkan/meta/radv_meta_resolve.c index 5019e19c577..ecd2124ccf6 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve.c +++ b/src/amd/vulkan/meta/radv_meta_resolve.c @@ -164,15 +164,15 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_im } cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, src_image, NULL) | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, src_image, NULL) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image, NULL); + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, 0, src_image, NULL); radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image, NULL); + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, dst_image, NULL); } enum radv_resolve_method { diff --git a/src/amd/vulkan/meta/radv_meta_resolve_cs.c b/src/amd/vulkan/meta/radv_meta_resolve_cs.c index 5b2fb74816a..8f00accaffd 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_cs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_cs.c @@ -516,7 +516,7 @@ radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, NULL, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL); } void @@ -537,8 +537,9 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm */ cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, NULL, NULL) | - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL, NULL); + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, 0, NULL, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, 0, NULL, + NULL); struct radv_image_view *src_iview = render->ds_att.iview; VkImageLayout src_layout = @@ -603,7 +604,7 @@ radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkIm cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, NULL, NULL); + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL); uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf); diff --git a/src/amd/vulkan/meta/radv_meta_resolve_fs.c b/src/amd/vulkan/meta/radv_meta_resolve_fs.c index ecb74b979d1..06404415ccf 100644 --- a/src/amd/vulkan/meta/radv_meta_resolve_fs.c +++ b/src/amd/vulkan/meta/radv_meta_resolve_fs.c @@ -447,10 +447,10 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_ivi const VkImageSubresourceRange dst_range = vk_image_view_subresource_range(&dst_iview->vk); cmd_buffer->state.flush_bits |= - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, 0, src_iview->image, &src_range) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, dst_iview->image, &dst_range); + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, 0, dst_iview->image, &dst_range); unsigned push_constants[2] = { src_offset->x - dst_offset->x, @@ -464,7 +464,7 @@ emit_resolve(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_ivi radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_iview->image, &dst_range); + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, dst_iview->image, &dst_range); } static void diff --git a/src/amd/vulkan/radv_acceleration_structure.c b/src/amd/vulkan/radv_acceleration_structure.c index 7453f9a5971..271359be2b5 100644 --- a/src/amd/vulkan/radv_acceleration_structure.c +++ b/src/amd/vulkan/radv_acceleration_structure.c @@ -469,9 +469,9 @@ radv_init_header_bind_pipeline(VkCommandBuffer commandBuffer, uint32_t key) /* Wait for encoding to finish. */ cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, NULL, NULL) | + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_READ_BIT, NULL, NULL); + VK_ACCESS_2_SHADER_READ_BIT, 0, NULL, NULL); device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.accel_struct_build.header_pipeline); @@ -580,9 +580,9 @@ radv_update_bind_pipeline(VkCommandBuffer commandBuffer) /* Wait for update scratch initialization to finish.. */ cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_WRITE_BIT, NULL, NULL) | + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL) | radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - VK_ACCESS_2_SHADER_READ_BIT, NULL, NULL); + VK_ACCESS_2_SHADER_READ_BIT, 0, NULL, NULL); device->vk.dispatch_table.CmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, device->meta_state.accel_struct_build.update_pipeline); @@ -828,7 +828,7 @@ radv_CmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuffer, const VkCopy sizeof(consts), &consts); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, - VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL, NULL); + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, 0, NULL, NULL); radv_indirect_dispatch( cmd_buffer, src_buffer->bo, @@ -934,7 +934,7 @@ radv_CmdCopyAccelerationStructureToMemoryKHR(VkCommandBuffer commandBuffer, sizeof(consts), &consts); cmd_buffer->state.flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, - VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, NULL, NULL); + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, 0, NULL, NULL); radv_indirect_dispatch( cmd_buffer, src_buffer->bo, diff --git a/src/amd/vulkan/radv_cmd_buffer.c b/src/amd/vulkan/radv_cmd_buffer.c index 02bf1f0b9c6..f4f57fdd966 100644 --- a/src/amd/vulkan/radv_cmd_buffer.c +++ b/src/amd/vulkan/radv_cmd_buffer.c @@ -6584,7 +6584,8 @@ can_skip_buffer_l2_flushes(struct radv_device *device) enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stages, VkAccessFlags2 src_flags, - const struct radv_image *image, const VkImageSubresourceRange *range) + VkAccessFlags3KHR src3_flags, const struct radv_image *image, + const VkImageSubresourceRange *range) { const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); @@ -6654,7 +6655,8 @@ radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 dst_stages, VkAccessFlags2 dst_flags, - const struct radv_image *image, const VkImageSubresourceRange *range) + VkAccessFlags3KHR dst3_flags, const struct radv_image *image, + const VkImageSubresourceRange *range) { struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); const struct radv_physical_device *pdev = radv_device_physical(device); @@ -6766,7 +6768,7 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ const VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, barrier->src_stage_mask, barrier->src_access_mask, iview->image, &range); + radv_src_access_flush(cmd_buffer, barrier->src_stage_mask, barrier->src_access_mask, 0, iview->image, &range); } if (render->ds_att.iview) { struct radv_image_view *iview = render->ds_att.iview; @@ -6774,7 +6776,7 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ const VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); cmd_buffer->state.flush_bits |= radv_src_access_flush( - cmd_buffer, barrier->src_stage_mask, barrier->src_access_mask, render->ds_att.iview->image, &range); + cmd_buffer, barrier->src_stage_mask, barrier->src_access_mask, 0, render->ds_att.iview->image, &range); } radv_stage_flush(cmd_buffer, barrier->src_stage_mask); @@ -6787,7 +6789,7 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ const VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); cmd_buffer->state.flush_bits |= - radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, barrier->dst_access_mask, iview->image, &range); + radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, barrier->dst_access_mask, 0, iview->image, &range); } if (render->ds_att.iview) { struct radv_image_view *iview = render->ds_att.iview; @@ -6795,7 +6797,7 @@ radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_ const VkImageSubresourceRange range = vk_image_view_subresource_range(&iview->vk); cmd_buffer->state.flush_bits |= - radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, barrier->dst_access_mask, iview->image, &range); + radv_dst_access_flush(cmd_buffer, barrier->dst_stage_mask, barrier->dst_access_mask, 0, iview->image, &range); } radv_gang_barrier(cmd_buffer, barrier->src_stage_mask, barrier->dst_stage_mask); @@ -7373,7 +7375,7 @@ radv_EndCommandBuffer(VkCommandBuffer commandBuffer) */ if (cmd_buffer->state.rb_noncoherent_dirty && !can_skip_buffer_l2_flushes(device)) cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_TRANSFER_WRITE_BIT, NULL, NULL); + VK_ACCESS_2_TRANSFER_WRITE_BIT, 0, NULL, NULL); /* Since NGG streamout uses GDS, we need to make GDS idle when * we leave the IB, otherwise another process might overwrite @@ -8863,7 +8865,7 @@ radv_handle_color_fbfetch_output(struct radv_cmd_buffer *cmd_buffer, uint32_t in /* Consider previous rendering work for WAW hazards. */ cmd_buffer->state.flush_bits |= - radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, att->iview->image, &range); /* Force a transition to FEEDBACK_LOOP_OPTIMAL to decompress DCC. */ @@ -8875,7 +8877,7 @@ radv_handle_color_fbfetch_output(struct radv_cmd_buffer *cmd_buffer, uint32_t in cmd_buffer->state.flush_bits |= radv_dst_access_flush( cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_INPUT_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, att->iview->image, &range); + VK_ACCESS_2_INPUT_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, 0, att->iview->image, &range); cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER; } @@ -8908,7 +8910,7 @@ radv_handle_depth_fbfetch_output(struct radv_cmd_buffer *cmd_buffer) /* Consider previous rendering work for WAW hazards. */ cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, att->iview->image, &range); + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, 0, att->iview->image, &range); /* Force a transition to FEEDBACK_LOOP_OPTIMAL to decompress HTILE. */ radv_handle_image_transition(cmd_buffer, att->iview->image, att->layout, @@ -8918,9 +8920,10 @@ radv_handle_depth_fbfetch_output(struct radv_cmd_buffer *cmd_buffer) att->layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; att->stencil_layout = VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; - cmd_buffer->state.flush_bits |= radv_dst_access_flush( - cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_INPUT_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT, att->iview->image, &range); + cmd_buffer->state.flush_bits |= + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_INPUT_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT, 0, + att->iview->image, &range); cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER; } @@ -12266,11 +12269,11 @@ radv_trace_trace_rays(struct radv_cmd_buffer *cmd_buffer, const VkTraceRaysIndir util_dynarray_append(&cmd_buffer->ray_history, struct radv_rra_ray_history_data *, data); - cmd_buffer->state.flush_bits |= - RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | - radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, NULL, - NULL) | - radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_SHADER_READ_BIT, NULL, NULL); + cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_CS_PARTIAL_FLUSH | + radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_SHADER_WRITE_BIT, 0, NULL, NULL) | + radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_SHADER_READ_BIT, 0, NULL, NULL); radv_update_buffer_cp(cmd_buffer, device->rra_trace.ray_history_addr + offsetof(struct radv_ray_history_header, dispatch_index), @@ -12491,7 +12494,7 @@ radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *ima /* Transitioning from LAYOUT_UNDEFINED layout not everyone is consistent * in considering previous rendering work for WAW hazards. */ state->flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, image, range); + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, 0, image, range); if (image->planes[0].surface.has_stencil && !(range->aspectMask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))) { @@ -12499,7 +12502,7 @@ radv_initialize_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image *ima * read-modify-write operation. */ state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_SHADER_READ_BIT, image, range); + VK_ACCESS_2_SHADER_READ_BIT, 0, image, range); } state->flush_bits |= radv_clear_htile(cmd_buffer, image, range, htile_value, false); @@ -12622,7 +12625,7 @@ radv_init_color_image_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_i * consistent in considering previous rendering work for WAW hazards. */ cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, image, range); + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, 0, image, range); if (radv_image_has_cmask(image)) { static const uint32_t cmask_clear_values[4] = {0xffffffff, 0xdddddddd, 0xeeeeeeee, 0xffffffff}; @@ -12854,6 +12857,28 @@ radv_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer) radv_describe_barrier_end_delayed(cmd_buffer); } +static enum radv_cmd_flush_bits +radv_get_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stage_mask, + VkAccessFlags2 src_access_mask, const struct radv_image *image, + const VkImageSubresourceRange *range, const void *pNext) +{ + const VkMemoryBarrierAccessFlags3KHR *barrier3 = vk_find_struct_const(pNext, MEMORY_BARRIER_ACCESS_FLAGS_3_KHR); + const VkAccessFlags3KHR src3_flags = barrier3 ? barrier3->srcAccessMask3 : 0; + + return radv_src_access_flush(cmd_buffer, src_stage_mask, src_access_mask, src3_flags, image, range); +} + +static enum radv_cmd_flush_bits +radv_get_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 dst_stage_mask, + VkAccessFlags2 dst_access_mask, const struct radv_image *image, + const VkImageSubresourceRange *range, const void *pNext) +{ + const VkMemoryBarrierAccessFlags3KHR *barrier3 = vk_find_struct_const(pNext, MEMORY_BARRIER_ACCESS_FLAGS_3_KHR); + const VkAccessFlags3KHR dst3_flags = barrier3 ? barrier3->dstAccessMask3 : 0; + + return radv_dst_access_flush(cmd_buffer, dst_stage_mask, dst_access_mask, dst3_flags, image, range); +} + static void radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t dep_count, const VkDependencyInfo *dep_infos, enum rgp_barrier_reason reason) @@ -12876,17 +12901,21 @@ radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t dep_count, const VkDep for (uint32_t i = 0; i < dep_info->memoryBarrierCount; i++) { const VkMemoryBarrier2 *barrier = &dep_info->pMemoryBarriers[i]; src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL, NULL); + src_flush_bits |= radv_get_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL, + NULL, barrier->pNext); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL, NULL); + dst_flush_bits |= radv_get_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL, + NULL, barrier->pNext); } for (uint32_t i = 0; i < dep_info->bufferMemoryBarrierCount; i++) { const VkBufferMemoryBarrier2 *barrier = &dep_info->pBufferMemoryBarriers[i]; src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL, NULL); + src_flush_bits |= radv_get_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, NULL, + NULL, barrier->pNext); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL, NULL); + dst_flush_bits |= radv_get_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, NULL, + NULL, barrier->pNext); } for (uint32_t i = 0; i < dep_info->imageMemoryBarrierCount; i++) { @@ -12894,11 +12923,11 @@ radv_barrier(struct radv_cmd_buffer *cmd_buffer, uint32_t dep_count, const VkDep VK_FROM_HANDLE(radv_image, image, barrier->image); src_stage_mask |= barrier->srcStageMask; - src_flush_bits |= radv_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, image, - &barrier->subresourceRange); + src_flush_bits |= radv_get_src_access_flush(cmd_buffer, barrier->srcStageMask, barrier->srcAccessMask, image, + &barrier->subresourceRange, barrier->pNext); dst_stage_mask |= barrier->dstStageMask; - dst_flush_bits |= radv_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, image, - &barrier->subresourceRange); + dst_flush_bits |= radv_get_dst_access_flush(cmd_buffer, barrier->dstStageMask, barrier->dstAccessMask, image, + &barrier->subresourceRange, barrier->pNext); } has_image_transitions |= dep_info->imageMemoryBarrierCount > 0; diff --git a/src/amd/vulkan/radv_cmd_buffer.h b/src/amd/vulkan/radv_cmd_buffer.h index b571d30402e..47c315261cf 100644 --- a/src/amd/vulkan/radv_cmd_buffer.h +++ b/src/amd/vulkan/radv_cmd_buffer.h @@ -738,12 +738,12 @@ void radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const unsigned radv_instance_rate_prolog_index(unsigned num_attributes, uint32_t instance_rate_inputs); enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 src_stages, - VkAccessFlags2 src_flags, const struct radv_image *image, - const VkImageSubresourceRange *range); + VkAccessFlags2 src_flags, VkAccessFlags3KHR src3_flags, + const struct radv_image *image, const VkImageSubresourceRange *range); enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkPipelineStageFlags2 dst_stages, - VkAccessFlags2 dst_flags, const struct radv_image *image, - const VkImageSubresourceRange *range); + VkAccessFlags2 dst_flags, VkAccessFlags3KHR dst3_flags, + const struct radv_image *image, const VkImageSubresourceRange *range); struct radv_resolve_barrier { VkPipelineStageFlags2 src_stage_mask;