diff --git a/src/amd/vulkan/radv_meta_fast_clear.c b/src/amd/vulkan/radv_meta_fast_clear.c index 77cf16c8b99..5b7d4e33814 100644 --- a/src/amd/vulkan/radv_meta_fast_clear.c +++ b/src/amd/vulkan/radv_meta_fast_clear.c @@ -28,6 +28,12 @@ #include "radv_private.h" #include "sid.h" +enum radv_color_op { + FAST_CLEAR_ELIMINATE, + FMASK_DECOMPRESS, + DCC_DECOMPRESS, +}; + static nir_shader * build_dcc_decompress_compute_shader(struct radv_device *dev) { @@ -621,19 +627,46 @@ radv_process_color_image_layer(struct radv_cmd_buffer *cmd_buffer, struct radv_i static void radv_process_color_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, - const VkImageSubresourceRange *subresourceRange, bool decompress_dcc) + const VkImageSubresourceRange *subresourceRange, enum radv_color_op op) { struct radv_device *device = cmd_buffer->device; struct radv_meta_saved_state saved_state; + bool old_predicating = false; bool flush_cb = false; + uint64_t pred_offset; VkPipeline *pipeline; - if (decompress_dcc) { - pipeline = &device->meta_state.fast_clear_flush.dcc_decompress_pipeline; - } else if (radv_image_has_fmask(image) && !image->tc_compatible_cmask) { - pipeline = &device->meta_state.fast_clear_flush.fmask_decompress_pipeline; - } else { + switch (op) { + case FAST_CLEAR_ELIMINATE: pipeline = &device->meta_state.fast_clear_flush.cmask_eliminate_pipeline; + pred_offset = image->fce_pred_offset; + break; + case FMASK_DECOMPRESS: + pipeline = &device->meta_state.fast_clear_flush.fmask_decompress_pipeline; + pred_offset = 0; /* FMASK_DECOMPRESS is never predicated. */ + + /* Flushing CB is required before and after FMASK_DECOMPRESS. */ + flush_cb = true; + break; + case DCC_DECOMPRESS: + pipeline = &device->meta_state.fast_clear_flush.dcc_decompress_pipeline; + pred_offset = image->dcc_pred_offset; + + /* Flushing CB is required before and after DCC_DECOMPRESS. */ + flush_cb = true; + break; + default: + unreachable("Invalid color op"); + } + + if (radv_dcc_enabled(image, subresourceRange->baseMipLevel) && + (image->info.array_size != radv_get_layerCount(image, subresourceRange) || + subresourceRange->baseArrayLayer != 0)) { + /* Only use predication if the image has DCC with mipmaps or + * if the range of layers covers the whole image because the + * predication is based on mip level. + */ + pred_offset = 0; } if (!*pipeline) { @@ -646,16 +679,17 @@ radv_process_color_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image * } } - if (pipeline == &device->meta_state.fast_clear_flush.dcc_decompress_pipeline || - pipeline == &device->meta_state.fast_clear_flush.fmask_decompress_pipeline) { - /* Flushing CB is required before and after DCC_DECOMPRESS or - * FMASK_DECOMPRESS. - */ - flush_cb = true; - } - radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_PASS); + if (pred_offset) { + pred_offset += 8 * subresourceRange->baseMipLevel; + + old_predicating = cmd_buffer->state.predicating; + + radv_emit_set_predication_state_from_image(cmd_buffer, image, pred_offset, true); + cmd_buffer->state.predicating = true; + } + radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline); @@ -663,7 +697,7 @@ radv_process_color_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image * uint32_t width, height; /* Do not decompress levels without DCC. */ - if (decompress_dcc && !radv_dcc_enabled(image, subresourceRange->baseMipLevel + l)) + if (op == DCC_DECOMPRESS && !radv_dcc_enabled(image, subresourceRange->baseMipLevel + l)) continue; width = radv_minify(image->info.width, subresourceRange->baseMipLevel + l); @@ -691,52 +725,7 @@ radv_process_color_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image * cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META; - radv_meta_restore(&saved_state, cmd_buffer); -} - -static void -radv_emit_color_decompress(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, - const VkImageSubresourceRange *subresourceRange, bool decompress_dcc) -{ - bool use_predication = false; - bool old_predicating = false; - - assert(cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL); - - if (decompress_dcc || - (!(radv_image_has_fmask(image) && !image->tc_compatible_cmask) && image->fce_pred_offset)) { - use_predication = true; - } - - /* If we are asked for DCC decompression without DCC predicates we cannot - * use the FCE predicate. */ - if (decompress_dcc && image->dcc_pred_offset == 0) - use_predication = false; - - if (radv_dcc_enabled(image, subresourceRange->baseMipLevel) && - (image->info.array_size != radv_get_layerCount(image, subresourceRange) || - subresourceRange->baseArrayLayer != 0)) { - /* Only use predication if the image has DCC with mipmaps or - * if the range of layers covers the whole image because the - * predication is based on mip level. - */ - use_predication = false; - } - - if (use_predication) { - uint64_t pred_offset = decompress_dcc ? image->dcc_pred_offset : image->fce_pred_offset; - pred_offset += 8 * subresourceRange->baseMipLevel; - - old_predicating = cmd_buffer->state.predicating; - - radv_emit_set_predication_state_from_image(cmd_buffer, image, pred_offset, true); - cmd_buffer->state.predicating = true; - } - - radv_process_color_image(cmd_buffer, image, subresourceRange, decompress_dcc); - - if (use_predication) { - uint64_t pred_offset = decompress_dcc ? image->dcc_pred_offset : image->fce_pred_offset; + if (pred_offset) { pred_offset += 8 * subresourceRange->baseMipLevel; cmd_buffer->state.predicating = old_predicating; @@ -751,41 +740,53 @@ radv_emit_color_decompress(struct radv_cmd_buffer *cmd_buffer, struct radv_image } } + radv_meta_restore(&saved_state, cmd_buffer); + if (image->fce_pred_offset != 0) { - /* Clear the image's fast-clear eliminate predicate because - * FMASK and DCC also imply a fast-clear eliminate. + /* Clear the image's fast-clear eliminate predicate because FMASK_DECOMPRESS and + * DCC_DECOMPRESS also perform a fast-clear eliminate. */ radv_update_fce_metadata(cmd_buffer, image, subresourceRange, false); } /* Mark the image as being decompressed. */ - if (decompress_dcc) + if (op == DCC_DECOMPRESS) radv_update_dcc_metadata(cmd_buffer, image, subresourceRange, false); } +static void +radv_fast_clear_eliminate(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, + const VkImageSubresourceRange *subresourceRange) +{ + struct radv_barrier_data barrier = {0}; + + barrier.layout_transitions.fast_clear_eliminate = 1; + radv_describe_layout_transition(cmd_buffer, &barrier); + + radv_process_color_image(cmd_buffer, image, subresourceRange, FAST_CLEAR_ELIMINATE); +} + +static void +radv_fmask_decompress(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, + const VkImageSubresourceRange *subresourceRange) +{ + struct radv_barrier_data barrier = {0}; + + barrier.layout_transitions.fmask_decompress = 1; + radv_describe_layout_transition(cmd_buffer, &barrier); + + radv_process_color_image(cmd_buffer, image, subresourceRange, FMASK_DECOMPRESS); +} + void radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *subresourceRange) { - struct radv_barrier_data barrier = {0}; - if (radv_image_has_fmask(image) && !image->tc_compatible_cmask) { - barrier.layout_transitions.fmask_decompress = 1; + radv_fmask_decompress(cmd_buffer, image, subresourceRange); } else { - barrier.layout_transitions.fast_clear_eliminate = 1; + radv_fast_clear_eliminate(cmd_buffer, image, subresourceRange); } - radv_describe_layout_transition(cmd_buffer, &barrier); - - assert(cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL); - radv_emit_color_decompress(cmd_buffer, image, subresourceRange, false); -} - -static void -radv_decompress_dcc_gfx(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, - const VkImageSubresourceRange *subresourceRange) -{ - assert(radv_dcc_enabled(image, subresourceRange->baseMipLevel)); - radv_emit_color_decompress(cmd_buffer, image, subresourceRange, true); } static void @@ -911,7 +912,7 @@ radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image radv_describe_layout_transition(cmd_buffer, &barrier); if (cmd_buffer->queue_family_index == RADV_QUEUE_GENERAL) - radv_decompress_dcc_gfx(cmd_buffer, image, subresourceRange); + radv_process_color_image(cmd_buffer, image, subresourceRange, DCC_DECOMPRESS); else radv_decompress_dcc_compute(cmd_buffer, image, subresourceRange); }