diff --git a/src/amd/vulkan/meson.build b/src/amd/vulkan/meson.build index 1d3e85daecb..4bdc115703e 100644 --- a/src/amd/vulkan/meson.build +++ b/src/amd/vulkan/meson.build @@ -162,6 +162,7 @@ libradv_files = files( 'radv_query.h', 'radv_video.c', 'radv_video.h', + 'radv_video_enc.c', 'radv_wsi.c', 'radv_wsi.h', ) diff --git a/src/amd/vulkan/radv_cmd_buffer.h b/src/amd/vulkan/radv_cmd_buffer.h index 62d94cce7db..6fbdec79a36 100644 --- a/src/amd/vulkan/radv_cmd_buffer.h +++ b/src/amd/vulkan/radv_cmd_buffer.h @@ -437,6 +437,21 @@ struct radv_cmd_state { bool uses_dynamic_vertex_binding_stride; }; +struct radv_enc_state { + uint32_t task_size_offset; + uint32_t total_task_size; + unsigned shifter; + unsigned bits_in_shifter; + uint32_t num_zeros; + uint32_t byte_index; + unsigned bits_output; + unsigned bits_size; + bool emulation_prevention; + bool is_even_frame; + unsigned task_id; + uint32_t copy_start_offset; +}; + struct radv_cmd_buffer_upload { uint8_t *map; unsigned offset; @@ -537,6 +552,7 @@ struct radv_cmd_buffer { struct radv_video_session_params *params; struct rvcn_sq_var sq; struct rvcn_decode_buffer_s *decode_buffer; + struct radv_enc_state enc; uint64_t feedback_query_va; } video; diff --git a/src/amd/vulkan/radv_physical_device.h b/src/amd/vulkan/radv_physical_device.h index ca341243396..ddd378ffa13 100644 --- a/src/amd/vulkan/radv_physical_device.h +++ b/src/amd/vulkan/radv_physical_device.h @@ -17,7 +17,7 @@ #include "radv_instance.h" #include "radv_queue.h" #include "radv_radeon_winsys.h" - +#include "ac_vcn_enc.h" #include "wsi_common.h" #include "nir.h" @@ -69,6 +69,13 @@ struct radv_physical_device_cache_key { uint32_t use_ngg_culling : 1; }; +enum radv_video_enc_hw_ver { + RADV_VIDEO_ENC_HW_1_2, + RADV_VIDEO_ENC_HW_2, + RADV_VIDEO_ENC_HW_3, + RADV_VIDEO_ENC_HW_4, +}; + struct radv_physical_device { struct vk_physical_device vk; @@ -171,6 +178,9 @@ struct radv_physical_device { uint32_t stream_handle_base; uint32_t stream_handle_counter; uint32_t av1_version; + rvcn_enc_cmd_t vcn_enc_cmds; + enum radv_video_enc_hw_ver enc_hw_ver; + uint32_t encoder_interface_version; struct radv_physical_device_cache_key cache_key; }; diff --git a/src/amd/vulkan/radv_video.c b/src/amd/vulkan/radv_video.c index 1f4162b0e69..3c1b688c2fc 100644 --- a/src/amd/vulkan/radv_video.c +++ b/src/amd/vulkan/radv_video.c @@ -337,6 +337,9 @@ radv_video_patch_session_parameters(struct vk_video_session_parameters *params) case VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR: default: return; + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + radv_video_patch_encode_session_parameters(params); + break; } } @@ -379,6 +382,31 @@ radv_CreateVideoSessionKHR(VkDevice _device, const VkVideoSessionCreateInfoKHR * vid->stream_type = RDECODE_CODEC_AV1; vid->dpb_type = DPB_DYNAMIC_TIER_2; break; + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + vid->encode = true; + vid->enc_session.encode_standard = RENCODE_ENCODE_STANDARD_H264; + vid->enc_session.aligned_picture_width = align(vid->vk.max_coded.width, 16); + vid->enc_session.aligned_picture_height = align(vid->vk.max_coded.height, 16); + vid->enc_session.padding_width = vid->enc_session.aligned_picture_width - vid->vk.max_coded.width; + vid->enc_session.padding_height = vid->enc_session.aligned_picture_height - vid->vk.max_coded.height; + vid->enc_session.display_remote = 0; + vid->enc_session.pre_encode_mode = 0; + vid->enc_session.pre_encode_chroma_enabled = 0; + switch (vid->vk.enc_usage.tuning_mode) { + case VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR: + default: + vid->enc_preset_mode = RENCODE_PRESET_MODE_BALANCE; + break; + case VK_VIDEO_ENCODE_TUNING_MODE_LOW_LATENCY_KHR: + case VK_VIDEO_ENCODE_TUNING_MODE_ULTRA_LOW_LATENCY_KHR: + vid->enc_preset_mode = RENCODE_PRESET_MODE_SPEED; + break; + case VK_VIDEO_ENCODE_TUNING_MODE_HIGH_QUALITY_KHR: + case VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR: + vid->enc_preset_mode = RENCODE_PRESET_MODE_QUALITY; + break; + } + break; default: return VK_ERROR_FEATURE_NOT_PRESENT; } @@ -451,6 +479,7 @@ radv_GetPhysicalDeviceVideoCapabilitiesKHR(VkPhysicalDevice physicalDevice, cons { VK_FROM_HANDLE(radv_physical_device, pdev, physicalDevice); const struct video_codec_cap *cap = NULL; + bool is_encode = false; switch (pVideoProfile->videoCodecOperation) { #ifndef _WIN32 @@ -463,6 +492,10 @@ radv_GetPhysicalDeviceVideoCapabilitiesKHR(VkPhysicalDevice physicalDevice, cons case VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR: cap = &pdev->info.dec_caps.codec_info[AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_AV1]; break; + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + cap = &pdev->info.enc_caps.codec_info[AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC]; + is_encode = true; + break; #endif default: unreachable("unsupported operation"); @@ -472,17 +505,40 @@ radv_GetPhysicalDeviceVideoCapabilitiesKHR(VkPhysicalDevice physicalDevice, cons cap = NULL; pCapabilities->flags = 0; - pCapabilities->minBitstreamBufferOffsetAlignment = 128; - pCapabilities->minBitstreamBufferSizeAlignment = 128; pCapabilities->pictureAccessGranularity.width = VL_MACROBLOCK_WIDTH; pCapabilities->pictureAccessGranularity.height = VL_MACROBLOCK_HEIGHT; pCapabilities->minCodedExtent.width = VL_MACROBLOCK_WIDTH; pCapabilities->minCodedExtent.height = VL_MACROBLOCK_HEIGHT; - struct VkVideoDecodeCapabilitiesKHR *dec_caps = - (struct VkVideoDecodeCapabilitiesKHR *)vk_find_struct(pCapabilities->pNext, VIDEO_DECODE_CAPABILITIES_KHR); - if (dec_caps) - dec_caps->flags = VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR; + struct VkVideoDecodeCapabilitiesKHR *dec_caps = NULL; + struct VkVideoEncodeCapabilitiesKHR *enc_caps = NULL; + if (!is_encode) { + dec_caps = + (struct VkVideoDecodeCapabilitiesKHR *)vk_find_struct(pCapabilities->pNext, VIDEO_DECODE_CAPABILITIES_KHR); + if (dec_caps) + dec_caps->flags = VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR; + pCapabilities->minBitstreamBufferOffsetAlignment = 128; + pCapabilities->minBitstreamBufferSizeAlignment = 128; + } else { + enc_caps = + (struct VkVideoEncodeCapabilitiesKHR *)vk_find_struct(pCapabilities->pNext, VIDEO_ENCODE_CAPABILITIES_KHR); + + if (enc_caps) { + enc_caps->flags = 0; + enc_caps->rateControlModes = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR | + VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR | + VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR; + enc_caps->maxRateControlLayers = RADV_ENC_MAX_RATE_LAYER; + enc_caps->maxBitrate = 0; + enc_caps->maxQualityLevels = 2; + enc_caps->encodeInputPictureGranularity.width = 1; + enc_caps->encodeInputPictureGranularity.height = 1; + enc_caps->supportedEncodeFeedbackFlags = VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BUFFER_OFFSET_BIT_KHR | + VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BYTES_WRITTEN_BIT_KHR; + } + pCapabilities->minBitstreamBufferOffsetAlignment = 16; + pCapabilities->minBitstreamBufferSizeAlignment = 16; + } switch (pVideoProfile->videoCodecOperation) { case VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR: { @@ -573,6 +629,33 @@ radv_GetPhysicalDeviceVideoCapabilitiesKHR(VkPhysicalDevice physicalDevice, cons pCapabilities->stdHeaderVersion.specVersion = VK_STD_VULKAN_VIDEO_CODEC_AV1_DECODE_SPEC_VERSION; break; } + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: { + struct VkVideoEncodeH264CapabilitiesKHR *ext = (struct VkVideoEncodeH264CapabilitiesKHR *)vk_find_struct( + pCapabilities->pNext, VIDEO_ENCODE_H264_CAPABILITIES_KHR); + pCapabilities->maxDpbSlots = NUM_H2645_REFS; + pCapabilities->maxActiveReferencePictures = NUM_H2645_REFS; + ext->flags = VK_VIDEO_ENCODE_H264_CAPABILITY_HRD_COMPLIANCE_BIT_KHR; + ext->maxLevelIdc = cap ? cap->max_level : 0; + ext->maxSliceCount = 128; + ext->maxPPictureL0ReferenceCount = 1; + ext->maxBPictureL0ReferenceCount = 0; + ext->maxL1ReferenceCount = 0; + ext->maxTemporalLayerCount = 4; + ext->expectDyadicTemporalLayerPattern = false; + ext->minQp = 0; + ext->maxQp = 51; + ext->prefersGopRemainingFrames = false; + ext->requiresGopRemainingFrames = false; + ext->stdSyntaxFlags = VK_VIDEO_ENCODE_H264_STD_CONSTRAINED_INTRA_PRED_FLAG_SET_BIT_KHR | + VK_VIDEO_ENCODE_H264_STD_ENTROPY_CODING_MODE_FLAG_UNSET_BIT_KHR | + VK_VIDEO_ENCODE_H264_STD_ENTROPY_CODING_MODE_FLAG_SET_BIT_KHR; + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_3) + ext->stdSyntaxFlags |= VK_VIDEO_ENCODE_H264_STD_WEIGHTED_BIPRED_IDC_EXPLICIT_BIT_KHR; + + strcpy(pCapabilities->stdHeaderVersion.extensionName, VK_STD_VULKAN_VIDEO_CODEC_H264_ENCODE_EXTENSION_NAME); + pCapabilities->stdHeaderVersion.specVersion = VK_STD_VULKAN_VIDEO_CODEC_H264_ENCODE_SPEC_VERSION; + break; + } default: break; } @@ -666,6 +749,10 @@ radv_GetVideoSessionMemoryRequirementsKHR(VkDevice _device, VkVideoSessionKHR vi uint32_t memory_type_bits = (1u << pdev->memory_properties.memoryTypeCount) - 1; + if (vid->encode) { + return radv_video_get_encode_session_memory_requirements(device, vid, pMemoryRequirementsCount, + pMemoryRequirements); + } VK_OUTARRAY_MAKE_TYPED(VkVideoSessionMemoryRequirementsKHR, out, pMemoryRequirements, pMemoryRequirementsCount); /* 1 buffer for session context */ if (pdev->info.family >= CHIP_POLARIS10) { @@ -2751,6 +2838,9 @@ radv_CmdBeginVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoBeginCod cmd_buffer->video.vid = vid; cmd_buffer->video.params = params; + + if (vid->encode) + radv_video_enc_begin_coding(cmd_buffer); } static void @@ -2833,6 +2923,10 @@ radv_CmdControlVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoCoding struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); struct radv_physical_device *pdev = radv_device_physical(device); + if (cmd_buffer->video.vid->encode) { + radv_video_enc_control_video_coding(cmd_buffer, pCodingControlInfo); + return; + } if (pCodingControlInfo->flags & VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR) { if (radv_has_uvd(pdev)) radv_uvd_cmd_reset(cmd_buffer); @@ -2844,6 +2938,12 @@ radv_CmdControlVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoCoding VKAPI_ATTR void VKAPI_CALL radv_CmdEndVideoCodingKHR(VkCommandBuffer commandBuffer, const VkVideoEndCodingInfoKHR *pEndCodingInfo) { + VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + + if (cmd_buffer->video.vid->encode) { + radv_video_enc_end_coding(cmd_buffer); + return; + } } static void diff --git a/src/amd/vulkan/radv_video.h b/src/amd/vulkan/radv_video.h index f7388744de8..a0fa2c94a41 100644 --- a/src/amd/vulkan/radv_video.h +++ b/src/amd/vulkan/radv_video.h @@ -13,11 +13,16 @@ #include "vk_video.h" +#include "ac_vcn.h" + #define VL_MACROBLOCK_WIDTH 16 #define VL_MACROBLOCK_HEIGHT 16 struct radv_physical_device; struct rvcn_sq_var; +struct radv_cmd_buffer; + +#define RADV_ENC_MAX_RATE_LAYER 4 struct radv_vid_mem { struct radv_device_memory *mem; @@ -31,6 +36,7 @@ struct radv_video_session { uint32_t stream_handle; unsigned stream_type; bool interlaced; + bool encode; enum { DPB_MAX_RES = 0, DPB_DYNAMIC_TIER_1, DPB_DYNAMIC_TIER_2 } dpb_type; unsigned db_alignment; @@ -38,6 +44,13 @@ struct radv_video_session { struct radv_vid_mem ctx; unsigned dbg_frame_cnt; + rvcn_enc_session_init_t enc_session; + rvcn_enc_layer_control_t rc_layer_control; + rvcn_enc_rate_ctl_layer_init_t rc_layer_init[RADV_ENC_MAX_RATE_LAYER]; + rvcn_enc_rate_ctl_per_picture_t rc_per_pic[RADV_ENC_MAX_RATE_LAYER]; + uint32_t enc_preset_mode; + uint32_t enc_rate_control_method; + bool enc_rate_control_default; }; VK_DEFINE_NONDISP_HANDLE_CASTS(radv_video_session, vk.base, VkVideoSessionKHR, VK_OBJECT_TYPE_VIDEO_SESSION_KHR) @@ -58,4 +71,14 @@ void radv_vcn_sq_header(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq, bool e void radv_vcn_sq_tail(struct radeon_cmdbuf *cs, struct rvcn_sq_var *sq); +void radv_init_physical_device_encoder(struct radv_physical_device *pdevice); +void radv_video_enc_begin_coding(struct radv_cmd_buffer *cmd_buffer); +void radv_video_enc_end_coding(struct radv_cmd_buffer *cmd_buffer); +void radv_video_enc_control_video_coding(struct radv_cmd_buffer *cmd_buffer, + const VkVideoCodingControlInfoKHR *pCodingControlInfo); +VkResult radv_video_get_encode_session_memory_requirements(struct radv_device *device, struct radv_video_session *vid, + uint32_t *pMemoryRequirementsCount, + VkVideoSessionMemoryRequirementsKHR *pMemoryRequirements); +void radv_video_patch_encode_session_parameters(struct vk_video_session_parameters *params); + #endif /* RADV_VIDEO_H */ diff --git a/src/amd/vulkan/radv_video_enc.c b/src/amd/vulkan/radv_video_enc.c new file mode 100644 index 00000000000..68ada453b8d --- /dev/null +++ b/src/amd/vulkan/radv_video_enc.c @@ -0,0 +1,1478 @@ +/************************************************************************** + * + * Copyright 2017 Advanced Micro Devices, Inc. + * Copyright 2023 Red Hat Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +#include "radv_buffer.h" +#include "radv_cs.h" +#include "radv_device_memory.h" +#include "radv_entrypoints.h" +#include "radv_image_view.h" +#include "radv_physical_device.h" +#include "radv_video.h" + +#include "ac_vcn_enc.h" + +#define RENCODE_V4_FW_INTERFACE_MAJOR_VERSION 1 +#define RENCODE_V4_FW_INTERFACE_MINOR_VERSION 7 + +#define RENCODE_V4_IB_PARAM_ENCODE_STATISTICS 0x0000001a + +#define RENCODE_V3_FW_INTERFACE_MAJOR_VERSION 1 +#define RENCODE_V3_FW_INTERFACE_MINOR_VERSION 27 + +#define RENCODE_V2_IB_PARAM_SESSION_INFO 0x00000001 +#define RENCODE_V2_IB_PARAM_TASK_INFO 0x00000002 +#define RENCODE_V2_IB_PARAM_SESSION_INIT 0x00000003 +#define RENCODE_V2_IB_PARAM_LAYER_CONTROL 0x00000004 +#define RENCODE_V2_IB_PARAM_LAYER_SELECT 0x00000005 +#define RENCODE_V2_IB_PARAM_RATE_CONTROL_SESSION_INIT 0x00000006 +#define RENCODE_V2_IB_PARAM_RATE_CONTROL_LAYER_INIT 0x00000007 +#define RENCODE_V2_IB_PARAM_RATE_CONTROL_PER_PICTURE 0x00000008 +#define RENCODE_V2_IB_PARAM_QUALITY_PARAMS 0x00000009 +#define RENCODE_V2_IB_PARAM_DIRECT_OUTPUT_NALU 0x0000000a +#define RENCODE_V2_IB_PARAM_SLICE_HEADER 0x0000000b +#define RENCODE_V2_IB_PARAM_INPUT_FORMAT 0x0000000c +#define RENCODE_V2_IB_PARAM_OUTPUT_FORMAT 0x0000000d +#define RENCODE_V2_IB_PARAM_ENCODE_PARAMS 0x0000000f +#define RENCODE_V2_IB_PARAM_INTRA_REFRESH 0x00000010 +#define RENCODE_V2_IB_PARAM_ENCODE_CONTEXT_BUFFER 0x00000011 +#define RENCODE_V2_IB_PARAM_VIDEO_BITSTREAM_BUFFER 0x00000012 +#define RENCODE_V2_IB_PARAM_FEEDBACK_BUFFER 0x00000015 +#define RENCODE_V2_IB_PARAM_ENCODE_STATISTICS 0x00000019 + +#define RENCODE_V2_H264_IB_PARAM_SLICE_CONTROL 0x00200001 +#define RENCODE_V2_H264_IB_PARAM_SPEC_MISC 0x00200002 +#define RENCODE_V2_H264_IB_PARAM_ENCODE_PARAMS 0x00200003 +#define RENCODE_V2_H264_IB_PARAM_DEBLOCKING_FILTER 0x00200004 + +#define RENCODE_V2_FW_INTERFACE_MAJOR_VERSION 1 +#define RENCODE_V2_FW_INTERFACE_MINOR_VERSION 1 + +#define RENCODE_IB_PARAM_SESSION_INFO 0x00000001 +#define RENCODE_IB_PARAM_TASK_INFO 0x00000002 +#define RENCODE_IB_PARAM_SESSION_INIT 0x00000003 +#define RENCODE_IB_PARAM_LAYER_CONTROL 0x00000004 +#define RENCODE_IB_PARAM_LAYER_SELECT 0x00000005 +#define RENCODE_IB_PARAM_RATE_CONTROL_SESSION_INIT 0x00000006 +#define RENCODE_IB_PARAM_RATE_CONTROL_LAYER_INIT 0x00000007 +#define RENCODE_IB_PARAM_RATE_CONTROL_PER_PICTURE 0x00000008 +#define RENCODE_IB_PARAM_QUALITY_PARAMS 0x00000009 +#define RENCODE_IB_PARAM_SLICE_HEADER 0x0000000a +#define RENCODE_IB_PARAM_ENCODE_PARAMS 0x0000000b +#define RENCODE_IB_PARAM_INTRA_REFRESH 0x0000000c +#define RENCODE_IB_PARAM_ENCODE_CONTEXT_BUFFER 0x0000000d +#define RENCODE_IB_PARAM_VIDEO_BITSTREAM_BUFFER 0x0000000e +#define RENCODE_IB_PARAM_FEEDBACK_BUFFER 0x00000010 +#define RENCODE_IB_PARAM_DIRECT_OUTPUT_NALU 0x00000020 +#define RENCODE_IB_PARAM_ENCODE_STATISTICS 0x00000024 + +#define RENCODE_H264_IB_PARAM_SLICE_CONTROL 0x00200001 +#define RENCODE_H264_IB_PARAM_SPEC_MISC 0x00200002 +#define RENCODE_H264_IB_PARAM_ENCODE_PARAMS 0x00200003 +#define RENCODE_H264_IB_PARAM_DEBLOCKING_FILTER 0x00200004 + +#define RENCODE_FW_INTERFACE_MAJOR_VERSION 1 +#define RENCODE_FW_INTERFACE_MINOR_VERSION 9 + +void +radv_init_physical_device_encoder(struct radv_physical_device *pdev) +{ + if (pdev->info.family >= CHIP_NAVI31) { + pdev->enc_hw_ver = RADV_VIDEO_ENC_HW_4; + pdev->encoder_interface_version = ((RENCODE_V4_FW_INTERFACE_MAJOR_VERSION << RENCODE_IF_MAJOR_VERSION_SHIFT) | + (RENCODE_V4_FW_INTERFACE_MINOR_VERSION << RENCODE_IF_MINOR_VERSION_SHIFT)); + } else if (pdev->info.family >= CHIP_NAVI21) { + pdev->enc_hw_ver = RADV_VIDEO_ENC_HW_3; + pdev->encoder_interface_version = ((RENCODE_V3_FW_INTERFACE_MAJOR_VERSION << RENCODE_IF_MAJOR_VERSION_SHIFT) | + (RENCODE_V3_FW_INTERFACE_MINOR_VERSION << RENCODE_IF_MINOR_VERSION_SHIFT)); + } else if (pdev->info.family >= CHIP_RENOIR) { + pdev->enc_hw_ver = RADV_VIDEO_ENC_HW_2; + pdev->encoder_interface_version = ((RENCODE_V2_FW_INTERFACE_MAJOR_VERSION << RENCODE_IF_MAJOR_VERSION_SHIFT) | + (RENCODE_V2_FW_INTERFACE_MINOR_VERSION << RENCODE_IF_MINOR_VERSION_SHIFT)); + } else { + pdev->enc_hw_ver = RADV_VIDEO_ENC_HW_1_2; + pdev->encoder_interface_version = ((RENCODE_FW_INTERFACE_MAJOR_VERSION << RENCODE_IF_MAJOR_VERSION_SHIFT) | + (RENCODE_FW_INTERFACE_MINOR_VERSION << RENCODE_IF_MINOR_VERSION_SHIFT)); + } + + if (pdev->info.family >= CHIP_RENOIR) { + pdev->vcn_enc_cmds.session_info = RENCODE_V2_IB_PARAM_SESSION_INFO; + pdev->vcn_enc_cmds.task_info = RENCODE_V2_IB_PARAM_TASK_INFO; + pdev->vcn_enc_cmds.session_init = RENCODE_V2_IB_PARAM_SESSION_INIT; + pdev->vcn_enc_cmds.layer_control = RENCODE_V2_IB_PARAM_LAYER_CONTROL; + pdev->vcn_enc_cmds.layer_select = RENCODE_V2_IB_PARAM_LAYER_SELECT; + pdev->vcn_enc_cmds.rc_session_init = RENCODE_V2_IB_PARAM_RATE_CONTROL_SESSION_INIT; + pdev->vcn_enc_cmds.rc_layer_init = RENCODE_V2_IB_PARAM_RATE_CONTROL_LAYER_INIT; + pdev->vcn_enc_cmds.rc_per_pic = RENCODE_V2_IB_PARAM_RATE_CONTROL_PER_PICTURE; + pdev->vcn_enc_cmds.quality_params = RENCODE_V2_IB_PARAM_QUALITY_PARAMS; + pdev->vcn_enc_cmds.nalu = RENCODE_V2_IB_PARAM_DIRECT_OUTPUT_NALU; + pdev->vcn_enc_cmds.slice_header = RENCODE_V2_IB_PARAM_SLICE_HEADER; + pdev->vcn_enc_cmds.input_format = RENCODE_V2_IB_PARAM_INPUT_FORMAT; + pdev->vcn_enc_cmds.output_format = RENCODE_V2_IB_PARAM_OUTPUT_FORMAT; + pdev->vcn_enc_cmds.enc_params = RENCODE_V2_IB_PARAM_ENCODE_PARAMS; + pdev->vcn_enc_cmds.intra_refresh = RENCODE_V2_IB_PARAM_INTRA_REFRESH; + pdev->vcn_enc_cmds.ctx = RENCODE_V2_IB_PARAM_ENCODE_CONTEXT_BUFFER; + pdev->vcn_enc_cmds.bitstream = RENCODE_V2_IB_PARAM_VIDEO_BITSTREAM_BUFFER; + pdev->vcn_enc_cmds.feedback = RENCODE_V2_IB_PARAM_FEEDBACK_BUFFER; + pdev->vcn_enc_cmds.slice_control_h264 = RENCODE_V2_H264_IB_PARAM_SLICE_CONTROL; + pdev->vcn_enc_cmds.spec_misc_h264 = RENCODE_V2_H264_IB_PARAM_SPEC_MISC; + pdev->vcn_enc_cmds.enc_params_h264 = RENCODE_V2_H264_IB_PARAM_ENCODE_PARAMS; + pdev->vcn_enc_cmds.deblocking_filter_h264 = RENCODE_V2_H264_IB_PARAM_DEBLOCKING_FILTER; + if (pdev->enc_hw_ver == RADV_VIDEO_ENC_HW_4) { + pdev->vcn_enc_cmds.enc_statistics = RENCODE_V4_IB_PARAM_ENCODE_STATISTICS; + } else + pdev->vcn_enc_cmds.enc_statistics = RENCODE_V2_IB_PARAM_ENCODE_STATISTICS; + } else { + pdev->vcn_enc_cmds.session_info = RENCODE_IB_PARAM_SESSION_INFO; + pdev->vcn_enc_cmds.task_info = RENCODE_IB_PARAM_TASK_INFO; + pdev->vcn_enc_cmds.session_init = RENCODE_IB_PARAM_SESSION_INIT; + pdev->vcn_enc_cmds.layer_control = RENCODE_IB_PARAM_LAYER_CONTROL; + pdev->vcn_enc_cmds.layer_select = RENCODE_IB_PARAM_LAYER_SELECT; + pdev->vcn_enc_cmds.rc_session_init = RENCODE_IB_PARAM_RATE_CONTROL_SESSION_INIT; + pdev->vcn_enc_cmds.rc_layer_init = RENCODE_IB_PARAM_RATE_CONTROL_LAYER_INIT; + pdev->vcn_enc_cmds.rc_per_pic = RENCODE_IB_PARAM_RATE_CONTROL_PER_PICTURE; + pdev->vcn_enc_cmds.quality_params = RENCODE_IB_PARAM_QUALITY_PARAMS; + pdev->vcn_enc_cmds.nalu = RENCODE_IB_PARAM_DIRECT_OUTPUT_NALU; + pdev->vcn_enc_cmds.slice_header = RENCODE_IB_PARAM_SLICE_HEADER; + pdev->vcn_enc_cmds.enc_params = RENCODE_IB_PARAM_ENCODE_PARAMS; + pdev->vcn_enc_cmds.intra_refresh = RENCODE_IB_PARAM_INTRA_REFRESH; + pdev->vcn_enc_cmds.ctx = RENCODE_IB_PARAM_ENCODE_CONTEXT_BUFFER; + pdev->vcn_enc_cmds.bitstream = RENCODE_IB_PARAM_VIDEO_BITSTREAM_BUFFER; + pdev->vcn_enc_cmds.feedback = RENCODE_IB_PARAM_FEEDBACK_BUFFER; + pdev->vcn_enc_cmds.slice_control_h264 = RENCODE_H264_IB_PARAM_SLICE_CONTROL; + pdev->vcn_enc_cmds.spec_misc_h264 = RENCODE_H264_IB_PARAM_SPEC_MISC; + pdev->vcn_enc_cmds.enc_params_h264 = RENCODE_H264_IB_PARAM_ENCODE_PARAMS; + pdev->vcn_enc_cmds.deblocking_filter_h264 = RENCODE_H264_IB_PARAM_DEBLOCKING_FILTER; + pdev->vcn_enc_cmds.enc_statistics = RENCODE_IB_PARAM_ENCODE_STATISTICS; + } +} + +/* to process invalid frame rate */ +static void +radv_vcn_enc_invalid_frame_rate(uint32_t *den, uint32_t *num) +{ + if (*den == 0 || *num == 0) { + *den = 1; + *num = 30; + } +} + +static uint32_t +radv_vcn_per_frame_integer(uint32_t bitrate, uint32_t den, uint32_t num) +{ + uint64_t rate_den = (uint64_t)bitrate * (uint64_t)den; + + return (uint32_t)(rate_den / num); +} + +static uint32_t +radv_vcn_per_frame_frac(uint32_t bitrate, uint32_t den, uint32_t num) +{ + uint64_t rate_den = (uint64_t)bitrate * (uint64_t)den; + uint64_t remainder = rate_den % num; + + return (uint32_t)((remainder << 32) / num); +} + +static void +radv_enc_set_emulation_prevention(struct radv_cmd_buffer *cmd_buffer, bool set) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + if (set != enc->emulation_prevention) { + enc->emulation_prevention = set; + enc->num_zeros = 0; + } +} + +static uint32_t +radv_enc_value_bits(uint32_t value) +{ + uint32_t i = 1; + + while (value > 1) { + i++; + value >>= 1; + } + + return i; +} + +static const unsigned index_to_shifts[4] = {24, 16, 8, 0}; + +static void +radv_enc_output_one_byte(struct radv_cmd_buffer *cmd_buffer, unsigned char byte) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_enc_state *enc = &cmd_buffer->video.enc; + if (enc->byte_index == 0) + cs->buf[cs->cdw] = 0; + cs->buf[cs->cdw] |= ((unsigned int)(byte) << index_to_shifts[enc->byte_index]); + enc->byte_index++; + + if (enc->byte_index >= 4) { + enc->byte_index = 0; + cs->cdw++; + } +} + +static void +radv_enc_emulation_prevention(struct radv_cmd_buffer *cmd_buffer, unsigned char byte) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + if (enc->emulation_prevention) { + if ((enc->num_zeros >= 2) && ((byte == 0x00) || (byte == 0x01) || (byte == 0x02) || (byte == 0x03))) { + radv_enc_output_one_byte(cmd_buffer, 0x03); + enc->bits_output += 8; + enc->num_zeros = 0; + } + enc->num_zeros = (byte == 0 ? (enc->num_zeros + 1) : 0); + } +} + +static void +radv_enc_code_fixed_bits(struct radv_cmd_buffer *cmd_buffer, unsigned int value, unsigned int num_bits) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + unsigned int bits_to_pack = 0; + enc->bits_size += num_bits; + + while (num_bits > 0) { + unsigned int value_to_pack = value & (0xffffffff >> (32 - num_bits)); + bits_to_pack = num_bits > (32 - enc->bits_in_shifter) ? (32 - enc->bits_in_shifter) : num_bits; + + if (bits_to_pack < num_bits) + value_to_pack = value_to_pack >> (num_bits - bits_to_pack); + + enc->shifter |= value_to_pack << (32 - enc->bits_in_shifter - bits_to_pack); + num_bits -= bits_to_pack; + enc->bits_in_shifter += bits_to_pack; + + while (enc->bits_in_shifter >= 8) { + unsigned char output_byte = (unsigned char)(enc->shifter >> 24); + enc->shifter <<= 8; + radv_enc_emulation_prevention(cmd_buffer, output_byte); + radv_enc_output_one_byte(cmd_buffer, output_byte); + enc->bits_in_shifter -= 8; + enc->bits_output += 8; + } + } +} + +static void +radv_enc_reset(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + enc->emulation_prevention = false; + enc->shifter = 0; + enc->bits_in_shifter = 0; + enc->bits_output = 0; + enc->num_zeros = 0; + enc->byte_index = 0; + enc->bits_size = 0; +} + +static void +radv_enc_byte_align(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + unsigned int num_padding_zeros = (32 - enc->bits_in_shifter) % 8; + + if (num_padding_zeros > 0) + radv_enc_code_fixed_bits(cmd_buffer, 0, num_padding_zeros); +} + +static void +radv_enc_flush_headers(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + if (enc->bits_in_shifter != 0) { + unsigned char output_byte = (unsigned char)(enc->shifter >> 24); + radv_enc_emulation_prevention(cmd_buffer, output_byte); + radv_enc_output_one_byte(cmd_buffer, output_byte); + enc->bits_output += enc->bits_in_shifter; + enc->shifter = 0; + enc->bits_in_shifter = 0; + enc->num_zeros = 0; + } + + if (enc->byte_index > 0) { + cs->cdw++; + enc->byte_index = 0; + } +} + +static void +radv_enc_code_ue(struct radv_cmd_buffer *cmd_buffer, unsigned int value) +{ + int x = -1; + unsigned int ue_code = value + 1; + value += 1; + + while (value) { + value = (value >> 1); + x += 1; + } + + unsigned int ue_length = (x << 1) + 1; + radv_enc_code_fixed_bits(cmd_buffer, ue_code, ue_length); +} + +static void +radv_enc_code_se(struct radv_cmd_buffer *cmd_buffer, int value) +{ + unsigned int v = 0; + + if (value != 0) + v = (value < 0 ? ((unsigned int)(0 - value) << 1) : (((unsigned int)(value) << 1) - 1)); + + radv_enc_code_ue(cmd_buffer, v); +} + +#define ENC_BEGIN \ + { \ + uint32_t begin = cs->cdw++; + +#define ENC_END \ + radeon_emit_direct(cs, begin, (cs->cdw - begin) * 4); \ + cmd_buffer->video.enc.total_task_size += cs->buf[begin]; \ + } + +static void +radv_enc_session_info(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.session_info); + radeon_emit(cs, pdev->encoder_interface_version); + + radv_cs_add_buffer(device->ws, cs, cmd_buffer->video.vid->sessionctx.mem->bo); + uint64_t va = radv_buffer_get_va(cmd_buffer->video.vid->sessionctx.mem->bo); + va += cmd_buffer->video.vid->sessionctx.offset; + radeon_emit(cs, va >> 32); + radeon_emit(cs, va & 0xffffffff); + radeon_emit(cs, RENCODE_ENGINE_TYPE_ENCODE); + ENC_END; +} + +static void +radv_enc_task_info(struct radv_cmd_buffer *cmd_buffer, bool feedback) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_enc_state *enc = &cmd_buffer->video.enc; + + enc->task_id++; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.task_info); + enc->task_size_offset = cs->cdw++; + radeon_emit(cs, enc->task_id); + radeon_emit(cs, feedback ? 1 : 0); + ENC_END; +} + +static void +radv_enc_session_init(struct radv_cmd_buffer *cmd_buffer, const struct VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radv_video_session *vid = cmd_buffer->video.vid; + struct radeon_cmdbuf *cs = cmd_buffer->cs; + unsigned alignment = 16; + + uint32_t w = enc_info->srcPictureResource.codedExtent.width; + uint32_t h = enc_info->srcPictureResource.codedExtent.height; + uint32_t aligned_picture_width = align(w, alignment); + uint32_t aligned_picture_height = align(h, alignment); + uint32_t padding_width = aligned_picture_width - w; + uint32_t padding_height = aligned_picture_height - h; + + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.session_init); + radeon_emit(cs, vid->enc_session.encode_standard); + radeon_emit(cs, aligned_picture_width); + radeon_emit(cs, aligned_picture_height); + radeon_emit(cs, padding_width); + radeon_emit(cs, padding_height); + radeon_emit(cs, vid->enc_session.pre_encode_mode); + radeon_emit(cs, vid->enc_session.pre_encode_chroma_enabled); + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, 0); // slice output enabled. + } + radeon_emit(cs, vid->enc_session.display_remote); + ENC_END; +} + +static void +radv_enc_layer_control(struct radv_cmd_buffer *cmd_buffer, const rvcn_enc_layer_control_t *rc_layer_control) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.layer_control); + radeon_emit(cs, rc_layer_control->max_num_temporal_layers); // max num temporal layesr + radeon_emit(cs, rc_layer_control->num_temporal_layers); // num temporal layers + ENC_END; +} + +static void +radv_enc_layer_select(struct radv_cmd_buffer *cmd_buffer, int tl_idx) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.layer_select); + radeon_emit(cs, tl_idx); // temporal layer index + ENC_END; +} + +static void +radv_enc_slice_control(struct radv_cmd_buffer *cmd_buffer, const struct VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + + uint32_t num_mbs_in_slice; + uint32_t width_in_mbs = enc_info->srcPictureResource.codedExtent.width / 16; + uint32_t height_in_mbs = enc_info->srcPictureResource.codedExtent.height / 16; + num_mbs_in_slice = width_in_mbs * height_in_mbs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.slice_control_h264); + radeon_emit(cs, RENCODE_H264_SLICE_CONTROL_MODE_FIXED_MBS); // slice control mode + radeon_emit(cs, num_mbs_in_slice); // num mbs per slice + ENC_END; +} + +static void +radv_enc_spec_misc_h264(struct radv_cmd_buffer *cmd_buffer, const struct VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + const struct VkVideoEncodeH264PictureInfoKHR *h264_picture_info = + vk_find_struct_const(enc_info->pNext, VIDEO_ENCODE_H264_PICTURE_INFO_KHR); + const StdVideoEncodeH264PictureInfo *pic = h264_picture_info->pStdPictureInfo; + const StdVideoH264SequenceParameterSet *sps = + vk_video_find_h264_enc_std_sps(&cmd_buffer->video.params->vk, pic->seq_parameter_set_id); + const StdVideoH264PictureParameterSet *pps = + vk_video_find_h264_enc_std_pps(&cmd_buffer->video.params->vk, pic->pic_parameter_set_id); + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.spec_misc_h264); + radeon_emit(cs, pps->flags.constrained_intra_pred_flag); // constrained_intra_pred_flag + radeon_emit(cs, pps->flags.entropy_coding_mode_flag); // cabac enable + radeon_emit(cs, 0); // cabac init idc + radeon_emit(cs, 1); // half pel enabled + radeon_emit(cs, 1); // quarter pel enabled + radeon_emit(cs, cmd_buffer->video.vid->vk.h264.profile_idc); // profile_idc + radeon_emit(cs, vk_video_get_h264_level(sps->level_idc)); + + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, 0); // v3 b_picture_enabled + radeon_emit(cs, pps->weighted_bipred_idc); // v3 weighted bipred idc + } + + ENC_END; +} + +static void +radv_enc_rc_session_init(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_video_session *vid = cmd_buffer->video.vid; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.rc_session_init); + radeon_emit(cs, vid->enc_rate_control_method); // rate_control_method); + radeon_emit(cs, 48); // vbv_buffer_level); + ENC_END; +} + +static void +radv_enc_rc_layer_init(struct radv_cmd_buffer *cmd_buffer, rvcn_enc_rate_ctl_layer_init_t *layer_init) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.rc_layer_init); + radeon_emit(cs, layer_init->target_bit_rate); // target bit rate + radeon_emit(cs, layer_init->peak_bit_rate); // peak bit rate + radeon_emit(cs, layer_init->frame_rate_num); // frame rate num + radeon_emit(cs, layer_init->frame_rate_den); // frame rate dem + radeon_emit(cs, layer_init->vbv_buffer_size); // vbv buffer size + radeon_emit(cs, layer_init->avg_target_bits_per_picture); // avg target bits per picture + radeon_emit(cs, layer_init->peak_bits_per_picture_integer); // peak bit per picture int + radeon_emit(cs, layer_init->peak_bits_per_picture_fractional); // peak bit per picture fract + ENC_END; +} + +static void +radv_enc_deblocking_filter_h264(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + const struct VkVideoEncodeH264PictureInfoKHR *h264_picture_info = + vk_find_struct_const(enc_info->pNext, VIDEO_ENCODE_H264_PICTURE_INFO_KHR); + const VkVideoEncodeH264NaluSliceInfoKHR *h264_slice = &h264_picture_info->pNaluSliceEntries[0]; + const StdVideoEncodeH264SliceHeader *slice = h264_slice->pStdSliceHeader; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.deblocking_filter_h264); + radeon_emit(cs, slice->disable_deblocking_filter_idc); + radeon_emit(cs, slice->slice_alpha_c0_offset_div2); + radeon_emit(cs, slice->slice_beta_offset_div2); + radeon_emit(cs, 0); // cb qp offset + radeon_emit(cs, 0); // cr qp offset + ENC_END; +} + +static void +radv_enc_quality_params(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.quality_params); + radeon_emit(cs, 0); + radeon_emit(cs, 0); + radeon_emit(cs, 0); + radeon_emit(cs, 0); + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_3) + radeon_emit(cs, 0); + ENC_END; +} + +static void +radv_enc_slice_header(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + uint32_t instruction[RENCODE_SLICE_HEADER_TEMPLATE_MAX_NUM_INSTRUCTIONS] = {0}; + uint32_t num_bits[RENCODE_SLICE_HEADER_TEMPLATE_MAX_NUM_INSTRUCTIONS] = {0}; + const struct VkVideoEncodeH264PictureInfoKHR *h264_picture_info = + vk_find_struct_const(enc_info->pNext, VIDEO_ENCODE_H264_PICTURE_INFO_KHR); + int slice_count = h264_picture_info->naluSliceEntryCount; + const StdVideoEncodeH264PictureInfo *pic = h264_picture_info->pStdPictureInfo; + const StdVideoH264SequenceParameterSet *sps = + vk_video_find_h264_enc_std_sps(&cmd_buffer->video.params->vk, pic->seq_parameter_set_id); + const StdVideoH264PictureParameterSet *pps = + vk_video_find_h264_enc_std_pps(&cmd_buffer->video.params->vk, pic->pic_parameter_set_id); + const VkVideoEncodeH264NaluSliceInfoKHR *slice_info = &h264_picture_info->pNaluSliceEntries[0]; + + unsigned int inst_index = 0; + unsigned int cdw_start = 0; + unsigned int cdw_filled = 0; + unsigned int bits_copied = 0; + + assert(slice_count <= 1); + + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.slice_header); + radv_enc_reset(cmd_buffer); + radv_enc_set_emulation_prevention(cmd_buffer, false); + + cdw_start = cs->cdw; + + if (pic->flags.IdrPicFlag) + radv_enc_code_fixed_bits(cmd_buffer, 0x65, 8); + else if (!pic->flags.is_reference) + radv_enc_code_fixed_bits(cmd_buffer, 0x01, 8); + else + radv_enc_code_fixed_bits(cmd_buffer, 0x41, 8); + + radv_enc_flush_headers(cmd_buffer); + instruction[inst_index] = RENCODE_HEADER_INSTRUCTION_COPY; + num_bits[inst_index] = enc->bits_output - bits_copied; + bits_copied = enc->bits_output; + inst_index++; + + instruction[inst_index] = RENCODE_H264_HEADER_INSTRUCTION_FIRST_MB; + inst_index++; + + switch (pic->primary_pic_type) { + case STD_VIDEO_H264_PICTURE_TYPE_I: + case STD_VIDEO_H264_PICTURE_TYPE_IDR: + default: + radv_enc_code_ue(cmd_buffer, 7); + break; + case STD_VIDEO_H264_PICTURE_TYPE_P: + radv_enc_code_ue(cmd_buffer, 5); + break; + case STD_VIDEO_H264_PICTURE_TYPE_B: + radv_enc_code_ue(cmd_buffer, 6); + break; + } + radv_enc_code_ue(cmd_buffer, 0x0); + radv_enc_code_fixed_bits(cmd_buffer, pic->frame_num % 32, sps->log2_max_frame_num_minus4 + 4); +#if 0 + if (enc->enc_pic.h264_enc_params.input_picture_structure != + RENCODE_H264_PICTURE_STRUCTURE_FRAME) { + radv_enc_code_fixed_bits(cmd_buffer, 0x1, 1); + radv_enc_code_fixed_bits(cmd_buffer, + enc->enc_pic.h264_enc_params.input_picture_structure == + RENCODE_H264_PICTURE_STRUCTURE_BOTTOM_FIELD + ? 1 + : 0, + 1); + } +#endif + + if (pic->flags.IdrPicFlag) + radv_enc_code_ue(cmd_buffer, pic->idr_pic_id); + + if (sps->pic_order_cnt_type == STD_VIDEO_H264_POC_TYPE_0) + radv_enc_code_fixed_bits(cmd_buffer, pic->PicOrderCnt % 32, sps->log2_max_pic_order_cnt_lsb_minus4 + 4); + + if (pps->flags.redundant_pic_cnt_present_flag) + radv_enc_code_ue(cmd_buffer, 0); + + if (pic->primary_pic_type == STD_VIDEO_H264_PICTURE_TYPE_B) { + radv_enc_code_fixed_bits(cmd_buffer, slice_info->pStdSliceHeader->flags.direct_spatial_mv_pred_flag, 1); + } + const StdVideoEncodeH264ReferenceListsInfo *ref_lists = pic->pRefLists; + /* ref_pic_list_modification() */ + if (pic->primary_pic_type != STD_VIDEO_H264_PICTURE_TYPE_IDR && + pic->primary_pic_type != STD_VIDEO_H264_PICTURE_TYPE_I) { + + /* num ref idx active override flag */ + radv_enc_code_fixed_bits(cmd_buffer, slice_info->pStdSliceHeader->flags.num_ref_idx_active_override_flag, 1); + if (slice_info->pStdSliceHeader->flags.num_ref_idx_active_override_flag) { + radv_enc_code_ue(cmd_buffer, ref_lists->num_ref_idx_l0_active_minus1); + if (pic->primary_pic_type == STD_VIDEO_H264_PICTURE_TYPE_B) + radv_enc_code_ue(cmd_buffer, ref_lists->num_ref_idx_l1_active_minus1); + } + + radv_enc_code_fixed_bits(cmd_buffer, ref_lists->flags.ref_pic_list_modification_flag_l0, 1); + if (ref_lists->flags.ref_pic_list_modification_flag_l0) { + for (unsigned op = 0; op < ref_lists->refList0ModOpCount; op++) { + const StdVideoEncodeH264RefListModEntry *entry = &ref_lists->pRefList0ModOperations[op]; + + radv_enc_code_ue(cmd_buffer, entry->modification_of_pic_nums_idc); + if (entry->modification_of_pic_nums_idc == + STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_SHORT_TERM_SUBTRACT || + entry->modification_of_pic_nums_idc == STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_SHORT_TERM_ADD) + radv_enc_code_ue(cmd_buffer, entry->abs_diff_pic_num_minus1); + else if (entry->modification_of_pic_nums_idc == STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_LONG_TERM) + radv_enc_code_ue(cmd_buffer, entry->long_term_pic_num); + } + } + + if (pic->primary_pic_type == STD_VIDEO_H264_PICTURE_TYPE_B) { + radv_enc_code_fixed_bits(cmd_buffer, ref_lists->flags.ref_pic_list_modification_flag_l1, 1); + if (ref_lists->flags.ref_pic_list_modification_flag_l1) { + for (unsigned op = 0; op < ref_lists->refList1ModOpCount; op++) { + const StdVideoEncodeH264RefListModEntry *entry = &ref_lists->pRefList1ModOperations[op]; + + radv_enc_code_ue(cmd_buffer, entry->modification_of_pic_nums_idc); + if (entry->modification_of_pic_nums_idc == + STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_SHORT_TERM_SUBTRACT || + entry->modification_of_pic_nums_idc == STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_SHORT_TERM_ADD) + radv_enc_code_ue(cmd_buffer, entry->abs_diff_pic_num_minus1); + else if (entry->modification_of_pic_nums_idc == STD_VIDEO_H264_MODIFICATION_OF_PIC_NUMS_IDC_LONG_TERM) + radv_enc_code_ue(cmd_buffer, entry->long_term_pic_num); + } + } + } + } + + if (pic->flags.IdrPicFlag) { + radv_enc_code_fixed_bits(cmd_buffer, 0x0, 1); + radv_enc_code_fixed_bits(cmd_buffer, pic->flags.long_term_reference_flag, 1); /* long_term_reference_flag */ + } else if (pic->flags.is_reference) { + radv_enc_code_fixed_bits(cmd_buffer, ref_lists->refPicMarkingOpCount > 0 ? 1 : 0, 1); + for (unsigned op = 0; op < ref_lists->refPicMarkingOpCount; op++) { + const StdVideoEncodeH264RefPicMarkingEntry *entry = &ref_lists->pRefPicMarkingOperations[op]; + radv_enc_code_ue(cmd_buffer, entry->memory_management_control_operation); + if (entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_UNMARK_SHORT_TERM || + entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_MARK_LONG_TERM) + radv_enc_code_ue(cmd_buffer, entry->difference_of_pic_nums_minus1); + if (entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_UNMARK_LONG_TERM) + radv_enc_code_ue(cmd_buffer, entry->long_term_pic_num); + if (entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_MARK_LONG_TERM || + entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_MARK_CURRENT_AS_LONG_TERM) + radv_enc_code_ue(cmd_buffer, entry->long_term_frame_idx); + if (entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_SET_MAX_LONG_TERM_INDEX) + radv_enc_code_ue(cmd_buffer, entry->max_long_term_frame_idx_plus1); + if (entry->memory_management_control_operation == STD_VIDEO_H264_MEM_MGMT_CONTROL_OP_END) + break; + } + } + + if (pic->primary_pic_type != STD_VIDEO_H264_PICTURE_TYPE_IDR && + pic->primary_pic_type != STD_VIDEO_H264_PICTURE_TYPE_I && pps->flags.entropy_coding_mode_flag) + radv_enc_code_ue(cmd_buffer, slice_info->pStdSliceHeader->cabac_init_idc); + + radv_enc_flush_headers(cmd_buffer); + instruction[inst_index] = RENCODE_HEADER_INSTRUCTION_COPY; + num_bits[inst_index] = enc->bits_output - bits_copied; + bits_copied = enc->bits_output; + inst_index++; + + instruction[inst_index] = RENCODE_H264_HEADER_INSTRUCTION_SLICE_QP_DELTA; + inst_index++; + + if (pps->flags.deblocking_filter_control_present_flag) { + radv_enc_code_ue(cmd_buffer, slice_info->pStdSliceHeader->disable_deblocking_filter_idc); + if (!slice_info->pStdSliceHeader->disable_deblocking_filter_idc) { + radv_enc_code_se(cmd_buffer, slice_info->pStdSliceHeader->slice_alpha_c0_offset_div2); + radv_enc_code_se(cmd_buffer, slice_info->pStdSliceHeader->slice_beta_offset_div2); + } + } + + radv_enc_flush_headers(cmd_buffer); + instruction[inst_index] = RENCODE_HEADER_INSTRUCTION_COPY; + num_bits[inst_index] = enc->bits_output - bits_copied; + bits_copied = enc->bits_output; + inst_index++; + + instruction[inst_index] = RENCODE_HEADER_INSTRUCTION_END; + + cdw_filled = cs->cdw - cdw_start; + for (int i = 0; i < RENCODE_SLICE_HEADER_TEMPLATE_MAX_TEMPLATE_SIZE_IN_DWORDS - cdw_filled; i++) + radeon_emit(cs, 0x00000000); + for (int j = 0; j < RENCODE_SLICE_HEADER_TEMPLATE_MAX_NUM_INSTRUCTIONS; j++) { + radeon_emit(cs, instruction[j]); + radeon_emit(cs, num_bits[j]); + } + ENC_END; +} + +static void +radv_enc_ctx(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *info) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_image_view *dpb_iv = NULL; + struct radv_image *dpb = NULL; + struct radv_image_plane *dpb_luma = NULL; + struct radv_image_plane *dpb_chroma = NULL; + uint64_t va = 0; + uint32_t luma_pitch = 0; + + if (info->pSetupReferenceSlot) + dpb_iv = radv_image_view_from_handle(info->pSetupReferenceSlot->pPictureResource->imageViewBinding); + else if (info->referenceSlotCount > 0) + dpb_iv = radv_image_view_from_handle(info->pReferenceSlots[0].pPictureResource->imageViewBinding); + + if (dpb_iv) { + dpb = dpb_iv->image; + dpb_luma = &dpb->planes[0]; + dpb_chroma = &dpb->planes[1]; + radv_cs_add_buffer(device->ws, cs, dpb->bindings[0].bo); + va = radv_buffer_get_va(dpb->bindings[0].bo) + dpb->bindings[0].offset; // TODO DPB resource + luma_pitch = dpb_luma->surface.u.gfx9.surf_pitch * dpb_luma->surface.blk_w; // rec_luma_pitch + } + + uint32_t swizzle_mode = 0; + + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) + swizzle_mode = RENCODE_REC_SWIZZLE_MODE_256B_D; + else if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_2) + swizzle_mode = RENCODE_REC_SWIZZLE_MODE_256B_S; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.ctx); + radeon_emit(cs, va >> 32); + radeon_emit(cs, va & 0xffffffff); + radeon_emit(cs, swizzle_mode); + radeon_emit(cs, luma_pitch); // rec_luma_pitch + radeon_emit(cs, luma_pitch); // rec_luma_pitch0); //rec_chromma_pitch + radeon_emit(cs, info->referenceSlotCount + 1); // num_reconstructed_pictures + + int i; + for (i = 0; i < info->referenceSlotCount + 1; i++) { + radeon_emit(cs, dpb_luma ? dpb_luma->surface.u.gfx9.surf_offset + i * dpb_luma->surface.u.gfx9.surf_slice_size + : 0); // luma offset + radeon_emit(cs, dpb_chroma + ? dpb_chroma->surface.u.gfx9.surf_offset + i * dpb_chroma->surface.u.gfx9.surf_slice_size + : 0); // chroma offset + + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) { + radeon_emit(cs, 0); /* unused offset 1 */ + radeon_emit(cs, 0); /* unused offset 2 */ + } + } + + for (; i < RENCODE_MAX_NUM_RECONSTRUCTED_PICTURES; i++) { + radeon_emit(cs, 0); + radeon_emit(cs, 0); + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) { + radeon_emit(cs, 0); /* unused offset 1 */ + radeon_emit(cs, 0); /* unused offset 2 */ + } + } + + if (pdev->enc_hw_ver == RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, 0); // colloc buffer offset + } + radeon_emit(cs, 0); // enc pic pre encode luma pitch + radeon_emit(cs, 0); // enc pic pre encode chroma pitch + + for (i = 0; i < RENCODE_MAX_NUM_RECONSTRUCTED_PICTURES; i++) { + radeon_emit(cs, 0); // pre encode luma offset + radeon_emit(cs, 0); // pre encode chroma offset + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) { + radeon_emit(cs, 0); /* unused offset 1 */ + radeon_emit(cs, 0); /* unused offset 2 */ + } + } + + if (pdev->enc_hw_ver == RADV_VIDEO_ENC_HW_2) { + radeon_emit(cs, 0); // enc pic yuv luma offset + radeon_emit(cs, 0); // enc pic yuv chroma offset + + radeon_emit(cs, 0); // two pass search center map offset + + // rgboffsets + radeon_emit(cs, 0); // red + radeon_emit(cs, 0); // green + radeon_emit(cs, 0); // blue + } else if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, 0); // red + radeon_emit(cs, 0); // green + radeon_emit(cs, 0); // blue + radeon_emit(cs, 0); // v3 two pass search center map offset + radeon_emit(cs, 0); + if (pdev->enc_hw_ver == RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, 0); + } + } + ENC_END; +} + +static void +radv_enc_bitstream(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, VkDeviceSize offset) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + uint64_t va = radv_buffer_get_va(buffer->bo) + buffer->offset; + radv_cs_add_buffer(device->ws, cs, buffer->bo); + + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.bitstream); + radeon_emit(cs, RENCODE_REC_SWIZZLE_MODE_LINEAR); + radeon_emit(cs, va >> 32); + radeon_emit(cs, va & 0xffffffff); + radeon_emit(cs, buffer->vk.size - offset); + radeon_emit(cs, offset); + ENC_END; +} + +static void +radv_enc_feedback(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + + radeon_emit(cs, pdev->vcn_enc_cmds.feedback); + radeon_emit(cs, RENCODE_FEEDBACK_BUFFER_MODE_LINEAR); + radeon_emit(cs, cmd_buffer->video.feedback_query_va >> 32); + radeon_emit(cs, cmd_buffer->video.feedback_query_va & 0xffffffff); + radeon_emit(cs, 16); // buffer_size + radeon_emit(cs, 40); // data_size + ENC_END; +} + +static void +radv_enc_intra_refresh(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.intra_refresh); + radeon_emit(cs, 0); // intra refresh mode + radeon_emit(cs, 0); // intra ref offset + radeon_emit(cs, 0); // intra region size + ENC_END; +} + +static void +radv_enc_rc_per_pic(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info, + rvcn_enc_rate_ctl_per_picture_t *per_pic) +{ + struct radv_video_session *vid = cmd_buffer->video.vid; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + + unsigned qp, min_qp, max_qp; + + qp = per_pic->qp_i; + min_qp = per_pic->min_qp_i; + max_qp = per_pic->max_qp_i; + + if (vid->enc_rate_control_default) { + switch (vid->vk.op) { + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: { + const struct VkVideoEncodeH264PictureInfoKHR *h264_picture_info = + vk_find_struct_const(enc_info->pNext, VIDEO_ENCODE_H264_PICTURE_INFO_KHR); + const VkVideoEncodeH264NaluSliceInfoKHR *h264_slice = &h264_picture_info->pNaluSliceEntries[0]; + min_qp = max_qp = qp = h264_slice->constantQp; + break; + } + default: + break; + } + } + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.rc_per_pic); + radeon_emit(cs, qp); // qp + radeon_emit(cs, min_qp); // min qp + radeon_emit(cs, max_qp); // max qp + radeon_emit(cs, per_pic->max_au_size_i); // max au + radeon_emit(cs, per_pic->enabled_filler_data); // enabled filler + radeon_emit(cs, per_pic->skip_frame_enable); // skip frame eanble + radeon_emit(cs, per_pic->enforce_hrd); // enforce hrd + ENC_END; +} + +static void +radv_enc_params(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + const struct VkVideoEncodeH264PictureInfoKHR *h264_picture_info = + vk_find_struct_const(enc_info->pNext, VIDEO_ENCODE_H264_PICTURE_INFO_KHR); + const StdVideoEncodeH264PictureInfo *h264_pic = h264_picture_info ? h264_picture_info->pStdPictureInfo : NULL; + struct radv_image_view *src_iv = radv_image_view_from_handle(enc_info->srcPictureResource.imageViewBinding); + struct radv_image *src_img = src_iv->image; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + uint64_t va = radv_buffer_get_va(src_img->bindings[0].bo) + src_img->bindings[0].offset; + uint64_t luma_va = va + src_img->planes[0].surface.u.gfx9.surf_offset; + uint64_t chroma_va = va + src_img->planes[1].surface.u.gfx9.surf_offset; + uint32_t pic_type; + unsigned int slot_idx = 0xffffffff; + + radv_cs_add_buffer(device->ws, cs, src_img->bindings[0].bo); + if (h264_pic) { + switch (h264_pic->primary_pic_type) { + case STD_VIDEO_H264_PICTURE_TYPE_P: + slot_idx = enc_info->pReferenceSlots[0].slotIndex; + pic_type = RENCODE_PICTURE_TYPE_P; + break; + case STD_VIDEO_H264_PICTURE_TYPE_B: + slot_idx = enc_info->pReferenceSlots[0].slotIndex; + pic_type = RENCODE_PICTURE_TYPE_B; + break; + case STD_VIDEO_H264_PICTURE_TYPE_I: + case STD_VIDEO_H264_PICTURE_TYPE_IDR: + default: + pic_type = RENCODE_PICTURE_TYPE_I; + break; + } + } else { + assert(0); + return; + } + + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.enc_params); + radeon_emit(cs, pic_type); // pic type + radeon_emit(cs, enc_info->dstBufferRange); // allowed max bitstream size + radeon_emit(cs, luma_va >> 32); + radeon_emit(cs, luma_va & 0xffffffff); + radeon_emit(cs, chroma_va >> 32); + radeon_emit(cs, chroma_va & 0xffffffff); + radeon_emit(cs, src_img->planes[0].surface.u.gfx9.surf_pitch); // luma pitch + radeon_emit(cs, src_img->planes[1].surface.u.gfx9.surf_pitch); // chroma pitch + radeon_emit(cs, src_img->planes[0].surface.u.gfx9.swizzle_mode); // swizzle mode + radeon_emit(cs, slot_idx); // ref0_idx + + if (enc_info->pSetupReferenceSlot) + radeon_emit(cs, enc_info->pSetupReferenceSlot->slotIndex); // reconstructed picture index + else + radeon_emit(cs, 0); + ENC_END; +} + +static void +radv_enc_params_h264(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.enc_params_h264); + + if (pdev->enc_hw_ver < RADV_VIDEO_ENC_HW_3) { + radeon_emit(cs, RENCODE_H264_PICTURE_STRUCTURE_FRAME); + radeon_emit(cs, RENCODE_H264_INTERLACING_MODE_PROGRESSIVE); + radeon_emit(cs, RENCODE_H264_PICTURE_STRUCTURE_FRAME); + radeon_emit(cs, 0xffffffff); // reference_picture1_index + } else { + // V3 + radeon_emit(cs, RENCODE_H264_PICTURE_STRUCTURE_FRAME); + radeon_emit(cs, 0); // input pic order cnt + radeon_emit(cs, RENCODE_H264_INTERLACING_MODE_PROGRESSIVE); + radeon_emit(cs, 0); // l0 ref pic0 pic_type + radeon_emit(cs, 0); // l0 ref pic0 is long term + radeon_emit(cs, 0); // l0 ref pic0 picture structure + radeon_emit(cs, 0); // l0 ref pic0 pic order cnt + radeon_emit(cs, 0xffffffff); // l0 ref pic1 index + radeon_emit(cs, 0); // l0 ref pic1 pic_type + radeon_emit(cs, 0); // l0 ref pic1 is long term + radeon_emit(cs, 0); // l0 ref pic1 picture structure + radeon_emit(cs, 0); // l0 ref pic1 pic order cnt + radeon_emit(cs, 0xffffffff); // l1 ref pic0 index + radeon_emit(cs, 0); // l1 ref pic0 pic_type + radeon_emit(cs, 0); // l1 ref pic0 is long term + radeon_emit(cs, 0); // l1 ref pic0 picture structure + radeon_emit(cs, 0); // l1 ref pic0 pic order cnt + } + ENC_END; +} + +static void +radv_enc_op_init(struct radv_cmd_buffer *cmd_buffer) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, RENCODE_IB_OP_INITIALIZE); + ENC_END; +} + +static void +radv_enc_op_close(struct radv_cmd_buffer *cmd_buffer) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, RENCODE_IB_OP_CLOSE_SESSION); + ENC_END; +} + +static void +radv_enc_op_enc(struct radv_cmd_buffer *cmd_buffer) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, RENCODE_IB_OP_ENCODE); + ENC_END; +} + +static void +radv_enc_op_init_rc(struct radv_cmd_buffer *cmd_buffer) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, RENCODE_IB_OP_INIT_RC); + ENC_END; +} + +static void +radv_enc_op_init_rc_vbv(struct radv_cmd_buffer *cmd_buffer) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, RENCODE_IB_OP_INIT_RC_VBV_BUFFER_LEVEL); + ENC_END; +} + +static void +radv_enc_op_preset(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + struct radeon_cmdbuf *cs = cmd_buffer->cs; + struct radv_video_session *vid = cmd_buffer->video.vid; + uint32_t preset_mode; + + if (vid->enc_preset_mode == RENCODE_PRESET_MODE_QUALITY) + preset_mode = RENCODE_IB_OP_SET_QUALITY_ENCODING_MODE; + else if (vid->enc_preset_mode == RENCODE_PRESET_MODE_BALANCE) + preset_mode = RENCODE_IB_OP_SET_BALANCE_ENCODING_MODE; + else + preset_mode = RENCODE_IB_OP_SET_SPEED_ENCODING_MODE; + ENC_BEGIN; + radeon_emit(cs, preset_mode); + ENC_END; +} + +static void +radv_enc_input_format(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.input_format); + radeon_emit(cs, 0); // input color volume + radeon_emit(cs, 0); // input color space + radeon_emit(cs, RENCODE_COLOR_RANGE_STUDIO); // input color range + radeon_emit(cs, 0); // input chroma subsampling + radeon_emit(cs, 0); // input chroma location + radeon_emit(cs, 0); // input color bit depth + radeon_emit(cs, 0); // input color packing format + ENC_END; +} + +static void +radv_enc_output_format(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radeon_cmdbuf *cs = cmd_buffer->cs; + ENC_BEGIN; + radeon_emit(cs, pdev->vcn_enc_cmds.output_format); + radeon_emit(cs, 0); // output color volume + radeon_emit(cs, RENCODE_COLOR_RANGE_STUDIO); // output color range + radeon_emit(cs, 0); // output chroma location + radeon_emit(cs, 0); // output color bit depth + ENC_END; +} + +static void +radv_enc_headers_h264(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + radv_enc_slice_header(cmd_buffer, enc_info); + radv_enc_params(cmd_buffer, enc_info); + radv_enc_params_h264(cmd_buffer); +} + +static void +begin(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + struct radv_video_session *vid = cmd_buffer->video.vid; + + radv_enc_session_info(cmd_buffer); + cmd_buffer->video.enc.total_task_size = 0; + radv_enc_task_info(cmd_buffer, false); + radv_enc_op_init(cmd_buffer); + radv_enc_session_init(cmd_buffer, enc_info); + if (vid->vk.op == VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR) { + radv_enc_slice_control(cmd_buffer, enc_info); + radv_enc_spec_misc_h264(cmd_buffer, enc_info); + radv_enc_deblocking_filter_h264(cmd_buffer, enc_info); + } + radv_enc_layer_control(cmd_buffer, &vid->rc_layer_control); + radv_enc_rc_session_init(cmd_buffer); + radv_enc_quality_params(cmd_buffer); + // temporal layers init + unsigned i = 0; + do { + radv_enc_layer_select(cmd_buffer, i); + radv_enc_rc_layer_init(cmd_buffer, &vid->rc_layer_init[i]); + radv_enc_layer_select(cmd_buffer, i); + radv_enc_rc_per_pic(cmd_buffer, enc_info, &vid->rc_per_pic[i]); + } while (++i < vid->rc_layer_control.num_temporal_layers); + radv_enc_op_init_rc(cmd_buffer); + radv_enc_op_init_rc_vbv(cmd_buffer); + radeon_emit_direct(cmd_buffer->cs, enc->task_size_offset, enc->total_task_size); +} + +static void +destroy(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_enc_state *enc = &cmd_buffer->video.enc; + radv_enc_session_info(cmd_buffer); + cmd_buffer->video.enc.total_task_size = 0; + radv_enc_task_info(cmd_buffer, false); + radv_enc_op_close(cmd_buffer); + radeon_emit_direct(cmd_buffer->cs, enc->task_size_offset, enc->total_task_size); +} + +static void +radv_vcn_encode_video(struct radv_cmd_buffer *cmd_buffer, const VkVideoEncodeInfoKHR *enc_info) +{ + VK_FROM_HANDLE(radv_buffer, dst_buffer, enc_info->dstBuffer); + struct radv_video_session *vid = cmd_buffer->video.vid; + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + struct radv_enc_state *enc = &cmd_buffer->video.enc; + + switch (vid->vk.op) { + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + break; + default: + assert(0); + return; + } + + begin(cmd_buffer, enc_info); + // before encode + // session info + radv_enc_session_info(cmd_buffer); + + cmd_buffer->video.enc.total_task_size = 0; + // task info + radv_enc_task_info(cmd_buffer, true); + + // encode headers + // ctx + if (vid->vk.op == VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR) { + radv_enc_headers_h264(cmd_buffer, enc_info); + radv_enc_ctx(cmd_buffer, enc_info); + } + // bitstream + radv_enc_bitstream(cmd_buffer, dst_buffer, enc_info->dstBufferOffset); + + // feedback + radv_enc_feedback(cmd_buffer); + + // v2 encode statistics + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_2) { + } + // intra_refresh + radv_enc_intra_refresh(cmd_buffer); + // v2 input format + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_2) { + radv_enc_input_format(cmd_buffer); + radv_enc_output_format(cmd_buffer); + } + // v2 output format + + // op_preset + radv_enc_op_preset(cmd_buffer, enc_info); + // op_enc + radv_enc_op_enc(cmd_buffer); + + radeon_emit_direct(cmd_buffer->cs, enc->task_size_offset, enc->total_task_size); + + destroy(cmd_buffer); +} + +static void +set_rate_control_defaults(struct radv_video_session *vid) +{ + uint32_t frame_rate_den = 1, frame_rate_num = 30; + vid->enc_rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE; + vid->rc_layer_control.num_temporal_layers = 1; + vid->rc_layer_control.max_num_temporal_layers = 1; + vid->rc_per_pic[0].qp_i = 26; + vid->rc_per_pic[0].min_qp_i = 0; + vid->rc_per_pic[0].max_qp_i = 51; + vid->rc_per_pic[0].max_au_size_i = 0; + vid->rc_per_pic[0].enabled_filler_data = 1; + vid->rc_per_pic[0].skip_frame_enable = 0; + vid->rc_per_pic[0].enforce_hrd = 1; + vid->rc_layer_init[0].frame_rate_den = frame_rate_den; + vid->rc_layer_init[0].frame_rate_num = frame_rate_num; + vid->rc_layer_init[0].vbv_buffer_size = 20000000; // rate_control->virtualBufferSizeInMs; + vid->rc_layer_init[0].target_bit_rate = 16000; + vid->rc_layer_init[0].peak_bit_rate = 32000; + vid->rc_layer_init[0].avg_target_bits_per_picture = + radv_vcn_per_frame_integer(16000, frame_rate_den, frame_rate_num); + vid->rc_layer_init[0].peak_bits_per_picture_integer = + radv_vcn_per_frame_integer(32000, frame_rate_den, frame_rate_num); + vid->rc_layer_init[0].peak_bits_per_picture_fractional = + radv_vcn_per_frame_frac(32000, frame_rate_den, frame_rate_num); + return; +} + +void +radv_video_enc_control_video_coding(struct radv_cmd_buffer *cmd_buffer, const VkVideoCodingControlInfoKHR *control_info) +{ + struct radv_video_session *vid = cmd_buffer->video.vid; + + switch (vid->vk.op) { + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + break; + default: + unreachable("Unsupported\n"); + } + + if (!control_info->pNext) { + set_rate_control_defaults(vid); + return; + } + const VkVideoEncodeRateControlInfoKHR *rate_control = (VkVideoEncodeRateControlInfoKHR *) + vk_find_struct_const(control_info->pNext, VIDEO_ENCODE_RATE_CONTROL_INFO_KHR); + if (!rate_control) { + set_rate_control_defaults(vid); + return; + } + const VkVideoEncodeH264RateControlInfoKHR *h264_rate_control = (VkVideoEncodeH264RateControlInfoKHR *) + vk_find_struct_const(rate_control->pNext, VIDEO_ENCODE_H264_RATE_CONTROL_INFO_KHR); + + vid->enc_rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE; + vid->enc_rate_control_default = false; + + if (rate_control->rateControlMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR) { + vid->enc_rate_control_default = true; + set_rate_control_defaults(vid); + return; + } + if (rate_control->rateControlMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR) + vid->enc_rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR; + else if (rate_control->rateControlMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR) + vid->enc_rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR; + + if (h264_rate_control) { + vid->rc_layer_control.max_num_temporal_layers = h264_rate_control->temporalLayerCount; + vid->rc_layer_control.num_temporal_layers = h264_rate_control->temporalLayerCount; + } + + for (unsigned l = 0; l < rate_control->layerCount; l++) { + const VkVideoEncodeRateControlLayerInfoKHR *layer = &rate_control->pLayers[l]; + const VkVideoEncodeH264RateControlLayerInfoKHR *h264_layer = (VkVideoEncodeH264RateControlLayerInfoKHR *) + vk_find_struct_const(layer->pNext, VIDEO_ENCODE_H264_RATE_CONTROL_LAYER_INFO_KHR); + uint32_t frame_rate_den, frame_rate_num; + vid->rc_layer_init[l].target_bit_rate = layer->averageBitrate; + vid->rc_layer_init[l].peak_bit_rate = layer->maxBitrate; + frame_rate_den = layer->frameRateDenominator; + frame_rate_num = layer->frameRateNumerator; + radv_vcn_enc_invalid_frame_rate(&frame_rate_den, &frame_rate_num); + vid->rc_layer_init[l].frame_rate_den = frame_rate_den; + vid->rc_layer_init[l].frame_rate_num = frame_rate_num; + vid->rc_layer_init[l].vbv_buffer_size = 20000000;//rate_control->virtualBufferSizeInMs; + vid->rc_layer_init[l].avg_target_bits_per_picture = radv_vcn_per_frame_integer(layer->averageBitrate, + frame_rate_den, frame_rate_num); + vid->rc_layer_init[l].peak_bits_per_picture_integer = radv_vcn_per_frame_integer(layer->maxBitrate, + frame_rate_den, frame_rate_num); + vid->rc_layer_init[l].peak_bits_per_picture_fractional = radv_vcn_per_frame_frac(layer->maxBitrate, + frame_rate_den, frame_rate_num); + + if (h264_layer) { + vid->rc_per_pic[l].min_qp_i = h264_layer->useMinQp ? h264_layer->minQp.qpI : 0; + vid->rc_per_pic[l].max_qp_i = h264_layer->useMaxQp ? h264_layer->maxQp.qpI : 51; + vid->rc_per_pic[l].max_au_size_i = h264_layer->useMaxFrameSize ? h264_layer->maxFrameSize.frameISize : 0; + } + vid->rc_per_pic[l].qp_i = vid->rc_per_pic[l].max_qp_i; + vid->rc_per_pic[l].enabled_filler_data = 1; + vid->rc_per_pic[l].skip_frame_enable = 0; + vid->rc_per_pic[l].enforce_hrd = 1; + } +} + +VKAPI_ATTR void VKAPI_CALL +radv_CmdEncodeVideoKHR(VkCommandBuffer commandBuffer, const VkVideoEncodeInfoKHR *pEncodeInfo) +{ + VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); + radv_vcn_encode_video(cmd_buffer, pEncodeInfo); +} + +VKAPI_ATTR VkResult VKAPI_CALL +radv_GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR( + VkPhysicalDevice physicalDevice, const VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR *pQualityLevelInfo, + VkVideoEncodeQualityLevelPropertiesKHR *pQualityLevelProperties) +{ + return VK_SUCCESS; +} + +void +radv_video_patch_encode_session_parameters(struct vk_video_session_parameters *params) +{ + switch (params->op) { + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: + default: + break; + } +} + +VKAPI_ATTR VkResult VKAPI_CALL +radv_GetEncodedVideoSessionParametersKHR(VkDevice device, + const VkVideoEncodeSessionParametersGetInfoKHR *pVideoSessionParametersInfo, + VkVideoEncodeSessionParametersFeedbackInfoKHR *pFeedbackInfo, + size_t *pDataSize, void *pData) +{ + VK_FROM_HANDLE(radv_video_session_params, templ, pVideoSessionParametersInfo->videoSessionParameters); + size_t total_size = 0; + size_t size_limit = 0; + + if (pData) + size_limit = *pDataSize; + + switch (templ->vk.op) { + case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: { + const struct VkVideoEncodeH264SessionParametersGetInfoKHR *h264_get_info = + vk_find_struct_const(pVideoSessionParametersInfo->pNext, VIDEO_ENCODE_H264_SESSION_PARAMETERS_GET_INFO_KHR); + if (h264_get_info->writeStdSPS) { + const StdVideoH264SequenceParameterSet *sps = + vk_video_find_h264_enc_std_sps(&templ->vk, h264_get_info->stdSPSId); + assert(sps); + vk_video_encode_h264_sps(sps, size_limit, &total_size, pData); + } + if (h264_get_info->writeStdPPS) { + const StdVideoH264PictureParameterSet *pps = + vk_video_find_h264_enc_std_pps(&templ->vk, h264_get_info->stdPPSId); + assert(pps); + vk_video_encode_h264_pps(pps, templ->vk.h264_enc.profile_idc == STD_VIDEO_H264_PROFILE_IDC_HIGH, size_limit, + &total_size, pData); + } + break; + } + default: + break; + } + + *pDataSize = total_size; + return VK_SUCCESS; +} + +void +radv_video_enc_begin_coding(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + radeon_check_space(device->ws, cmd_buffer->cs, 1024); + + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) + radv_vcn_sq_header(cmd_buffer->cs, &cmd_buffer->video.sq, true); +} + +void +radv_video_enc_end_coding(struct radv_cmd_buffer *cmd_buffer) +{ + struct radv_device *device = radv_cmd_buffer_device(cmd_buffer); + const struct radv_physical_device *pdev = radv_device_physical(device); + if (pdev->enc_hw_ver >= RADV_VIDEO_ENC_HW_4) + radv_vcn_sq_tail(cmd_buffer->cs, &cmd_buffer->video.sq); +} + +#define VCN_ENC_SESSION_SIZE 128 * 1024 + +VkResult +radv_video_get_encode_session_memory_requirements(struct radv_device *device, struct radv_video_session *vid, + uint32_t *pMemoryRequirementsCount, + VkVideoSessionMemoryRequirementsKHR *pMemoryRequirements) +{ + struct radv_physical_device *pdev = radv_device_physical(device); + uint32_t memory_type_bits = (1u << pdev->memory_properties.memoryTypeCount) - 1; + + VK_OUTARRAY_MAKE_TYPED(VkVideoSessionMemoryRequirementsKHR, out, pMemoryRequirements, pMemoryRequirementsCount); + + vk_outarray_append_typed(VkVideoSessionMemoryRequirementsKHR, &out, m) + { + m->memoryBindIndex = 0; + m->memoryRequirements.size = VCN_ENC_SESSION_SIZE; + m->memoryRequirements.alignment = 0; + m->memoryRequirements.memoryTypeBits = memory_type_bits; + } + + return vk_outarray_status(&out); +}